diff --git a/build/tasks.ps1 b/build/tasks.ps1 index 46491728..f76b844a 100644 --- a/build/tasks.ps1 +++ b/build/tasks.ps1 @@ -8,6 +8,8 @@ $dist_folder = "$root\dist" $msbuild_verbosity = "n" $projects = @( + "Sa.Utils.WorkQueue", + "Sa.Media", "Sa.Media.FFmpeg", diff --git a/src/.gitignore b/src/.gitignore index 1ce70e5a..c8e63aed 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -405,4 +405,5 @@ FodyWeavers.xsd Sa.Media.FFmpeg/runtimes/native/ nupkgs/ -.packages/ \ No newline at end of file +.packages/ +*.lscache \ No newline at end of file diff --git a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs index 80576f00..29a1974a 100644 --- a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs +++ b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs @@ -19,7 +19,7 @@ private async Task LoadAsync(PostgreSqlConfigurationOptions options) { try { - using var dataSource = new PgDataSource(new(options.ConnectionString)); + using var dataSource = IPgDataSource.Create(options.ConnectionString); return await dataSource.ExecuteReader(options.SelectSql, (reader, _) => { diff --git a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj index 5e15b30f..9fa80b67 100644 --- a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj +++ b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 add a PostgreSQL-based configuration source to IConfiguration diff --git a/src/Sa.Configuration/Sa.Configuration.csproj b/src/Sa.Configuration/Sa.Configuration.csproj index 02f9ca02..937ac6e3 100644 --- a/src/Sa.Configuration/Sa.Configuration.csproj +++ b/src/Sa.Configuration/Sa.Configuration.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 extensions for Configuration diff --git a/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs b/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs index e4377287..0b022182 100644 --- a/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs +++ b/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs @@ -1,10 +1,7 @@ - -namespace Sa.Data.PostgreSql; +namespace Sa.Data.PostgreSql; public interface IPgDataSourceSettingsBuilder { void WithConnectionString(string connectionString); void WithConnectionString(Func implementationFactory); - void WithSettings(PgDataSourceSettings settings); - void WithSettings(Func implementationFactory); } diff --git a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs index 98039b3e..6fa2dd79 100644 --- a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs +++ b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs @@ -1,6 +1,34 @@ -namespace Sa.Data.PostgreSql; +using Npgsql; -public sealed class PgDataSourceSettings(string connectionString) +namespace Sa.Data.PostgreSql; + +internal sealed class PgDataSourceSettings(string connectionString) { - public string ConnectionString { get; } = connectionString; + private string? _searchPath; + + public string ConnectionString => connectionString; + + public string GetSearchPath() + { + if (_searchPath is not null) + return _searchPath; + + try + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + return _searchPath = builder.SearchPath ?? "public"; + } + catch + { + return _searchPath = "public"; + } + } + + public void Validate() + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new InvalidOperationException("Connection string cannot be null or empty."); + } + } } diff --git a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs index db56cba1..0f376739 100644 --- a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs +++ b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs @@ -6,22 +6,14 @@ namespace Sa.Data.PostgreSql.Configuration; internal sealed class PgDataSourceSettingsBuilder(IServiceCollection services) : IPgDataSourceSettingsBuilder { public void WithConnectionString(string connectionString) - { - services.TryAddSingleton(new PgDataSourceSettings(connectionString)); - } + => services.TryAddSingleton(new PgDataSourceSettings(connectionString)); public void WithConnectionString(Func implementationFactory) - { - services.TryAddSingleton(sp => new PgDataSourceSettings(implementationFactory(sp))); - } + => services.TryAddSingleton(sp => new PgDataSourceSettings(implementationFactory(sp))); public void WithSettings(Func implementationFactory) - { - services.TryAddSingleton(implementationFactory); - } + => services.TryAddSingleton(implementationFactory); public void WithSettings(PgDataSourceSettings settings) - { - services.TryAddSingleton(settings); - } + => services.TryAddSingleton(settings); } diff --git a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs index e63c57bf..cf1e0731 100644 --- a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs +++ b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs @@ -1,6 +1,6 @@ -using System.Collections.ObjectModel; +using Npgsql; +using System.Collections.ObjectModel; using System.Runtime.CompilerServices; -using Npgsql; namespace Sa.Data.PostgreSql; diff --git a/src/Sa.Data.PostgreSql/IPgDataSource.cs b/src/Sa.Data.PostgreSql/IPgDataSource.cs index 8427f7c2..11978fc2 100644 --- a/src/Sa.Data.PostgreSql/IPgDataSource.cs +++ b/src/Sa.Data.PostgreSql/IPgDataSource.cs @@ -2,46 +2,71 @@ namespace Sa.Data.PostgreSql; -public interface IPgDataSource +public interface IPgDataSource : IDisposable, IAsyncDisposable { - public static IPgDataSource Create(string connectionString) => new PgDataSource(new PgDataSourceSettings(connectionString)); + public static IPgDataSource Create(string connectionString) + => new PgDataSource(new PgDataSourceSettings(connectionString)); + + string GetSearchPath(); ValueTask OpenDbConnection(CancellationToken cancellationToken); - Task ExecuteNonQuery(string sql, Action? initCommand, CancellationToken cancellationToken = default); + Task ExecuteNonQuery( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default); - Task ExecuteNonQuery(string sql, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) - => ExecuteNonQuery(sql, cmd => FillParams(cmd, parameters), cancellationToken); + Task ExecuteNonQuery( + string sql, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) + => ExecuteNonQuery(sql, cmd => FillParams(cmd, parameters), cancellationToken); Task ExecuteNonQuery(string sql, CancellationToken cancellationToken = default) => ExecuteNonQuery(sql, [], cancellationToken); - Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default); + Task ExecuteScalar( + string sql, Action? initCommand, CancellationToken cancellationToken = default); - async Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default) - => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); + async Task ExecuteScalar( + string sql, Action? initCommand, CancellationToken cancellationToken = default) + => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); // ExecuteReader - Task ExecuteReader(string sql, Action read, Action? initCommand, CancellationToken cancellationToken = default); - - Task ExecuteReader(string sql, Action read, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + Task ExecuteReader( + string sql, + Action read, + Action? initCommand, + CancellationToken cancellationToken = default); + + Task ExecuteReader( + string sql, + Action read, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) => ExecuteReader(sql, read, cmd => FillParams(cmd, parameters), cancellationToken); - async Task ExecuteReader(string sql, Action read, CancellationToken cancellationToken = default) + async Task ExecuteReader( + string sql, Action read, CancellationToken cancellationToken = default) => await ExecuteReader(sql, read, [], cancellationToken); // ExecuteReaderList - async Task> ExecuteReaderList(string sql, Func read, CancellationToken cancellationToken = default) + async Task> ExecuteReaderList( + string sql, Func read, CancellationToken cancellationToken = default) { List list = []; await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), cancellationToken); return list; } - async Task> ExecuteReaderList(string sql, Func read, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + async Task> ExecuteReaderList( + string sql, + Func read, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) { List list = []; await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), parameters, cancellationToken); @@ -57,7 +82,10 @@ Task ExecuteReaderFirst(string sql, CancellationToken cancellationToken = return ExecuteReaderFirst(sql, [], cancellationToken); } - async Task ExecuteReaderFirst(string sql, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + async Task ExecuteReaderFirst( + string sql, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) { T value = default!; @@ -85,9 +113,10 @@ await ExecuteReader(sql, (reader, _) => // BeginBinaryImport - - - ValueTask BeginBinaryImport(string sql, Func> write, CancellationToken cancellationToken = default); + ValueTask BeginBinaryImport( + string sql, + Func> write, + CancellationToken cancellationToken = default); void FillParams(NpgsqlCommand cmd, IReadOnlyCollection parameters) { diff --git a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs b/src/Sa.Data.PostgreSql/IPgDistributedLock.cs index 5ebf8989..1d4eec50 100644 --- a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs +++ b/src/Sa.Data.PostgreSql/IPgDistributedLock.cs @@ -2,5 +2,8 @@ public interface IPgDistributedLock { - Task TryExecuteInDistributedLock(long lockId, Func exclusiveLockTask, CancellationToken cancellationToken); + Task TryExecuteInDistributedLock( + long lockId, + Func exclusiveLockTask, + CancellationToken cancellationToken); } diff --git a/src/Sa.Data.PostgreSql/PgDataSource.cs b/src/Sa.Data.PostgreSql/PgDataSource.cs index 96f4a311..40fe0980 100644 --- a/src/Sa.Data.PostgreSql/PgDataSource.cs +++ b/src/Sa.Data.PostgreSql/PgDataSource.cs @@ -6,11 +6,15 @@ namespace Sa.Data.PostgreSql; /// NpgsqlDataSource lite /// /// connection string -public sealed class PgDataSource(PgDataSourceSettings settings) : IPgDataSource, IDisposable, IAsyncDisposable +internal sealed class PgDataSource(PgDataSourceSettings settings) : IPgDataSource { - private readonly Lazy _dataSource = new(() => NpgsqlDataSource.Create(settings.ConnectionString)); + private readonly Lazy _dataSource + = new(() => NpgsqlDataSource.Create(settings.ConnectionString)); - public ValueTask OpenDbConnection(CancellationToken cancellationToken) => _dataSource.Value.OpenConnectionAsync(cancellationToken); + public string GetSearchPath() => settings.GetSearchPath(); + + public ValueTask OpenDbConnection(CancellationToken cancellationToken) + => _dataSource.Value.OpenConnectionAsync(cancellationToken); public void Dispose() { @@ -28,7 +32,10 @@ public async ValueTask DisposeAsync() } } - public async ValueTask BeginBinaryImport(string sql, Func> write, CancellationToken cancellationToken = default) + public async ValueTask BeginBinaryImport( + string sql, + Func> write, + CancellationToken cancellationToken = default) { using NpgsqlConnection db = await OpenDbConnection(cancellationToken); using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken); @@ -36,7 +43,10 @@ public async ValueTask BeginBinaryImport(string sql, Func ExecuteNonQuery(string sql, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteNonQuery( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default) { using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); using NpgsqlCommand cmd = new(sql, connection); @@ -44,7 +54,10 @@ public async Task ExecuteNonQuery(string sql, Action? initCo return await cmd.ExecuteNonQueryAsync(cancellationToken); } - public async Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteScalar( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default) { using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); using NpgsqlCommand cmd = new(sql, connection); @@ -52,7 +65,11 @@ public async Task ExecuteNonQuery(string sql, Action? initCo return await cmd.ExecuteScalarAsync(cancellationToken); } - public async Task ExecuteReader(string sql, Action read, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteReader( + string sql, + Action read, + Action? initCommand, + CancellationToken cancellationToken = default) { int rowCount = 0; diff --git a/src/Sa.Data.PostgreSql/PgDistributedLock.cs b/src/Sa.Data.PostgreSql/PgDistributedLock.cs index 84eaab17..3b49dd87 100644 --- a/src/Sa.Data.PostgreSql/PgDistributedLock.cs +++ b/src/Sa.Data.PostgreSql/PgDistributedLock.cs @@ -9,7 +9,9 @@ namespace Sa.Data.PostgreSql; /// /// /// -internal sealed partial class PgDistributedLock(PgDataSourceSettings settings, ILogger? logger = null) : IPgDistributedLock +internal sealed partial class PgDistributedLock( + PgDataSourceSettings settings, + ILogger? logger = null) : IPgDistributedLock { private readonly ILogger _logger = logger ?? NullLogger.Instance; diff --git a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs index 4f38ad17..1e36b9ac 100644 --- a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs +++ b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs @@ -15,7 +15,9 @@ public static ValueTask ExecuteWithRetry( fun: fun, retryCount: retryCount, initialDelay: initialDelay - , next: (ex, i) => next != null ? next(ex, i) : (ex is NpgsqlException exception) && exception.IsTransient + , next: (ex, i) => next != null + ? next(ex, i) + : (ex is NpgsqlException exception) && exception.IsTransient , cancellationToken: cancellationToken); } } diff --git a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj index 88bc49b8..1862684d 100644 --- a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj +++ b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple client for Npqsql diff --git a/src/Sa.Data.PostgreSql/Setup.cs b/src/Sa.Data.PostgreSql/Setup.cs index 8cb25480..66e5ce5c 100644 --- a/src/Sa.Data.PostgreSql/Setup.cs +++ b/src/Sa.Data.PostgreSql/Setup.cs @@ -1,16 +1,33 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Npgsql; using Sa.Data.PostgreSql.Configuration; namespace Sa.Data.PostgreSql; public static class Setup { - public static IServiceCollection AddSaPostgreSqlDataSource(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddSaPostgreSqlDataSource( + this IServiceCollection services, + Action? configure = null) { PgDataSourceSettingsBuilder builder = new(services); configure?.Invoke(builder); - services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + PgDataSourceSettings? settings = sp.GetService(); + + if (settings is null) + { + var connection = sp.GetService()?.ConnectionString + ?? throw new InvalidOperationException("Empty connection string"); + settings = new(connection); + } + + settings.Validate(); + + return new PgDataSource(settings); + }); services.TryAddSingleton(); return services; } diff --git a/src/Sa.Data.S3/S3BucketSettings.cs b/src/Sa.Data.S3/S3BucketSettings.cs index b48aec41..9fae5e49 100644 --- a/src/Sa.Data.S3/S3BucketSettings.cs +++ b/src/Sa.Data.S3/S3BucketSettings.cs @@ -6,7 +6,7 @@ public class S3BucketSettings { public required string AccessKey { get; init; } - + public required string SecretKey { get; init; } public required string Bucket { get; init; } /// @@ -17,7 +17,6 @@ public class S3BucketSettings public string Region { get; init; } = "us-east-1"; - public required string SecretKey { get; init; } public string Service { get; init; } = "s3"; diff --git a/src/Sa.Data.S3/Sa.Data.S3.csproj b/src/Sa.Data.S3/Sa.Data.S3.csproj index 13c1c00e..8bb96edb 100644 --- a/src/Sa.Data.S3/Sa.Data.S3.csproj +++ b/src/Sa.Data.S3/Sa.Data.S3.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.9.0 Sa.Data.S3 Simple client for S3 (Sa.Data.S3) s3 diff --git a/src/Sa.Data.S3/Setup.cs b/src/Sa.Data.S3/Setup.cs index 3078f053..224c1834 100644 --- a/src/Sa.Data.S3/Setup.cs +++ b/src/Sa.Data.S3/Setup.cs @@ -11,19 +11,15 @@ public static IServiceCollection AddSaS3BucketClient( { services.TryAddSingleton(settings); - // https://www.milanjovanovic.tech/blog/the-right-way-to-use-httpclient-in-dotnet services .AddHttpClient((sp, client) => { client.Timeout = settings.TotalRequestTimeout; client.BaseAddress = new Uri(settings.Endpoint); }) - .ConfigurePrimaryHttpMessageHandler(() => + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler() { - return new SocketsHttpHandler() - { - PooledConnectionLifetime = TimeSpan.FromMinutes(15) - }; + PooledConnectionLifetime = TimeSpan.FromMinutes(15) }) .SetHandlerLifetime(Timeout.InfiniteTimeSpan) .AddStandardResilienceHandler(options => diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index bb38ba59..b84a92df 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -4,6 +4,9 @@ using System.Globalization; using System.Runtime.CompilerServices; +/// +/// fs://share/tenant/filename +/// internal sealed class FileSystemStorage( FileSystemStorageSettings settings, TimeProvider? timeProvider = null) : IFileStorage @@ -11,7 +14,10 @@ internal sealed class FileSystemStorage( private const string SchemeSeparator = "://"; private readonly string _basePath = Path.TrimEndingDirectorySeparator( - Path.GetFullPath(Path.Combine(settings.BasePath, settings.ScopeName))); + Path.GetFullPath(settings.BasePath)); + + private readonly string _basePathScope = Path.TrimEndingDirectorySeparator( + Path.GetFullPath(Path.Combine(settings.BasePath, settings.Basket))); private readonly string _schemePrefix = $"{settings.StorageType}{SchemeSeparator}"; private readonly string _storageType = settings.StorageType; @@ -19,7 +25,7 @@ internal sealed class FileSystemStorage( private readonly int _bufferSize = settings.BufferSize; private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; - public string ScopeName => settings.ScopeName; + public string Basket => settings.Basket; public string StorageType => _storageType; public bool IsReadOnly => _isReadOnly; @@ -53,7 +59,7 @@ public async Task UploadAsync( string filename = PathSanitizer.SanitizeRelativePath(metadata.FileName); - string relativePath = string.Concat(metadata.TenantId.ToString(), "/", filename); + string relativePath = string.Concat(Basket, "/", metadata.TenantId.ToString(), "/", filename); string filePath = Path.Combine(_basePath, relativePath); EnsurePathWithinBase(filePath); @@ -75,10 +81,10 @@ public async Task UploadAsync( await fileStream.CopyToAsync(fileStreamOutput, cancellationToken).ConfigureAwait(false); return new StorageResult( - string.Concat(_schemePrefix, relativePath), - Path.GetFullPath(filePath), - _storageType, - _timeProvider.GetUtcNow()); + FileId: string.Concat(_schemePrefix, relativePath), + AbsoluteUrl: Path.GetFullPath(filePath), + StorageType: _storageType, + UploadedAt: _timeProvider.GetUtcNow()); } public async Task DownloadAsync( @@ -187,7 +193,7 @@ private bool IsPathWithinBase(string path) { return Path.GetFullPath(path) .AsSpan() - .StartsWith(_basePath.AsSpan(), StringComparison.Ordinal); + .StartsWith(_basePathScope.AsSpan(), StringComparison.Ordinal); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -206,7 +212,7 @@ private void EnsurePathWithinBase(string path) if (!CanProcess(fileId)) return Task.FromResult(null); - // Парсинг формата: "storageType://tenantId/filename" + //parse: "storageType://basket/tenant/filename" ReadOnlySpan span = fileId.AsSpan(); int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); if (schemeEnd == -1) @@ -214,19 +220,26 @@ private void EnsurePathWithinBase(string path) var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; - // Парсинг "tenantId/filename" + // "tenantId/filename" int slashIndex = pathPart.IndexOf('/'); if (slashIndex == -1) return Task.FromResult(null); - var tenantSpan = pathPart[..slashIndex]; - var fileNameSpan = pathPart[(slashIndex + 1)..]; + var scopeSpan = pathPart[..slashIndex]; + + var nextSpan = pathPart[(slashIndex + 1)..]; + slashIndex = nextSpan.IndexOf('/'); + + var tenantSpan = nextSpan[..slashIndex]; + var fileNameSpan = nextSpan[(slashIndex + 1)..]; if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) return Task.FromResult(null); var metadata = new FileMetadata { + StorageType = StorageType, + Basket = scopeSpan.ToString(), FileName = fileNameSpan.ToString(), TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs index 16ff528c..5c1143e8 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs @@ -11,5 +11,63 @@ public sealed record FileSystemStorageOptions [StringLength(255)] public string BasePath { get; set; } = string.Empty; public bool IsReadOnly { get; set; } = false; - public string? ScopeName { get; set; } + [Required] + [StringLength(63, MinimumLength = 3)] + public string Basket { get; set; } = FileSystemStorageSettings.DefaultBasket; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(BasePath)) + { + throw new ValidationException("BasePath cannot be empty."); + } + + try + { + // Get the full path to resolve any relative paths + string fullPath = Path.GetFullPath(BasePath); + + // Check if the path contains any invalid characters + if (BasePath.IndexOfAny(Path.GetInvalidPathChars()) >= 0) + { + throw new ValidationException($"BasePath contains invalid characters: {BasePath}"); + } + } + catch (Exception ex) + { + throw new ValidationException($"Invalid BasePath format: {BasePath}. {ex.Message}"); + } + + if (string.IsNullOrWhiteSpace(Basket)) + { + throw new ValidationException("Basket cannot be empty."); + } + + + if (Basket.Length > 63 || Basket.Length < 3) + { + throw new ValidationException($"Basket exceeds maximum length of 63 characters."); + } + + if (!char.IsLetter(Basket[0]) && Basket[0] != '_') + { + throw new ValidationException("Basket must start with a letter or underscore."); + } + + + if (string.IsNullOrWhiteSpace(Basket)) + { + throw new ValidationException("Basket cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(StorageType)) + { + throw new ValidationException("StorageType cannot be empty."); + } + + if (StorageType.Length > 10) + { + throw new ValidationException($"StorageType exceeds maximum length of 63 characters."); + } + } } diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs index c4a0e4c6..e563b53f 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs @@ -4,19 +4,17 @@ namespace Sa.HybridFileStorage.FileSystem; public sealed record FileSystemStorageSettings { - [Required] public string StorageType { get; init; } = DefaultStorageType; - [StringLength(255)] + public string Basket { get; init; } = DefaultBasket; + public required string BasePath { get; init; } - [StringLength(100, MinimumLength = 1)] public bool IsReadOnly { get; init; } = false; - public string ScopeName { get; init; } = string.Empty; - public int BufferSize { get; init; } = 256 * 1024; public const string DefaultStorageType = "fs"; + public const string DefaultBasket = "share"; } diff --git a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj index a458697d..5e476585 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj +++ b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management diff --git a/src/Sa.HybridFileStorage.FileSystem/Setup.cs b/src/Sa.HybridFileStorage.FileSystem/Setup.cs index 7bfc7000..2f903c96 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Setup.cs +++ b/src/Sa.HybridFileStorage.FileSystem/Setup.cs @@ -23,12 +23,13 @@ public static IServiceCollection AddSaFileSystemFileStorage( { FileSystemStorageOptions options = new(); configure.Invoke(sp, options); + options.Validate(); return new FileSystemStorage(new FileSystemStorageSettings { BasePath = options.BasePath, IsReadOnly = options.IsReadOnly, - ScopeName = options.ScopeName ?? string.Empty, + Basket = options.Basket, StorageType = options.StorageType, }, sp.GetService()); }); diff --git a/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs b/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs deleted file mode 100644 index 152a8c72..00000000 --- a/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Sa.HybridFileStorage; -using System.Diagnostics.CodeAnalysis; -using System.Security; - -//static class ThrowHelper -//{ -// [DoesNotReturn] -// public static void ThrowWritableException() => -// throw new HybridFileStorageWritableException(); - -// [DoesNotReturn] -// public static void ThrowInvalidFileIdFormat() => -// throw new FormatException("Invalid file ID format."); - -// [DoesNotReturn] -// public static void ThrowSecurityException(string path, string basePath) => -// throw new SecurityException( -// $"Access denied. Path '{path}' is outside the allowed base directory '{basePath}'."); -//} diff --git a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs index 909c0ef0..662cf157 100644 --- a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs +++ b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs @@ -6,8 +6,9 @@ internal static class FileIdParser { public const string SchemeSeparator = "://"; - public static bool TryParseFileIdWithFilename( + public static bool TryParse( string fileId, + out string basket, out int tenantId, out long timestamp, out string fileName) @@ -15,6 +16,7 @@ public static bool TryParseFileIdWithFilename( tenantId = default; timestamp = default; fileName = string.Empty; + basket = string.Empty; if (string.IsNullOrEmpty(fileId)) return false; @@ -22,19 +24,23 @@ public static bool TryParseFileIdWithFilename( int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); if (schemeEnd == -1) return false; - var afterScheme = span[(schemeEnd + SchemeSeparator.Length)..]; - int tableEnd = afterScheme.IndexOf('/'); - if (tableEnd == -1) return false; - afterScheme = afterScheme[(tableEnd + 1)..]; + var afterSpan = span[(schemeEnd + SchemeSeparator.Length)..]; + int scopeEnd = afterSpan.IndexOf('/'); + if (scopeEnd == -1) return false; - int tenantEnd = afterScheme.IndexOf('/'); + basket = afterSpan[..scopeEnd].ToString(); + + afterSpan = afterSpan[(scopeEnd + 1)..]; + + + int tenantEnd = afterSpan.IndexOf('/'); if (tenantEnd == -1) return false; - var tenantSpan = afterScheme[..tenantEnd]; + var tenantSpan = afterSpan[..tenantEnd]; if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out tenantId)) return false; - var afterTenant = afterScheme[(tenantEnd + 1)..]; + var afterTenant = afterSpan[(tenantEnd + 1)..]; int timestampEnd = afterTenant.IndexOf('/'); if (timestampEnd == -1) return false; @@ -49,13 +55,15 @@ public static bool TryParseFileIdWithFilename( public static string FormatToFileId( string storageType, - string tableName, + string basket, int tenantId, DateTimeOffset date, string fileName) - => $"{storageType}://{tableName}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; + => $"{storageType}://{basket}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; - public static string NormalizeFileName(string fileName) => fileName.TrimStart('\\', '/').Replace('\\', '/'); + public static string NormalizeFileName(string fileName) + => fileName.TrimStart('\\', '/').Replace('\\', '/'); - public static string GetFileExtension(string fileName) => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); + public static string GetFileExtension(string fileName) + => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); } diff --git a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs index 1f2f1f90..bdf74c52 100644 --- a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs @@ -6,7 +6,6 @@ public interface IPostgresFileStorageConfiguration { IPostgresFileStorageConfiguration AddDataSource(Action? configure = null); IPostgresFileStorageConfiguration ConfigureOptions(Action configure); - IPostgresFileStorageConfiguration ConfigureOptions(Action configure); IPostgresFileStorageConfiguration WithStorageType(string storageType); IPostgresFileStorageConfiguration WithSchemaName(string schemaName); IPostgresFileStorageConfiguration WithTableName(string tableName); diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index 98a2d0e2..afe57568 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -7,20 +7,23 @@ namespace Sa.HybridFileStorage.Postgres; +/// +/// pg://scope/tenant/timestamp/filename +/// internal sealed class PostgresFileStorage( IPgDataSource dataSource, IPartitionManager partManager, RecyclableMemoryStreamManager streamManager, StorageOptions options, - string scopeName, + string basket, TimeProvider? timeProvider = null) : IFileStorage { private const string InsertSql = """ - INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, scope_name, created_at) - VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @scope_name, @created_at) - ON CONFLICT (id, tenant_id, scope_name, created_at) DO UPDATE SET + INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, basket, created_at) + VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @basket, @created_at) + ON CONFLICT (id, tenant_id, basket, created_at) DO UPDATE SET data = EXCLUDED.data, size = EXCLUDED.size, created_at = EXCLUDED.created_at @@ -29,19 +32,19 @@ ON CONFLICT (id, tenant_id, scope_name, created_at) DO UPDATE SET private const string DeleteSql = """ DELETE FROM {0} - WHERE tenant_id = @tenant_id AND scope_name = @scope_name + WHERE tenant_id = @tenant_id AND basket = @basket AND created_at >= @timestamp AND id = @id """; private const string SelectSql = """ SELECT data FROM {0} - WHERE tenant_id = @tenant_id AND scope_name = @scope_name + WHERE tenant_id = @tenant_id AND basket = @basket AND created_at >= @timestamp AND id = @id """; private readonly string _partName - = string.IsNullOrWhiteSpace(scopeName) ? "share" : Sanitize(scopeName); + = string.IsNullOrWhiteSpace(basket) ? "share" : Sanitize(basket); private readonly string _qualifiedTableName = $"{options.SchemaName}.\"{Sanitize(options.TableName)}\""; @@ -55,7 +58,8 @@ private readonly string _schemePrefix public bool IsReadOnly => options.IsReadOnly; - public string ScopeName => scopeName; + public string Basket => basket; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void EnsureWritable() @@ -67,9 +71,24 @@ private void EnsureWritable() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CanProcess(string? fileId) => - !string.IsNullOrEmpty(fileId) && - fileId.AsSpan().StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal); + public bool CanProcess(string? fileId) + { + var fileSpan = fileId.AsSpan(); + + if (!string.IsNullOrWhiteSpace(fileId) + && fileSpan.StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal)) return false; + + int schemeEnd = fileSpan.IndexOf(FileIdParser.SchemeSeparator.AsSpan()); + if (schemeEnd == -1) return false; + + var afterSpan = fileSpan[(schemeEnd + FileIdParser.SchemeSeparator.Length)..]; + int basketEnd = afterSpan.IndexOf('/'); + if (basketEnd == -1) return false; + + var basket = afterSpan[..basketEnd]; + + return basket.Equals(_partName, StringComparison.Ordinal); + } public async Task UploadAsync( UploadFileInput metadata, @@ -89,24 +108,25 @@ await partManager.EnsureParts( cancellationToken); string fileId = FileIdParser.FormatToFileId( - StorageType, options.TableName, metadata.TenantId, createdAtDay, metadata.FileName); + StorageType, _partName, metadata.TenantId, createdAtDay, metadata.FileName); string fileExtension = FileIdParser.GetFileExtension(metadata.FileName); + Stream ms = fileStream; - if (fileStream is not MemoryStream ms) + if (!fileStream.CanSeek) // fileStream is not MemoryStream ms) { ms = streamManager.GetStream(); await fileStream.CopyToAsync(ms, cancellationToken); } - try + if (fileStream.CanSeek) // fileStream is not MemoryStream ms) { - if (ms.CanSeek) - { - ms.Position = 0; - } + ms.Position = 0; + } + try + { var sql = string.Format(InsertSql, _qualifiedTableName); await dataSource.ExecuteNonQuery(sql, @@ -114,10 +134,10 @@ await dataSource.ExecuteNonQuery(sql, new NpgsqlParameter("id", fileId) , new NpgsqlParameter("name", metadata.FileName) , new NpgsqlParameter("file_ext", fileExtension) - , new NpgsqlParameter("data", fileStream) + , new NpgsqlParameter("data", ms) , new NpgsqlParameter("size", (int)ms.Length) , new NpgsqlParameter("tenant_id", metadata.TenantId) - , new NpgsqlParameter("scope_name", _partName) + , new NpgsqlParameter("basket", _partName) , new NpgsqlParameter("created_at", createdAt) ], cancellationToken); } @@ -136,7 +156,7 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio { EnsureWritable(); - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out int tenantId, out long timestamp, out _)) + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; } @@ -147,7 +167,7 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio int rowsAffected = await dataSource.ExecuteNonQuery(sql, [ new NpgsqlParameter("tenant_id", tenantId), - new NpgsqlParameter("scope_name", _partName), + new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ], cancellationToken); @@ -160,7 +180,7 @@ public async Task DownloadAsync( Func loadStream, CancellationToken cancellationToken) { - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out int tenantId, out long timestamp, out _)) + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; } @@ -174,7 +194,7 @@ public async Task DownloadAsync( }, [ new NpgsqlParameter("tenant_id", tenantId), - new NpgsqlParameter("scope_name", _partName), + new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ], cancellationToken); @@ -207,11 +227,13 @@ private static string Sanitize(ReadOnlySpan input) if (!CanProcess(fileId)) return Task.FromResult(null); - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out var tenantId, out _, out var fileName)) + if (!FileIdParser.TryParse(fileId, out _, out var tenantId, out _, out var fileName)) return Task.FromResult(null); var metadata = new FileMetadata { + Basket = _partName, + StorageType = StorageType, FileName = fileName, TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs index d7146669..586733e7 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs @@ -19,6 +19,12 @@ public PostgresFileStorageConfiguration(IServiceCollection services) _partConfiguration = services.AddSaPartitional((sp, builder) => { + var dataSource = sp.GetRequiredService(); + _options.StorageOptions.SchemaName = dataSource.GetSearchPath(); + _configure?.Invoke(sp, _options); + _options.StorageOptions.TableName = _options.StorageOptions.TableName.Trim('"'); + + builder.AddSchema(_options.StorageOptions.SchemaName, schema => { schema.AddTable(_options.StorageOptions.TableName, @@ -27,10 +33,10 @@ public PostgresFileStorageConfiguration(IServiceCollection services) "size INT NOT NULL", "file_ext TEXT NOT NULL", "tenant_id INT NOT NULL", - "scope_name TEXT NOT NULL", + "basket TEXT NOT NULL", "data BYTEA NOT NULL" ) - .PartByList("tenant_id", "scope_name") + .PartByList("tenant_id", "basket") .PartByRange(_options.PartOptions.PgPartBy, "created_at"); }); }) @@ -49,20 +55,19 @@ public PostgresFileStorageConfiguration(IServiceCollection services) services.AddSingleton(sp => { - _configure?.Invoke(sp, _options); - _configure = null; - - var pm = sp.GetRequiredService(); var dataSource = sp.GetRequiredService(); + var pm = sp.GetRequiredService(); var time = sp.GetService(); var sm = sp.GetRequiredService(); - StorageOptions options = _options.StorageOptions with - { - TableName = _options.StorageOptions.TableName.Trim('"') - }; + var storage = new PostgresFileStorage( + dataSource: dataSource, + partManager: pm, + streamManager: sm, + options: _options.StorageOptions, + basket: _options.PartOptions.Basket, + timeProvider: time); - var storage = new PostgresFileStorage(dataSource, pm, sm, options, _options.ScopeName, time); return storage; }); } @@ -91,19 +96,15 @@ public IPostgresFileStorageConfiguration AsReadOnly() return this; } - public IPostgresFileStorageConfiguration ConfigureOptions(Action configure) - { - configure?.Invoke(_options); - return this; - } - - public IPostgresFileStorageConfiguration ConfigureOptions(Action configure) + public IPostgresFileStorageConfiguration ConfigureOptions( + Action configure) { _configure = configure; return this; } - public IPostgresFileStorageConfiguration AddDataSource(Action? configure = null) + public IPostgresFileStorageConfiguration AddDataSource( + Action? configure = null) { _partConfiguration.AddDataSource(configure); return this; diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs index 23acbee8..34244c3a 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs @@ -19,6 +19,7 @@ public sealed class PartOptions { public int MigrationScheduleForwardDays { get; set; } = 2; public PgPartBy PgPartBy { get; set; } = PgPartBy.Day; + public string Basket { get; set; } = "share"; } public sealed class PostgresFileStorageOptions @@ -26,5 +27,4 @@ public sealed class PostgresFileStorageOptions public StorageOptions StorageOptions { get; } = new(); public PartOptions PartOptions { get; } = new(); public CleanupOptions CleanupOptions { get; } = new(); - public string ScopeName { get; set; } = string.Empty; } diff --git a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj index e09e3e72..3108c13f 100644 --- a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj +++ b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management in Pg diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index c10a16af..58ba67e0 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -11,12 +11,12 @@ internal sealed class S3FileStorage( S3FileStorageOptions options, TimeProvider? timeProvider = null) : IFileStorage { - private const string DefaultScopeName = "share"; + private const string DefaultBasket = "share"; private const string SchemeSeparator = "://"; - private readonly string _pathPrefix = string.IsNullOrWhiteSpace(options.ScopeName) - ? DefaultScopeName - : options.ScopeName; + private readonly string _pathPrefix = string.IsNullOrWhiteSpace(options.Basket) + ? DefaultBasket + : options.Basket; private readonly string _schemePrefix = $"{options.StorageType}{SchemeSeparator}"; private readonly string _storageType = options.StorageType; private readonly bool _isReadOnly = options.IsReadOnly; @@ -25,7 +25,7 @@ internal sealed class S3FileStorage( public string StorageType => _storageType; public bool IsReadOnly => _isReadOnly; - public string ScopeName => _pathPrefix; + public string Basket => _pathPrefix; [MethodImpl(MethodImplOptions.AggressiveInlining)] private void EnsureWritable() @@ -38,7 +38,6 @@ private void EnsureWritable() public bool CanProcess(string? fileId) { if (string.IsNullOrEmpty(fileId)) return false; - // Ordinal быстрее OrdinalIgnoreCase, т.к. схема всегда в нижнем регистре return fileId.AsSpan().StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal); } @@ -143,6 +142,8 @@ private static string GetFilePath(string fileId) return new FileMetadata { + Basket = Basket, + StorageType = StorageType, FileName = fileNameSpan.ToString(), TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs index 0cf4f5d1..cf856220 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs @@ -3,7 +3,7 @@ public sealed class S3FileStorageOptions { public string StorageType { get; init; } = "s3"; - public string ScopeName { get; init; } = string.Empty; + public string Basket { get; init; } = "share"; /// /// http://localhost:9000 diff --git a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj index 4bcd60a5..6b5c60a1 100644 --- a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj +++ b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj @@ -1,23 +1,23 @@  - + - - 0.8.3 - File storage management in S3 - + + 0.9.0 + File storage management in S3 + - - - + + + - - - - + + + + - - - + + + diff --git a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs index 3f39f486..05122d42 100644 --- a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs +++ b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs @@ -6,9 +6,9 @@ public interface IFileStorage { /// - /// scope domain + /// BL - scope domain /// - string ScopeName { get; } + string Basket { get; } /// /// Gets the type of the storage. @@ -34,12 +34,15 @@ public interface IFileStorage /// Stream of the file to upload. /// Cancellation token. /// The result of the file upload. - Task UploadAsync(UploadFileInput metadata, Stream fileStream, CancellationToken cancellationToken); + Task UploadAsync( + UploadFileInput metadata, + Stream fileStream, + CancellationToken cancellationToken); /// /// Downloads a file from the storage by its ID. /// - /// pg://files/1/1773210911/test.txt + /// pg://share/1/1773210911/test.txt /// Cancellation token. /// The file stream. Task DownloadAsync( diff --git a/src/Sa.HybridFileStorage/Domain/StorageResult.cs b/src/Sa.HybridFileStorage/Domain/StorageResult.cs index d6d49dfb..cd6ceddf 100644 --- a/src/Sa.HybridFileStorage/Domain/StorageResult.cs +++ b/src/Sa.HybridFileStorage/Domain/StorageResult.cs @@ -2,7 +2,7 @@ /// /// Represents the result of a file upload operation. -/// FileId follows URI format: {storage_type}://{path_to_resource}[?parameters] +/// FileId follows URI format: {storage_type}://{basket}/{tenant}/{path_to_resource}[?parameters] /// Examples: /// /// // PostgreSQL storage @@ -15,7 +15,7 @@ /// new StorageResult("file:///var/www/uploads/image.png", "/api/files/download/file/var/www/uploads/image.png", "file", DateTimeOffset.Now) /// /// -/// Unique file identifier in URI format: {storage_type}://{path}[?params] +/// Unique file identifier in URI format: {storage_type}://{basket}/{tenant}/{path}[?params] /// Publicly accessible URL for downloading the file /// Type of storage backend used ("pg", "s3", "file", "azure") /// Timestamp when the file was uploaded diff --git a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs index 3e8b6002..3672b5ed 100644 --- a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs +++ b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs @@ -4,4 +4,6 @@ public sealed record UploadFileInput { public int TenantId { get; init; } = 0; public string FileName { get; init; } = string.Empty; + + public static UploadFileInput Empty { get; } = new(); } diff --git a/src/Sa.HybridFileStorage/FileMetadata.cs b/src/Sa.HybridFileStorage/FileMetadata.cs index 37bcf909..65e3f4d5 100644 --- a/src/Sa.HybridFileStorage/FileMetadata.cs +++ b/src/Sa.HybridFileStorage/FileMetadata.cs @@ -2,6 +2,8 @@ public sealed class FileMetadata { + public required string Basket { get; init; } public required string FileName { get; init; } public int TenantId { get; init; } + public required string StorageType { get; init; } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorage.cs b/src/Sa.HybridFileStorage/HybridFileStorage.cs index 69f86d84..f89ff90b 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorage.cs @@ -9,35 +9,35 @@ internal sealed class HybridFileStorage( InterceptorContainer interceptors) : IHybridFileStorage { - public IReadOnlyCollection Storages => container.Storages; + public IEnumerable Storages => container.Storages; - private void EnsureWritable(string scopeName) + private void EnsureWritable(string basket) { - if (!container.Storages.Any(c => c.ScopeName == scopeName)) + if (!container.Storages.Any(c => c.Basket == basket)) { throw new HybridFileStorageNoAvailableException(); } - if (Storages.All(f => f.ScopeName == scopeName && f.IsReadOnly)) + if (Storages.All(f => f.Basket == basket && f.IsReadOnly)) { throw new HybridFileStorageWritableException(); } } public async Task UploadAsync( + string basket, UploadFileInput input, - string scopeName, Stream fileStream, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(basket, nameof(basket)); - EnsureWritable(scopeName); + EnsureWritable(basket); return await ExecuteStorageOperationAsync( - container.Storages.Where(c => !c.IsReadOnly && c.ScopeName == scopeName), + container.Storages.Where(c => !c.IsReadOnly && c.Basket == basket), async (storage, ct) => await interceptors.ExecuteBeforeUploadAsync(storage, input, fileStream, ct), async (storage, ct) => await storage.UploadAsync(input, fileStream, ct), interceptors.ExecuteAfterUploadAsync, @@ -48,14 +48,13 @@ public async Task UploadAsync( public async Task DownloadAsync( string fileId, - string scopeName, Func loadStream, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId, scopeName), + CanProcess(fileId), async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct), async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct), @@ -64,14 +63,14 @@ public async Task DownloadAsync( ); } - public async Task DeleteAsync(string fileId, string scopeName, CancellationToken cancellationToken = default) + public async Task DeleteAsync( + string fileId, + CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); - - EnsureWritable(scopeName); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId, scopeName), + CanProcess(fileId).Where(c => !c.IsReadOnly), async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct), async (storage, ct) => await storage.DeleteAsync(fileId, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct), @@ -117,12 +116,11 @@ private static async Task ExecuteStorageOperationAsync( public async Task GetMetadataAsync( string fileId, - string scopeName, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); - foreach (var fs in CanStorages(fileId, scopeName)) + foreach (var fs in container.Storages) { var meta = await fs.GetMetadataAsync(fileId, cancellationToken); if (meta != null) return meta; @@ -132,6 +130,6 @@ private static async Task ExecuteStorageOperationAsync( } - internal IEnumerable CanStorages(string fileId, string scopeName) - => container.Storages.Where(c => c.ScopeName == scopeName && c.CanProcess(fileId)); + internal IEnumerable CanProcess(string fileId) + => container.Storages.Where(c => c.CanProcess(fileId)); } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs index bd1f9a40..2a75ebe9 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs @@ -15,5 +15,5 @@ public IHybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) return this; } - public IReadOnlyCollection Storages => _storages; + public IEnumerable Storages => _storages; } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index 1ee20ad5..e4d782be 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -5,44 +5,66 @@ namespace Sa.HybridFileStorage; public static class HybridFileStorageExtensions { - public static async Task CopyToScopeAsync( + public static async Task CopyFromFileAsync( + this IHybridFileStorage storage, + string filePath, + string basket, + UploadFileInput input, + int bufferSize = 81920, + CancellationToken ct = default) + { + // копируем файл в хранилище + await using var fs = new FileStream(filePath, new FileStreamOptions + { + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + BufferSize = bufferSize, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + }); + + return await storage.UploadAsync( + basket: basket, + input: input, + fileStream: fs, + cancellationToken: ct); + } + + public static async Task CopyToBasketAsync( this IHybridFileStorage storage, string fileId, - string sourceScopeName, - string targetScopeName, - int? targetTenantId = default, + string basket, + Func? configure = null, CancellationToken ct = default) { - var metadata = await storage.GetMetadataAsync(fileId, sourceScopeName, ct); + var metadata = await storage.GetMetadataAsync(fileId, ct); if (metadata is null) { ThrowSourceFileNotFound(fileId); } - int tenantId = targetTenantId ?? metadata.TenantId; + UploadFileInput uploadInput = configure?.Invoke(metadata) ?? new UploadFileInput + { + TenantId = metadata.TenantId, + FileName = metadata.FileName + }; - if (tenantId == metadata.TenantId - && string.Equals(sourceScopeName, targetScopeName, StringComparison.Ordinal)) + if (uploadInput.TenantId == metadata.TenantId + && string.Equals(metadata.Basket, basket, StringComparison.Ordinal) + && string.Equals(metadata.FileName, uploadInput.FileName, StringComparison.Ordinal)) { ThrowFileAlreadyInTargetScope(); } - var uploadInput = new UploadFileInput - { - FileName = metadata.FileName, - TenantId = tenantId, - }; - StorageResult result = default!; bool downloaded = await storage.DownloadAsync( fileId, - sourceScopeName, async (sourceStream, downloadCt) => { result = await storage.UploadAsync( + basket, uploadInput, - targetScopeName, sourceStream, downloadCt); }, @@ -66,14 +88,11 @@ static void ThrowSourceFileNotFound(string fileId) => throw new FileNotFoundException($"Source file not found: {fileId}"); - - public static async Task> CopyToScopeBatchAsync( this IHybridFileStorage storage, IEnumerable fileIds, - string sourceScopeName, - string targetScopeName, - int? targetTenantId = default, + string basket, + Func? configure = null, BatchOptions? options = default, CancellationToken cancellationToken = default) { @@ -108,7 +127,11 @@ public static async Task> CopyToScopeBatchAsync( var ct = cts?.Token ?? cancellationToken; - return await storage.CopyToScopeAsync(fileId, sourceScopeName, targetScopeName, targetTenantId, ct); + return await storage.CopyToBasketAsync( + fileId, + basket, + configure, + ct: ct); } catch (Exception ex) { diff --git a/src/Sa.HybridFileStorage/IHybridFileStorage.cs b/src/Sa.HybridFileStorage/IHybridFileStorage.cs index f01585dd..54fb21da 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorage.cs @@ -12,15 +12,20 @@ public interface IHybridFileStorage /// /// storages /// - IReadOnlyCollection Storages { get; } + IEnumerable Storages { get; } /// - /// Deletes the file associated with the specified file ID asynchronously. + /// Uploads a file asynchronously using the provided input and file stream. /// - /// The unique identifier for the file to be deleted. + /// Metadata about the file being uploaded. + /// A stream containing the file data to be uploaded. /// A cancellation token to cancel the operation if needed. - /// True if the file was successfully deleted; otherwise, false. - Task DeleteAsync(string fileId, string scopeName, CancellationToken cancellationToken = default); + /// A containing the result of the upload operation. + Task UploadAsync( + string basket, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken = default); /// /// Downloads the file associated with the specified file ID asynchronously. @@ -31,26 +36,19 @@ public interface IHybridFileStorage /// True if the file was successfully downloaded; otherwise, false. Task DownloadAsync( string fileId, - string scopeName, Func loadStream, CancellationToken cancellationToken = default); + /// - /// Uploads a file asynchronously using the provided input and file stream. + /// Deletes the file associated with the specified file ID asynchronously. /// - /// Metadata about the file being uploaded. - /// A stream containing the file data to be uploaded. + /// The unique identifier for the file to be deleted. /// A cancellation token to cancel the operation if needed. - /// A containing the result of the upload operation. - Task UploadAsync( - UploadFileInput input, - string scopeName, - Stream fileStream, - CancellationToken cancellationToken = default); - + /// True if the file was successfully deleted; otherwise, false. + Task DeleteAsync(string fileId, CancellationToken cancellationToken = default); Task GetMetadataAsync( string fileId, - string scopeName, CancellationToken cancellationToken = default); } diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs index 0f3108fd..ddf5bcd5 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs @@ -4,5 +4,5 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageContainer : IHybridFileStorageContainerConfiguration { - IReadOnlyCollection Storages { get; } + IEnumerable Storages { get; } } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index 324174a5..0e116836 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -1,5 +1,6 @@ using Sa.HybridFileStorage.Domain; using System.Collections.Concurrent; +using System.Globalization; namespace Sa.HybridFileStorage; @@ -8,7 +9,7 @@ public sealed class InMemoryFileStorage( InMemoryFileStorageOptions? options = null, TimeProvider? timeProvider = null) : IFileStorage { - private readonly InMemoryFileStorageOptions _options = options ?? new (string.Empty); + private readonly InMemoryFileStorageOptions _options = options ?? new(string.Empty); private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; @@ -17,12 +18,14 @@ public sealed class InMemoryFileStorage( private readonly ConcurrentDictionary _storage = []; - public string ScopeName => _options.ScopeName; + public string Basket => _options.Basket; public string StorageType => DefaultStorageType; public bool IsReadOnly => _options.IsReadOnly; + public const string SchemeSeparator = "://"; + private void EnsureWritable() { if (IsReadOnly) @@ -42,8 +45,9 @@ public async Task UploadAsync( await fileStream.CopyToAsync(memoryStream, cancellationToken); byte[] fileData = memoryStream.ToArray(); - string path = Path.Combine(metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); - string fileId = $"{StorageType}://{path}"; + string path = Path.Combine(Basket, metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); + //"storageType://basket/tenant/filename" + string fileId = $"{StorageType}{SchemeSeparator}{path}"; _storage[fileId] = fileData; @@ -72,8 +76,42 @@ public Task DeleteAsync(string fileId, CancellationToken cancellationToken public bool CanProcess(string fileId) => fileId.StartsWith(StorageType); - public Task GetMetadataAsync(string fileId, CancellationToken cancellationToken = default) + public async Task GetMetadataAsync(string fileId, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + if (!_storage.ContainsKey(fileId)) return null; + + //parse: "storageType://basket/tenant/filename" + ReadOnlySpan span = fileId.AsSpan(); + int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); + if (schemeEnd == -1) + return null; + + var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; + + // "tenantId/filename" + int slashIndex = pathPart.IndexOf('/'); + if (slashIndex == -1) + return null; + + var scopeSpan = pathPart[..slashIndex]; + + var nextSpan = pathPart[(slashIndex + 1)..]; + slashIndex = nextSpan.IndexOf('/'); + + var tenantSpan = nextSpan[..slashIndex]; + var fileNameSpan = nextSpan[(slashIndex + 1)..]; + + if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) + return null; + + var metadata = new FileMetadata + { + StorageType = StorageType, + Basket = scopeSpan.ToString(), + FileName = fileNameSpan.ToString(), + TenantId = tenantId + }; + + return metadata; } } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs index 12f5cebb..db45c76c 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs @@ -1,3 +1,3 @@ namespace Sa.HybridFileStorage; -public sealed record InMemoryFileStorageOptions(string ScopeName, bool IsReadOnly = false); +public sealed record InMemoryFileStorageOptions(string Basket = "share", bool IsReadOnly = false); diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index 4ccca1fc..a607c6d7 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -17,7 +17,7 @@ This interface defines a contract for hybrid file storage systems capable of han ## Key Features - ✅ **Unified API** — Single interface for all storage providers -- ✅ **Scope-based isolation** — Multi-tenant support via scopes +- ✅ **Basket-Tenant-based isolation** — Multi-tenant support - ✅ **Read-only mode** — Protect storage from accidental modifications - ✅ **Streaming support** — Memory-efficient file transfers - ✅ **Native AOT ready** — Full compatibility with .NET 10 Native AOT @@ -33,7 +33,7 @@ The `HybridFileStorageExtensions` class provides high-level methods for bulk fil All files are identified using a unified URI-like format: ``` -{storageType}://{scope}/{tenantId}/{fileName} +{storageType}://{basket}/{tenantId}/{fileName} ``` **Examples:** @@ -53,12 +53,14 @@ dotnet add package Sa.HybridFileStorage ```csharp // di -builder.Services.AddSaHybridStorage((_, b) => b.AddStorage(new InMemoryFileStorage())); +builder.AddStorage(new InMemoryFileStorage()) +builder.Services.AddSaHybridStorage(); // some test using var stream = "Hello, HybridFileStorage!".ToStream(); await storage.UploadAsync( + "basket", new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); diff --git a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj index 67354ffd..c8630cf9 100644 --- a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj +++ b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management diff --git a/src/Sa.Media.FFmpeg/FFMpegOptions.cs b/src/Sa.Media.FFmpeg/FFMpegOptions.cs index a0f9365e..5ce1f4c6 100644 --- a/src/Sa.Media.FFmpeg/FFMpegOptions.cs +++ b/src/Sa.Media.FFmpeg/FFMpegOptions.cs @@ -14,7 +14,7 @@ public sealed record FFMpegOptions public TimeSpan? Timeout => TimeoutSeconds > 0 ? TimeSpan.FromSeconds(TimeoutSeconds.Value) - : default; + : null; // Валидация после десериализации public void Validate() diff --git a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs index 20efee16..115e72da 100644 --- a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs @@ -1,4 +1,4 @@ -using Sa.Classes; +using Sa.Media.FFmpeg.Services; using System.Diagnostics; namespace Sa.Media.FFmpeg; diff --git a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj index f6ff3b28..343e86cc 100644 --- a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj +++ b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.9.1 FFmpeg wrapper true win-x64;win-arm64;linux-x64;linux-arm64;osx-x64 @@ -104,7 +104,6 @@ - diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs index c17701e4..a041695f 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs @@ -1,4 +1,6 @@ -namespace Sa.Media.FFmpeg.Services; +using System.Runtime.CompilerServices; + +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutor(IFFRawExecutor executor) : IFFMpegExecutor { @@ -31,6 +33,8 @@ public async Task ConvertToPcmS16Le( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; var channelCount = outputChannelCount.HasValue ? $"-ac {outputChannelCount}" : string.Empty; var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + @@ -67,6 +71,8 @@ public async Task ConvertToMp3( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f mp3 {Libmp3lameArg()} -ar 16000 -b:a 128k {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); @@ -81,21 +87,34 @@ public async Task ConvertToOgg( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f ogg {LibopuArg(isLibopus)} {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); return result.StandardError; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CheckFiles(string inputFileName, string outputFileName) + { + if (string.Equals(Path.GetFullPath(inputFileName), Path.GetFullPath(outputFileName), StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Input and output files must be different", nameof(outputFileName)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string Libmp3lameArg() => Constants.IsOsLinux ? "-c:a libmp3lame" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string OverArg(bool isOverwrite) => isOverwrite ? "-y" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string LibopuArg(bool isLibopus) { if (!Constants.IsOsLinux) return string.Empty; return isLibopus ? "-c:a libopus" : "-c:a libvorbis"; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string QuotePath(string path) => $"\"{path}\""; } diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs index 5e8b91e1..023221af 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs @@ -1,6 +1,4 @@ -using Sa.Classes; - -namespace Sa.Media.FFmpeg.Services; +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutorFactory( IFFMpegLocator? mpegLocator = null, diff --git a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs index 013bc352..352a9ecf 100644 --- a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs @@ -1,5 +1,4 @@ -using Sa.Classes; -using System.Diagnostics; +using System.Diagnostics; using System.Text; namespace Sa.Media.FFmpeg.Services; diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs new file mode 100644 index 00000000..05a5adcf --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs @@ -0,0 +1,21 @@ +namespace Sa.Media.FFmpeg.Services; + +/// +/// Represents the result of a process execution, including exit code and output streams. +/// +public record ProcessExecutionResult( + /// + /// The exit code returned by the executed process. + /// A value of 0 typically indicates success. + /// + int ExitCode, + + /// + /// The standard output (stdout) captured from the process. + /// + string StandardOutput, + + /// + /// The standard error (stderr) captured from the process. + /// + string StandardError); diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs new file mode 100644 index 00000000..56804e32 --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs @@ -0,0 +1,439 @@ +using System.Diagnostics; +using System.Text; + +namespace Sa.Media.FFmpeg.Services; + +internal interface IProcessExecutor +{ + /// + /// Executes a process with real-time output handling + /// + Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + /// + /// Executes a process and returns complete output + /// + async Task ExecuteWithResultAsync( + ProcessStartInfo startInfo + , bool captureErrorOutput = true + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + var output = new StringBuilder(); + var error = new StringBuilder(); + + var exitcode = await ExecuteAsync( + startInfo + , s => output.AppendLine(s) + , e => error.AppendLine(e) + , timeout + , cancellationToken + ).ConfigureAwait(false); + + var result = new ProcessExecutionResult( + exitcode, + StandardOutput: output.ToString(), + StandardError: error.ToString()); + + if (result.ExitCode == 0 || captureErrorOutput) return result; + + throw new ProcessExecutionResultException(result); + } + + /// + /// Executes stdout as a stream. + /// Stderr is captured and checked on completion + /// + Task ExecuteStdOutAsync( + ProcessStartInfo startInfo + , Stream inputStream + , Func onOutput + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + static IProcessExecutor Default { get; } = new ProcessExecutor(); +} + + + +internal sealed class ProcessExecutor : IProcessExecutor +{ + public async Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + + cancellationToken.ThrowIfCancellationRequested(); + + startInfo.RedirectStandardOutput = outputDataReceived != null; + startInfo.RedirectStandardError = errorDataReceived != null; + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException($"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + + int exitCode = await ExecuteProcessWithHandlersAsync( + process, outputDataReceived, errorDataReceived, timeout, cancellationToken) + .ConfigureAwait(false); + + return exitCode; + } + + private static async Task ExecuteProcessWithHandlersAsync( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + int exitCode; + + try + { + await Run(process, outputDataReceived, errorDataReceived, timeout, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new ProcessTimeoutException("Process execution timed out"); + } + catch (Exception ex) + { + throw new ProcessExecutionException(process.ExitCode, "Process execution failed", ex); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + return exitCode; + } + + private static async Task Run( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + var outputCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var errorCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (outputDataReceived != null) + { + SetupOutputDataReceived(process, outputDataReceived, outputCompletion); + } + + if (errorDataReceived != null) + { + SetupErrorDataReceived(process, errorDataReceived, errorCompletion); + } + + if (process.StartInfo.RedirectStandardOutput) + { + process.BeginOutputReadLine(); + } + + if (process.StartInfo.RedirectStandardError) + { + process.BeginErrorReadLine(); + } + + using var timeoutCts = timeout.HasValue && timeout.Value != TimeSpan.Zero + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + var waitTask = process.WaitForExitAsync(linkedCts.Token); + + // Wait for all streams to finish reading + List outputTasks = [waitTask]; + + if (outputDataReceived != null) + { + outputTasks.Add(outputCompletion.Task); + } + + if (errorDataReceived != null) + { + outputTasks.Add(errorCompletion.Task); + } + + await Task.WhenAll(outputTasks).ConfigureAwait(false); + } + + private static void SetupErrorDataReceived( + Process process, + Action errorDataReceived, + TaskCompletionSource errorCompletion) + { + process.ErrorDataReceived += (_, e) => + { + if (e.Data != null) errorDataReceived(e.Data); + else errorCompletion.TrySetResult(true); + }; + } + + private static void SetupOutputDataReceived( + Process process, + Action outputDataReceived, + TaskCompletionSource outputCompletion) + { + process.OutputDataReceived += (_, e) => + { + if (e.Data != null) outputDataReceived(e.Data); + else outputCompletion.TrySetResult(true); + }; + } + + /// + /// Запускает процесс, и передаёт поток stdout в callback. + /// Поток stderr собирается автоматически. + /// При завершении — проверяется код возврата. + /// + /// Настройки процесса. + /// Поток для stdin + /// Callback, получающий stdout. Должен быть асинхронным. + /// Задача, завершающаяся после обработки потока и проверки результата. + public async Task ExecuteStdOutAsync( + ProcessStartInfo startInfo, + Stream inputStream, + Func onOutput, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Настройка + startInfo.RedirectStandardInput = true; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + + + var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException( + $"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + StringBuilder stderrBuilder = new(); + int exitCode; + try + { + using var timeoutCts = timeout.HasValue + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + + List backgroundTasks = [ + WriteToStdInAsync(process, inputStream, linkedCts.Token), // write stdin + ReadStandardErrorToBuilderAsync(process, stderrBuilder, linkedCts.Token), // read stderr + ]; + + Stream stdoutStream = process.StandardOutput.BaseStream; + try + { + await onOutput(stdoutStream, linkedCts.Token).ConfigureAwait(false); + } + finally + { + await stdoutStream.DisposeAsync(); + } + await Task.WhenAll(backgroundTasks).ConfigureAwait(false); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + if (exitCode != 0) + { + throw new ProcessExecutionException(exitCode, $"Process failed (exit={exitCode}): {stderrBuilder}"); + } + } + + + private static async Task ReadStandardErrorToBuilderAsync( + Process process, + StringBuilder errorBuilder, + CancellationToken cancellationToken = default) + { + try + { + string error = await process.StandardError.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(error)) + { + errorBuilder.Append(error); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (errorBuilder.Length == 0) + errorBuilder.Append("stderr: read interrupted (cancellation)."); + } + catch (Exception ex) + { + errorBuilder.Append($"stderr: read failed: {ex.Message}"); + } + } + + /// + /// Асинхронно записывает входной поток в stdin процесса. + /// Автоматически закрывает stdin после завершения. + /// + private static async Task WriteToStdInAsync( + Process process, + Stream inputStream, + CancellationToken cancellationToken = default) + { + try + { + // input to stdin + await using (inputStream.ConfigureAwait(false)) + { + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken) + .ConfigureAwait(false); + } + + // Завершаем запись + await process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false); + process.StandardInput.Close(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + process.StandardInput.Close(); + } + catch (IOException) + { + process.StandardInput.Close(); + } + catch (Exception ex) + { + process.StandardInput.Close(); + throw new IOException("Failed to write input stream to process stdin.", ex); + } + } + + private static int SafeDisposeProcess(Process process) + { + int exitCode = -1; + + try + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + + process.StandardInput?.Close(); + process.StandardOutput?.Close(); + + // graceful shutdown + if (process.WaitForExit(500)) + { + exitCode = process.ExitCode; + return exitCode; + } + + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + throw; + } + catch (NotSupportedException) + { + process.Kill(); + } + catch (UnauthorizedAccessException) + { + return exitCode; + } + + if (process.WaitForExit(2000)) + { + exitCode = process.ExitCode; + } + else + { + Console.WriteLine("Process did not terminate after Kill()"); + } + } + catch (InvalidOperationException) + { + Console.WriteLine("Process is invalid or already disposed."); + } + catch (Exception ex) + { + Console.WriteLine($"Unexpected error during process termination: {ex.Message}"); + } + finally + { + try + { + process.Dispose(); + } + catch + { + // skeep + } + } + + return exitCode; + } +} + + +// Custom exceptions +public sealed class ProcessExecutionException(int exitCode, string message, Exception? inner = null) + : Exception(message, inner) +{ + public int Exitcode => exitCode; +} + +public sealed class ProcessExecutionResultException(ProcessExecutionResult result) + : Exception($"Process failed (exit={result.ExitCode}): {result.StandardError}") +{ + public ProcessExecutionResult Result { get; } = result; +} + +public sealed class ProcessStartException(string message) : IOException(message) +{ +} + +public sealed class ProcessTimeoutException(string message) : TimeoutException(message) +{ +} diff --git a/src/Sa.Media.FFmpeg/Setup.cs b/src/Sa.Media.FFmpeg/Setup.cs index a19ad15e..6d78ef8b 100644 --- a/src/Sa.Media.FFmpeg/Setup.cs +++ b/src/Sa.Media.FFmpeg/Setup.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using Sa.Classes; using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpeg; diff --git a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs index 3585656b..fcb7fea0 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs @@ -5,8 +5,9 @@ namespace Sa.Outbox.PostgreSql.Commands; -internal sealed class ErrorDeliveryCommand(IPgDataSource dataSource, SqlOutboxBuilder sqlTemplate) - : IErrorDeliveryCommand +internal sealed class ErrorDeliveryCommand( + IPgDataSource dataSource, + SqlOutboxBuilder sqlTemplate): IErrorDeliveryCommand { private readonly SqlCacheSplitter sqlCache = new(len => sqlTemplate.SqlError(len)); diff --git a/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs index d967cbe7..327ff756 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs @@ -3,7 +3,9 @@ namespace Sa.Outbox.PostgreSql.Commands; -internal sealed class InsertMsgTypeCommand(IPgDataSource dataSource, SqlOutboxBuilder template) : IInsertMsgTypeCommand +internal sealed class InsertMsgTypeCommand( + IPgDataSource dataSource, + SqlOutboxBuilder template) : IInsertMsgTypeCommand { public Task Execute(long id, string typeName, CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs index 4b93c4dc..97703cc4 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs @@ -1,4 +1,5 @@ -using Sa.Data.PostgreSql; +using Npgsql; +using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.SqlBuilder; namespace Sa.Outbox.PostgreSql.Commands; @@ -11,8 +12,15 @@ NpqsqlOutboxReader outboxReader { public async Task> Execute(CancellationToken cancellationToken) { - return await dataSource.ExecuteReaderList(sql.SqlSelectTetant, - reader => outboxReader.Message.GetTenantId(reader), - cancellationToken); + try + { + return await dataSource.ExecuteReaderList(sql.SqlSelectTetant, + reader => outboxReader.Message.GetTenantId(reader), + cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.UndefinedTable) + { + return []; + } } } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs index 9735dcea..9be33a89 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs @@ -1,12 +1,17 @@ using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.Serialization; +using System.Diagnostics.CodeAnalysis; namespace Sa.Outbox.PostgreSql.Configuration; public interface IPgOutboxConfiguration { - IPgOutboxConfiguration WithMessageSerializer(Func messageSerializerFactory); - IPgOutboxConfiguration WithMessageSerializer(TService instance) where TService : class, IOutboxMessageSerializer; + IPgOutboxConfiguration WithMessageSerializer( + Func messageSerializerFactory); + IPgOutboxConfiguration WithMessageSerializer(TService instance) + where TService : class, IOutboxMessageSerializer; + IPgOutboxConfiguration WithMessageSerializer<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TService>() + where TService : class, IOutboxMessageSerializer; IPgOutboxConfiguration WithOutboxSettings(Action? configure = null); IPgOutboxConfiguration WithDataSource(Action? configure = null); } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs index 7f583cbb..3c21476f 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.Serialization; +using System.Diagnostics.CodeAnalysis; namespace Sa.Outbox.PostgreSql.Configuration; @@ -25,7 +26,8 @@ public IPgOutboxConfiguration WithDataSource(Action messageSerializerFactory) + public IPgOutboxConfiguration WithMessageSerializer( + Func messageSerializerFactory) { services.RemoveAll(); services.TryAddSingleton(messageSerializerFactory); @@ -40,6 +42,14 @@ public IPgOutboxConfiguration WithMessageSerializer(TService instance) return this; } + public IPgOutboxConfiguration WithMessageSerializer<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TService>() + where TService : class, IOutboxMessageSerializer + { + services.RemoveAll(); + services.TryAddSingleton(); + return this; + } + internal IPgOutboxConfiguration WithDefaultSerializer() { services.TryAddSingleton(OutboxMessageSerializer.Instance); @@ -52,6 +62,13 @@ private void RegisterOutboxSettings() { PgOutboxSettings settings = new(); + var connectionSchema = sp.GetService()?.GetSearchPath(); + + if (!string.IsNullOrWhiteSpace(connectionSchema)) + { + settings.TableSettings.WithSchema(connectionSchema); + } + var configureActions = sp.GetServices>(); foreach (var configureAction in configureActions) diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs index 40c85a8d..df4ac86a 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs @@ -72,8 +72,8 @@ public static PgOutboxTableSettings UseBaseTableName( /// public static PgOutboxTableSettings UseBaseTableName( this PgOutboxTableSettings settings, - string baseTableName, - string schemaName) + string schemaName, + string baseTableName) { if (string.IsNullOrWhiteSpace(schemaName)) throw new ArgumentException("Schema name cannot be null or empty", nameof(schemaName)); diff --git a/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs b/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs index 1402f854..89153283 100644 --- a/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs +++ b/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs @@ -4,7 +4,8 @@ namespace Sa.Outbox.PostgreSql.Partitional; -internal sealed class MsgMigrationSupport(IOutboxPartitionalSupport? partitionalSupport = null) : IPartTableMigrationSupport +internal sealed class MsgMigrationSupport(IOutboxPartitionalSupport? partitionalSupport = null) + : IPartTableMigrationSupport { public async Task GetParts(CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj index c592b68c..cd3c74a6 100644 --- a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj +++ b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple Outbox for Pg (publishing and using messages) diff --git a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs index 228b520f..3f069bf0 100644 --- a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs +++ b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs @@ -11,5 +11,6 @@ public sealed record LoadGroupResult(int CopiedRows, Guid NewOffset) internal interface IOutboxTaskLoader { - Task LoadNewTasks(OutboxMessageFilter filter, int batchSize, CancellationToken cancellationToken = default); + Task LoadNewTasks( + OutboxMessageFilter filter, int batchSize, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs b/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs index 62c3e8fe..985a9e1d 100644 --- a/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs +++ b/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs @@ -4,8 +4,9 @@ namespace Sa.Outbox.PostgreSql.Services; -internal sealed class OutboxPartRepository(IPartitionManager partManager, PgOutboxTableSettings tableSettings) - : IOutboxPartRepository +internal sealed class OutboxPartRepository( + IPartitionManager partManager, + PgOutboxTableSettings tableSettings): IOutboxPartRepository { public Task EnsureMsgParts(IEnumerable outboxParts, CancellationToken cancellationToken) => EnsureParts(tableSettings.Message.TableName, outboxParts, cancellationToken); @@ -32,13 +33,20 @@ public async Task EnsureErrorParts(IEnumerable dates, Cance public Task Migrate() => partManager.Migrate(CancellationToken.None); - private async Task EnsureParts(string databaseTableName, IEnumerable outboxParts, CancellationToken cancellationToken) + private async Task EnsureParts( + string databaseTableName, + IEnumerable outboxParts, + CancellationToken cancellationToken) { int i = 0; foreach (OutboxPartInfo part in outboxParts.Distinct()) { i++; - await partManager.EnsureParts(databaseTableName, part.CreatedAt, [part.TenantId, part.Part], cancellationToken); + await partManager.EnsureParts( + databaseTableName, + part.CreatedAt, + [part.TenantId, part.Part], + cancellationToken); } return i; diff --git a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs index 6bbf80d5..b7005e9c 100644 --- a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs +++ b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs @@ -33,17 +33,6 @@ public async Task RentDelivery( return await startCmd.ExecuteFill(writeBuffer, lockDuration, filter, cancellationToken); } - private async Task EnsureParts(OutboxMessageFilter filter, CancellationToken cancellationToken) - { - await partRepository.EnsureMsgParts( - [new OutboxPartInfo(filter.TenantId, filter.Part, filter.NowDate)], - cancellationToken); - - await partRepository.EnsureTaskParts( - [new OutboxPartInfo(filter.TenantId, filter.ConsumerGroupId, filter.NowDate)], - cancellationToken); - } - public async Task ReturnDelivery( ReadOnlyMemory> messages, OutboxMessageFilter filter, @@ -61,6 +50,23 @@ public async Task ReturnDelivery( return await finishCmd.Execute(messages, errors, filter, cancellationToken); } + public Task ExtendDelivery( + TimeSpan lockExpiration, + OutboxMessageFilter filter, + CancellationToken cancellationToken) + => extendCmd.Execute(lockExpiration, filter, cancellationToken); + + private async Task EnsureParts(OutboxMessageFilter filter, CancellationToken cancellationToken) + { + await partRepository.EnsureMsgParts( + [new OutboxPartInfo(filter.TenantId, filter.Part, filter.NowDate)], + cancellationToken); + + await partRepository.EnsureTaskParts( + [new OutboxPartInfo(filter.TenantId, filter.ConsumerGroupId, filter.NowDate)], + cancellationToken); + } + private async ValueTask> GetErrors( ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -79,10 +85,4 @@ private async ValueTask> GetErrors ExtendDelivery( - TimeSpan lockExpiration, - OutboxMessageFilter filter, - CancellationToken cancellationToken) - => extendCmd.Execute(lockExpiration, filter, cancellationToken); } diff --git a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs index d1e491e9..5278fdde 100644 --- a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs +++ b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs @@ -8,7 +8,9 @@ namespace Sa.Outbox.PostgreSql.SqlBuilder; /// /// Provides SQL query templates for working with PostgreSQL outbox tables. /// -internal sealed class SqlOutboxBuilder(PgOutboxTableSettings settings, ObjectPool objectPool) +internal sealed class SqlOutboxBuilder( + PgOutboxTableSettings settings, + ObjectPool objectPool) { internal PgOutboxTableSettings Settings => settings; diff --git a/src/Sa.Outbox/Delivery/ConsumeSettings.cs b/src/Sa.Outbox/Delivery/ConsumeSettings.cs index 07bc6732..d64f9fbf 100644 --- a/src/Sa.Outbox/Delivery/ConsumeSettings.cs +++ b/src/Sa.Outbox/Delivery/ConsumeSettings.cs @@ -11,8 +11,13 @@ public sealed class ConsumeSettings { /// /// Maximum number of processing iterations when greedy mode is enabled. - /// -1 means unlimited iterations. + /// -1 means unlimited iterations (greedy mode). /// + /// + /// Consumes data in chunks of size ConsumeBatchSize repeatedly, up to a total of MaxBatchSize. + /// The MaxProcessingIterations limits the number of these consumption cycles. + /// When set to -1 (greedy mode), there is no iteration limit — the system will continue consuming batches until it reaches MaxBatchSize or runs out of data. + /// public int MaxProcessingIterations { get; set; } = 10; /// diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index 2a2a118d..a5a66f85 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -8,7 +8,8 @@ namespace Sa.Outbox.Delivery; internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDeliveryBuilder { - public IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + public IDeliveryBuilder AddDeliveryScoped< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, Action? configure = null ) @@ -19,7 +20,8 @@ internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDe return this; } - public IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + public IDeliveryBuilder AddDelivery< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, Action? configure = null ) @@ -30,7 +32,8 @@ internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDe return this; } - public IDeliveryBuilder AddDeliveryBatching<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() + public IDeliveryBuilder AddDeliveryBatching< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() where TImplementation : class, IDeliveryBatcher { services diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 080a9b60..27790942 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -41,7 +41,7 @@ public async Task ProcessMessages( continueProcessing = ShouldContinueProcessing( sentCount, iterations, - settings.ConsumeSettings, + consumeSettings, cancellationToken); } while (continueProcessing); @@ -72,12 +72,9 @@ private async Task ProcessForEachTenant( ConsumerGroupSettings settings, CancellationToken cancellationToken) { - if (settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism == 1) - { - return await ProcessTenantsSequential(tenantIds, settings, cancellationToken); - } - - return await ProcessTenantsParallel(tenantIds, settings, cancellationToken); + return (settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism == 1) + ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken) + : await ProcessTenantsParallel(tenantIds, settings, cancellationToken); } private async Task ProcessTenantsSequential( diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index c9566b2b..4356996c 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -10,11 +10,14 @@ namespace Sa.Outbox.Delivery; /// internal sealed class DeliveryTenant( IOutboxDeliveryManager deliveryMan, - TimeProvider timeProvider, IDeliveryCourier deliveryCourier, IDeliveryBatcher batcher, - FilterFactory filterFactory) : IDeliveryTenant + FilterFactory filterFactory, + TimeProvider? timeProvider = null) : IDeliveryTenant { + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + public async Task ProcessInTenant( int tenantId, ConsumerGroupSettings settings, @@ -50,15 +53,12 @@ private OutboxMessageFilter CreateFilter(int tenantId, ConsumerGroupSe return filterFactory.CreateFilter( tenantId: tenantId, consumerGroupId: settings.ConsumerGroupId, - now: timeProvider.GetUtcNow(), + now: GetUtcNow(), lookbackInterval: settings.ConsumeSettings.LookbackInterval, batchingWindow: settings.ConsumeSettings.BatchingWindow); } - private static IMemoryOwner> RentMemory(int size) - { - return MemoryPool>.Shared.Rent(size); - } + private DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); private async Task CalculateBatchSizeAsync( ConsumeSettings consumeSettings, @@ -96,7 +96,7 @@ private Task ReleaseMessagesAsync( { return deliveryMan.ReturnDelivery( messages, - filter with { NowDate = timeProvider.GetUtcNow() }, + filter with { NowDate = GetUtcNow() }, cancellationToken); } @@ -108,7 +108,7 @@ private IDisposable RenewerLocker( settings.LockRenewal , t => { - var nowDate = timeProvider.GetUtcNow(); + var nowDate = GetUtcNow(); return deliveryMan.ExtendDelivery( settings.LockDuration , filter with @@ -119,4 +119,7 @@ private IDisposable RenewerLocker( , t); } , cancellationToken: cancellationToken); + + private static IMemoryOwner> RentMemory(int size) + => MemoryPool>.Shared.Rent(size); } diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs b/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs new file mode 100644 index 00000000..9e1fe3b4 --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs @@ -0,0 +1,9 @@ +using Sa.Schedule; + +namespace Sa.Outbox.Delivery.Job; + +internal sealed class DeliveryScheduleProvider(IScheduler scheduler) : IDeliveryScheduleProvider +{ + public IJobScheduler GetJob(Guid jobId) + => scheduler.GetSchedule(jobId) ?? throw new InvalidOperationException(); +} diff --git a/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs new file mode 100644 index 00000000..28b27938 --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs @@ -0,0 +1,11 @@ +using Sa.Schedule; + +namespace Sa.Outbox.Delivery.Job; + +public interface IDeliveryScheduleProvider +{ + IJobScheduler GetJob(Guid jobId); + + int GetInstanceCount(Guid jobId) => GetJob(jobId).ConcurrencyLimit; + void SetInstanceCount(Guid jobId, int count) => GetJob(jobId).ConcurrencyLimit = count; +} diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index 43bd3ef4..098e650a 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.Schedule; using System.Diagnostics.CodeAnalysis; @@ -8,11 +9,11 @@ internal static class Setup { public static IServiceCollection AddDeliveryJob< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - this IServiceCollection services, - string consumerGroupId, - bool isSingleton, - Action? сonfigure = null) - where TConsumer : class, IConsumer + this IServiceCollection services, + string consumerGroupId, + bool isSingleton, + Action? сonfigure = null) + where TConsumer : class, IConsumer { ArgumentNullException.ThrowIfNullOrWhiteSpace(consumerGroupId); @@ -42,6 +43,8 @@ public static IServiceCollection AddDeliveryJob< .EveryTime(scheduleSettings.Interval) .WithInitialDelay(scheduleSettings.InitialDelay) .WithTag(settings) + .WithConcurrencyLimit(scheduleSettings.ConcurrencyLimit) + .WithMaxConcurrency(scheduleSettings.MaxConcurrency) .WithName(scheduleSettings.Name ?? typeof(TConsumer).Name) .ConfigureErrorHandling(c => c .IfErrorRetry(scheduleSettings.RetryCountOnError) @@ -54,6 +57,8 @@ public static IServiceCollection AddDeliveryJob< builder.AddInterceptor(); }); + services.TryAddSingleton(); + return services; } } diff --git a/src/Sa.Outbox/Delivery/OutboxContext.cs b/src/Sa.Outbox/Delivery/OutboxContext.cs index a04ba001..24a32a5a 100644 --- a/src/Sa.Outbox/Delivery/OutboxContext.cs +++ b/src/Sa.Outbox/Delivery/OutboxContext.cs @@ -6,12 +6,14 @@ namespace Sa.Outbox.Delivery; [DebuggerDisplay("#{PayloadId}")] -/// -/// OutboxMessage -/// -internal sealed class OutboxContext(OutboxDeliveryMessage delivery, TimeProvider timeProvider) +internal sealed class OutboxContext( + OutboxDeliveryMessage delivery, + TimeProvider? timeProvider = null) : IOutboxContextOperations { + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + public Guid OutboxId => delivery.OutboxId; public string PayloadId => delivery.Message.PayloadId; public TMessage Payload => delivery.Message.Payload; @@ -133,7 +135,7 @@ private void ErrorWithCode(DeliveryStatusCode errorCode, Exception exception, st null); } - public DateTimeOffset GetUtcNow() => (timeProvider ?? TimeProvider.System).GetUtcNow(); + public DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); private readonly static DeliveryPermanentException DeliveryPermanentException diff --git a/src/Sa.Outbox/Delivery/OutboxContextFactory.cs b/src/Sa.Outbox/Delivery/OutboxContextFactory.cs index 9b495d5c..811056e9 100644 --- a/src/Sa.Outbox/Delivery/OutboxContextFactory.cs +++ b/src/Sa.Outbox/Delivery/OutboxContextFactory.cs @@ -1,6 +1,6 @@ namespace Sa.Outbox.Delivery; -internal sealed class OutboxContextFactory(TimeProvider? timeProvider) : IOutboxContextFactory +internal sealed class OutboxContextFactory(TimeProvider? timeProvider = null) : IOutboxContextFactory { public IOutboxContextOperations Create(OutboxDeliveryMessage deliveryMessage) { diff --git a/src/Sa.Outbox/Delivery/ScheduleSettings.cs b/src/Sa.Outbox/Delivery/ScheduleSettings.cs index 9a704924..a049e1f6 100644 --- a/src/Sa.Outbox/Delivery/ScheduleSettings.cs +++ b/src/Sa.Outbox/Delivery/ScheduleSettings.cs @@ -19,5 +19,10 @@ public sealed class ScheduleSettings /// public TimeSpan InitialDelay { get; internal set; } = TimeSpan.FromSeconds(10); - public int RetryCountOnError { get; internal set; } = 2; + public int RetryCountOnError { get; internal set; } = 1; + + public int ConcurrencyLimit { get; internal set; } = 1; + + public int MaxConcurrency { get; internal set; } = 48; + } diff --git a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs b/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs index 50353fe3..f5543753 100644 --- a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs +++ b/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs @@ -39,6 +39,17 @@ public static ScheduleSettings WithImmediate(this ScheduleSettings settings) return settings; } + public static ScheduleSettings WithConcurrencyLimit(this ScheduleSettings settings, int limit) + { + settings.ConcurrencyLimit = limit; + return settings; + } + public static ScheduleSettings WithMaxConcurrency(this ScheduleSettings settings, int maxLimit) + { + settings.MaxConcurrency = maxLimit; + return settings; + } + /// /// Sets the number of retry attempts on error. /// @@ -92,4 +103,5 @@ public static ScheduleSettings WithInitialDelaySeconds(this ScheduleSettings set public static ScheduleSettings WithInitialDelayMinutes(this ScheduleSettings settings, int minutes) => settings.WithInitialDelay(TimeSpan.FromMinutes(minutes)); + } diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index 40f57447..0c3eb0a7 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -7,7 +7,8 @@ namespace Sa.Outbox.Delivery; internal static class Setup { - public static IServiceCollection AddOutboxDelivery(this IServiceCollection services, Action configure) + public static IServiceCollection AddOutboxDelivery( + this IServiceCollection services, Action? configure = null) { services.AddMessagesMetadata(); @@ -29,7 +30,7 @@ public static IServiceCollection AddOutboxDelivery(this IServiceCollection servi services.TryAddSingleton(); - configure.Invoke(new DeliveryBuilder(services)); + configure?.Invoke(new DeliveryBuilder(services)); services.TryAddSingleton(); diff --git a/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs b/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs index 8278ec60..364f2443 100644 --- a/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs +++ b/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs @@ -2,7 +2,9 @@ namespace Sa.Outbox.Exceptions; -public class DeliveryPermanentException(string message, Exception? innerException = null, DeliveryStatusCode statusCode = DeliveryStatusCode.Error) - : DeliveryException(message, innerException, statusCode) +public class DeliveryPermanentException( + string message, + Exception? innerException = null, + DeliveryStatusCode statusCode = DeliveryStatusCode.Error) : DeliveryException(message, innerException, statusCode) { } diff --git a/src/Sa.Outbox/IOutboxBuilder.cs b/src/Sa.Outbox/IOutboxBuilder.cs index 9f426713..53f79122 100644 --- a/src/Sa.Outbox/IOutboxBuilder.cs +++ b/src/Sa.Outbox/IOutboxBuilder.cs @@ -1,5 +1,6 @@ using Sa.Outbox.Delivery; using Sa.Outbox.Metadata; +using Sa.Outbox.Partitional; using Sa.Outbox.Publication; using System.Diagnostics.CodeAnalysis; @@ -34,4 +35,19 @@ public interface IOutboxBuilder IOutboxBuilder WithMetadata(Action configure); + + /// + /// shortcut + /// + IOutboxBuilder AddMetadata( + string partName, + Func? getPayloadId = null) where TMessage : class + { + return WithMetadata((_, m) => m.AddMetadata(partName, getPayloadId)); + } + + IOutboxBuilder AddMetadata() where TMessage : class, IOutboxPublishable + { + return WithMetadata((_, m) => m.AddMetadata()); + } } diff --git a/src/Sa.Outbox/IOutboxPublishable.cs b/src/Sa.Outbox/IOutboxPublishable.cs new file mode 100644 index 00000000..8d4d5c7c --- /dev/null +++ b/src/Sa.Outbox/IOutboxPublishable.cs @@ -0,0 +1,14 @@ +namespace Sa.Outbox; + +public interface IOutboxPublishable +{ + string GetPayloadId(); + + int GetTenantId(); + /// + /// Gets the logical identifier of the partition associated with this type. + /// Used for routing messages to specific tables, shards, or storage groups (e.g., outbox_orders, outbox_eu). + /// + /// "orders", "notifications", "us_west" + static abstract string PartName { get; } +} diff --git a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs index 33ee9c7b..d47a751d 100644 --- a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs +++ b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs @@ -2,6 +2,12 @@ public interface IOutboxMessageMetadataBuilder { - IOutboxMessageMetadataBuilder AddMetadata(string partName, Func? getPayloadId = null) - where TMessage : class; + IOutboxMessageMetadataBuilder AddMetadata( + string partName, + Func? getPayloadId = null) where TMessage : class; + + IOutboxMessageMetadataBuilder AddMetadata() where TMessage : class, IOutboxPublishable + { + return AddMetadata(TMessage.PartName, m => m.GetPayloadId()); + } } diff --git a/src/Sa.Outbox/Metadata/MetadataConfiguration.cs b/src/Sa.Outbox/Metadata/MetadataConfiguration.cs index da1186ea..b16a0ef4 100644 --- a/src/Sa.Outbox/Metadata/MetadataConfiguration.cs +++ b/src/Sa.Outbox/Metadata/MetadataConfiguration.cs @@ -6,8 +6,6 @@ internal sealed class MetadataConfiguration : IOutboxMessageMetadataBuilder, IOu private static readonly Func s_Dummy = _ => string.Empty; - private static readonly OutboxMessageMetadata s_Default = new("root", s_Dummy); - public IOutboxMessageMetadataBuilder AddMetadata(string partName, Func? getPayloadId = null) where T : class @@ -40,7 +38,7 @@ public OutboxMessageMetadata GetMetadata(Type messageType) return metadata; } - return s_Default; + return OutboxMessageMetadata.Empty; } diff --git a/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs b/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs index 855193e8..a80bc77c 100644 --- a/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs +++ b/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs @@ -3,4 +3,10 @@ internal sealed record OutboxMessageMetadata( string PartName, - Func GetPayloadId); + Func GetPayloadId) +{ + public static readonly OutboxMessageMetadata Empty = new("root", m => + { + return (m is IOutboxPublishable msg) ? msg.GetPayloadId() : string.Empty; + }); +} diff --git a/src/Sa.Outbox/Metadata/Setup.cs b/src/Sa.Outbox/Metadata/Setup.cs index 090be7ca..0c54cd08 100644 --- a/src/Sa.Outbox/Metadata/Setup.cs +++ b/src/Sa.Outbox/Metadata/Setup.cs @@ -10,12 +10,16 @@ public static IServiceCollection AddMessagesMetadata( Action? configure = null) { - services.AddSingleton(sp => + if (configure != null) { - var configuration = new MetadataConfiguration(); - configure?.Invoke(sp, configuration); - return configuration; - }); + // multiple configuration + services.AddSingleton(sp => + { + var configuration = new MetadataConfiguration(); + configure.Invoke(sp, configuration); + return configuration; + }); + } services.TryAddSingleton(sp => diff --git a/src/Sa.Outbox/OutboxBuilder.cs b/src/Sa.Outbox/OutboxBuilder.cs index 92f12074..8a53ee09 100644 --- a/src/Sa.Outbox/OutboxBuilder.cs +++ b/src/Sa.Outbox/OutboxBuilder.cs @@ -12,14 +12,14 @@ internal sealed class OutboxBuilder : IOutboxBuilder { private readonly IServiceCollection _services; - private OutboxBuilder(IServiceCollection services) - { - _services = services; - } + private OutboxBuilder(IServiceCollection services) => _services = services; public static OutboxBuilder Create(IServiceCollection services) { - services.AddMessagePublisher(); + services + .AddMessagePublisher() + .AddOutboxDelivery(); + return new OutboxBuilder(services); } @@ -43,12 +43,13 @@ public IOutboxBuilder WithDeliveries(Action build) public IOutboxBuilder WithTenants(Action configure) { - _services.AddTenantProvider(configure); + _services.AddTenantSettings(configure); return this; } - public IOutboxBuilder WithDeliveryBatcher<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() - where TImplementation : class, IDeliveryBatcher + public IOutboxBuilder WithDeliveryBatcher< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() + where TImplementation : class, IDeliveryBatcher { _services .RemoveAll() diff --git a/src/Sa.Outbox/Partitional/ITenantSource.cs b/src/Sa.Outbox/Partitional/ITenantSource.cs index 7f23f04f..5ab829bf 100644 --- a/src/Sa.Outbox/Partitional/ITenantSource.cs +++ b/src/Sa.Outbox/Partitional/ITenantSource.cs @@ -1,5 +1,4 @@ - -namespace Sa.Outbox; +namespace Sa.Outbox.Partitional; /// /// Tenant provider for partitional support. diff --git a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs index 47c1a032..3e76da12 100644 --- a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs +++ b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs @@ -24,7 +24,8 @@ public async ValueTask GetTenantIds(CancellationToken cancellationToken) return await tenantProvider.GetTenantIds(cancellationToken); } - private async Task> GetPairs(IEnumerable parts, CancellationToken cancellationToken) + private async Task> GetPairs( + IEnumerable parts, CancellationToken cancellationToken) { int[] tenantIds = await GetTenantIds(cancellationToken); if (tenantIds.Length == 0) return []; diff --git a/src/Sa.Outbox/Partitional/Setup.cs b/src/Sa.Outbox/Partitional/Setup.cs index 22c0e151..a4d4fba7 100644 --- a/src/Sa.Outbox/Partitional/Setup.cs +++ b/src/Sa.Outbox/Partitional/Setup.cs @@ -8,15 +8,17 @@ internal static class Setup { public static IServiceCollection AddOutboxPartitional(this IServiceCollection services) { + services.TryAddSingleton(); + // support - messaging to each tenant services.TryAddSingleton(); + services.TryAddSingleton(); return services; } - public static IServiceCollection AddTenantProvider(this IServiceCollection services, Action configure) + public static IServiceCollection AddTenantSettings( + this IServiceCollection services, Action configure) { // support - messaging to each tenant - services.TryAddSingleton(); - services .RemoveAll() .AddSingleton(sp => diff --git a/src/Sa.Outbox/Partitional/TenantSettings.cs b/src/Sa.Outbox/Partitional/TenantSettings.cs index a07d99e6..4398201d 100644 --- a/src/Sa.Outbox/Partitional/TenantSettings.cs +++ b/src/Sa.Outbox/Partitional/TenantSettings.cs @@ -1,4 +1,4 @@ -namespace Sa.Outbox; +namespace Sa.Outbox.Partitional; /// /// Represents the settings for partitioning in the Outbox processing system. @@ -10,7 +10,7 @@ public sealed class TenantSettings /// Gets or sets a value indicating whether the system should automatically detect tenants /// by scanning incoming database messages /// - public bool AutoDetect { get; private set; } = false; + public bool AutoDetect { get; private set; } = true; /// /// Gets or sets a function that retrieves tenant IDs asynchronously. @@ -56,4 +56,10 @@ public TenantSettings WithAutoDetect() AutoDetect = true; return this; } + + public TenantSettings WithNoAutoDetect() + { + AutoDetect = false; + return this; + } } diff --git a/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs b/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs index 5c69f11e..c6f9845f 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs @@ -1,15 +1,15 @@ namespace Sa.Outbox.PlugServices; -public interface IOutboxPlugin : IAsyncDisposable -{ - string Name { get; } - string Version { get; } - string Provider { get; } // "Postgres", "SqlServer", "Redis", etc. +//public interface IOutboxPlugin : IAsyncDisposable +//{ +// string Name { get; } +// string Version { get; } +// string Provider { get; } // "Postgres", "SqlServer", "Redis", etc. - IOutboxBulkWriter BulkWriter { get; } - IOutboxDeliveryManager DeliveryManager { get; } - IOutboxTenantDetector TenantDetector { get; } +// IOutboxBulkWriter BulkWriter { get; } +// IOutboxDeliveryManager DeliveryManager { get; } +// IOutboxTenantDetector TenantDetector { get; } - ValueTask InitializeAsync(CancellationToken cancellationToken = default); - ValueTask HealthCheckAsync(CancellationToken cancellationToken = default); -} +// ValueTask InitializeAsync(CancellationToken cancellationToken = default); +// ValueTask HealthCheckAsync(CancellationToken cancellationToken = default); +//} diff --git a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs index 702a3852..78f26aa1 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs @@ -1,4 +1,6 @@ -namespace Sa.Outbox.PlugServices; +using Sa.Outbox.Partitional; + +namespace Sa.Outbox.PlugServices; /// /// Discovers tenant IDs at runtime from system data (e.g., message queues, inbox tables). diff --git a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs index e081aa9d..6fea2c0e 100644 --- a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs @@ -15,7 +15,7 @@ public interface IOutboxMessagePublisher /// with the number of successfully published messages as the result. ValueTask Publish( IReadOnlyCollection messages, - int tenantId = 0, + int tenantId, CancellationToken cancellationToken = default); @@ -27,27 +27,44 @@ async ValueTask Publish( Func getTenantId, CancellationToken cancellationToken = default) { - - var lookup = messages.ToLookup(getTenantId); ulong totals = 0; - - foreach (var tenantId in lookup.Select(g => g.Key)) + foreach (IGrouping group in messages.ToLookup(getTenantId)) { - var group = lookup[tenantId]; - var tenantMessages = group.ToArray(); + totals += await Publish([.. group], group.Key, cancellationToken); + } + + return totals; + } + - totals += await Publish(tenantMessages, tenantId, cancellationToken); + async ValueTask Publish( + IReadOnlyCollection messages, + CancellationToken cancellationToken = default) + where TMessage : class, IOutboxPublishable + { + ulong totals = 0; + foreach (var group in messages.ToLookup(m => m.GetTenantId())) + { + totals += await Publish([.. group], group.Key, cancellationToken); } return totals; } + /// /// Publishes a single message. /// ValueTask PublishSingle( TMessage message, - int tenantId = 0, + int tenantId, CancellationToken cancellationToken = default) => Publish([message], tenantId, cancellationToken); + + + ValueTask PublishSingle( + TMessage message, + CancellationToken cancellationToken = default) + where TMessage : class, IOutboxPublishable + => Publish([message], message.GetTenantId(), cancellationToken); } diff --git a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs index 007d9e14..451d441e 100644 --- a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs @@ -8,8 +8,7 @@ internal sealed class OutboxMessagePublisher( TimeProvider timeProvider, IOutboxBulkWriter bulkWriter, OutboxPublishSettings publishSettings, - IOutboxMessageMetadataProvider metadataProvider -) : IOutboxMessagePublisher + IOutboxMessageMetadataProvider metadataProvider) : IOutboxMessagePublisher { public async ValueTask Publish( IReadOnlyCollection messages, @@ -25,7 +24,8 @@ private async ValueTask Send( int tenantId, CancellationToken cancellationToken) { - var typeInfo = metadataProvider.GetMetadata(); + OutboxMessageMetadata typeInfo = metadataProvider.GetMetadata(); + DateTimeOffset now = timeProvider.GetUtcNow(); int maxBatchSize = publishSettings.MaxBatchSize; @@ -41,6 +41,7 @@ private async ValueTask Send( OutboxMessage[] payloads = DefaultArrayPool.Shared.Rent>(len); Span> payloadsSpan = payloads; + try { int count = 0; diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index 69bac808..75f4ba30 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple Outbox infra for publishing and using messages diff --git a/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs index a2f140d6..200c1a23 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs @@ -4,7 +4,15 @@ namespace Sa.Partitional.PostgreSql.Cache; internal interface IPartCache { - Task InCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); - Task EnsureCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task InCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); + Task EnsureCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); Task RemoveCache(string tableName, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs index 3264c1c0..cdd46239 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs @@ -1,7 +1,7 @@ -using System.Collections.Concurrent; -using Sa.Classes; +using Sa.Classes; using Sa.Extensions; using Sa.Partitional.PostgreSql.Classes; +using System.Collections.Concurrent; namespace Sa.Partitional.PostgreSql.Cache; @@ -15,7 +15,11 @@ IPartRepository repository private readonly ConcurrentDictionary>> _cache = new(); - public async Task InCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public async Task InCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) { if (sqlBuilder[tableName] == null) return false; @@ -46,7 +50,11 @@ private async Task> SelectPartsInDb(string tableName, Canc } } - public async Task EnsureCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public async Task EnsureCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) { bool result = await InCache(tableName, date, partValues, cancellationToken); if (result) return true; diff --git a/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs b/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs index b2273341..3cdfab0a 100644 --- a/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs +++ b/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs @@ -8,7 +8,9 @@ internal static class Setup { readonly static Guid JobId = Guid.Parse("7da81411-9db7-4553-8e93-bd1f12d02b38"); - public static IServiceCollection AddPartCleaning(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddPartCleaning( + this IServiceCollection services, + Action? configure = null) { if (configure != null) diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs index 50c91c11..cdf86766 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs @@ -3,6 +3,8 @@ public interface ISettingsBuilder { + string DefaultSchema { get; } + ISettingsBuilder AddSchema(Action schemaBuilder); ISettingsBuilder AddSchema(string schemaName, Action schemaBuilder); ITableSettingsStorage Build(); diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs index 1b39f36a..06764566 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs @@ -6,9 +6,11 @@ internal sealed class SettingsBuilder(string? searchPath) : ISettingsBuilder { private readonly Dictionary _schemas = []; + public string DefaultSchema { get; } = searchPath ?? "public"; + public ISettingsBuilder AddSchema(Action schemaBuilder) { - return AddSchema(searchPath ?? "public", schemaBuilder); + return AddSchema(DefaultSchema, schemaBuilder); } public ISettingsBuilder AddSchema(string schemaName, Action schemaBuilder) diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs index 7f6171ef..bc4c4cc0 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs @@ -1,20 +1,20 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Npgsql; using Sa.Data.PostgreSql; namespace Sa.Partitional.PostgreSql.Configuration.Builder; internal static class Setup { - public static IServiceCollection AddSettigs(this IServiceCollection services, Action build) + public static IServiceCollection AddSettings( + this IServiceCollection services, + Action build) { services.AddSingleton(build); services.TryAddSingleton(sp => { - string? searchPath = GetSearchPath(sp); - var builder = new SettingsBuilder(searchPath); + SettingsBuilder builder = new(GetDefaultSchema(sp)); var configurators = sp.GetServices>(); @@ -29,19 +29,11 @@ public static IServiceCollection AddSettigs(this IServiceCollection services, Ac return services; } - private static string? GetSearchPath(IServiceProvider sp) + private static string? GetDefaultSchema(IServiceProvider sp) { - var settings = sp.GetService(); - var connectionString = settings?.ConnectionString; - - if (string.IsNullOrWhiteSpace(connectionString)) - { - return null; - } - try { - return new NpgsqlConnectionStringBuilder(connectionString).SearchPath; + return sp.GetService()?.GetSearchPath(); } catch { diff --git a/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs b/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs index a06a2274..d48cda2b 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs @@ -15,7 +15,7 @@ internal sealed class PartConfiguration(IServiceCollection services) : IPartConf public IPartConfiguration AddPartTables(Action configure) { services - .AddSettigs(configure) + .AddSettings(configure) .AddSqlBuilder() ; diff --git a/src/Sa.Partitional.PostgreSql/IPartitionManager.cs b/src/Sa.Partitional.PostgreSql/IPartitionManager.cs index 62d78001..f16d9c96 100644 --- a/src/Sa.Partitional.PostgreSql/IPartitionManager.cs +++ b/src/Sa.Partitional.PostgreSql/IPartitionManager.cs @@ -1,4 +1,6 @@ -namespace Sa.Partitional.PostgreSql; +using Sa.Partitional.PostgreSql.Classes; + +namespace Sa.Partitional.PostgreSql; /// /// interface for managing partitions in the database @@ -31,6 +33,10 @@ public interface IPartitionManager /// An array of values that define the partitions (could be strings or numbers). /// A token to monitor for cancellation requests. /// A value task representing the asynchronous operation, with a boolean result indicating whether the partitions were ensured successfully. - Task EnsureParts(string tableName, DateTimeOffset date, Classes.StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task EnsureParts( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs index 55cf4852..6dfc3f5b 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs @@ -5,9 +5,7 @@ namespace Sa.Partitional.PostgreSql.Migration; internal sealed class PartMigrationService( IPartRepository repository , TimeProvider timeProvider - , MigrationScheduleSettings settings -) - : IMigrationService, IDisposable + , MigrationScheduleSettings settings): IMigrationService, IDisposable { private int s_triggered = 0; private readonly CancellationTokenSource _cts = new(); diff --git a/src/Sa.Partitional.PostgreSql/Part.cs b/src/Sa.Partitional.PostgreSql/Part.cs index d228d066..41ab0b5c 100644 --- a/src/Sa.Partitional.PostgreSql/Part.cs +++ b/src/Sa.Partitional.PostgreSql/Part.cs @@ -2,8 +2,7 @@ namespace Sa.Partitional.PostgreSql; -public sealed record Part(string Name, PartByRange PartBy) - : Enumeration(Name.GetHashCode(), Name) +public sealed record Part(string Name, PartByRange PartBy): Enumeration(Name.GetHashCode(), Name) { public const string RootId = "root"; diff --git a/src/Sa.Partitional.PostgreSql/PartitionManager.cs b/src/Sa.Partitional.PostgreSql/PartitionManager.cs index 127573dd..28e04401 100644 --- a/src/Sa.Partitional.PostgreSql/PartitionManager.cs +++ b/src/Sa.Partitional.PostgreSql/PartitionManager.cs @@ -6,7 +6,11 @@ namespace Sa.Partitional.PostgreSql; internal sealed class PartitionManager(IPartCache cache, IMigrationService migrationService) : IPartitionManager { - public Task EnsureParts(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public Task EnsureParts( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) => cache.EnsureCache(tableName, date, partValues, cancellationToken); public Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default) diff --git a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs index 47d59428..3cf77b55 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs @@ -10,7 +10,12 @@ namespace Sa.Partitional.PostgreSql; /// An array of values that define the partitioning criteria, which can be either string or numeric. /// The method used for partitioning (e.g., by range, list, etc.). /// The date from which this partition is valid. -public sealed record PartByRangeInfo(string Id, string RootTableName, StrOrNum[] PartValues, PgPartBy PartBy, DateTimeOffset FromDate); +public sealed record PartByRangeInfo( + string Id, + string RootTableName, + StrOrNum[] PartValues, + PgPartBy PartBy, + DateTimeOffset FromDate); /// /// Represents a repository interface for managing database partitions. @@ -18,10 +23,23 @@ public sealed record PartByRangeInfo(string Id, string RootTableName, StrOrNum[] /// public interface IPartRepository { - Task CreatePart(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task CreatePart( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); - Task Migrate(DateTimeOffset[] dates, Func> resolve, CancellationToken cancellationToken = default); - Task> GetPartsFromDate(string tableName, DateTimeOffset fromDate, CancellationToken cancellationToken = default); - Task> GetPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); + Task Migrate( + DateTimeOffset[] dates, + Func> resolve, + CancellationToken cancellationToken = default); + Task> GetPartsFromDate( + string tableName, + DateTimeOffset fromDate, + CancellationToken cancellationToken = default); + Task> GetPartsToDate( + string tableName, + DateTimeOffset toDate, + CancellationToken cancellationToken = default); Task DropPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs index 88ef8a12..7ab2e15f 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs @@ -39,9 +39,7 @@ public async Task CreatePart( StrOrNum[] partValues, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(partValues); - - ISqlTableBuilder builder = sqlBuilder[tableName] ?? throw new KeyNotFoundException(nameof(tableName)); + ISqlTableBuilder builder = sqlBuilder[tableName] ?? throw new KeyNotFoundException(tableName); string sql = builder.CreateSql(date, partValues); return await ExecuteDDL(sql, cancellationToken); @@ -85,7 +83,8 @@ public async Task Migrate(DateTimeOffset[] dates, CancellationToken cancell { if (tableSettings.PartByListFieldNames.Length > 0) { - throw new InvalidOperationException($"Migration support is required for table '{table}' because 'PartByListFieldNames' is specified."); + throw new InvalidOperationException( + $"Migration support is required for table '{table}' because 'PartByListFieldNames' is specified."); } } } @@ -108,7 +107,8 @@ public async Task> GetPartsFromDate( return await GetPartsFormDateWithRetry(sql, unixTime, cancellationToken); } - private async Task> GetPartsFormDateWithRetry(string sql, long unixTime, CancellationToken cancellationToken) + private async Task> GetPartsFormDateWithRetry( + string sql, long unixTime, CancellationToken cancellationToken) { return await PgRetryStrategy.ExecuteWithRetry( async t => @@ -149,7 +149,8 @@ public async Task> GetPartsToDate( } } - public async Task DropPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default) + public async Task DropPartsToDate( + string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default) { int droppedCount = 0; List list = await GetPartsToDate(tableName, toDate, cancellationToken); diff --git a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj index dda80a3a..a1c52b7c 100644 --- a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj +++ b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.9.0 For managing table partitioning in PostgreSQL diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs index ccd61013..40ec2caa 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs @@ -63,13 +63,11 @@ public string CreatePartSql(string tableName, DateTimeOffset date, StrOrNum[] pa #region privates private ISqlTableBuilder? Find(string tableName) { - ISqlTableBuilder? item = - ( - storage.Schemas.Count == 1 - ? builders.GetValueOrDefault(GetFullName(storage.Schemas.First(), tableName)) - : builders.Values.FirstOrDefault(c => c.FullName == tableName) - ) - ?? builders.GetValueOrDefault(tableName); + ISqlTableBuilder? item = tableName.Contains('.') + ? builders.GetValueOrDefault(tableName) + : builders.GetValueOrDefault(GetFullName(storage.Schemas.First(), tableName)); + + item ??= builders.Values.FirstOrDefault(c => c.FullName == tableName); if (item != null) return item; diff --git a/src/Sa.Schedule/Engine/IJobController.cs b/src/Sa.Schedule/Engine/IJobController.cs index 92d5f4e0..f958661e 100644 --- a/src/Sa.Schedule/Engine/IJobController.cs +++ b/src/Sa.Schedule/Engine/IJobController.cs @@ -13,22 +13,24 @@ internal enum CanJobExecuteResult /// internal interface IJobController { - // scope context - public IJobContext Context { get; } + int Index { get; } - // scope events - ValueTask WaitToRun(CancellationToken cancellationToken); + void Start(); - void Running(); + void Shutdown(); - void Stopped(TaskStatus status); + bool IsPaused { get; } - // iteration events - ValueTask CanExecute(CancellationToken cancellationToken); + void Pause(); - Task Execute(CancellationToken cancellationToken); + void Resume(); - void ExecutionFailed(Exception exception); + ValueTask WaitToRun(CancellationToken cancellationToken); + ValueTask WaitIfPaused(CancellationToken cancellationToken); + ValueTask CanExecute(CancellationToken cancellationToken); + + Task Execute(CancellationToken cancellationToken); void ExecutionCompleted(); + void ExecutionFailed(Exception exception); } diff --git a/src/Sa.Schedule/Engine/IJobFactory.cs b/src/Sa.Schedule/Engine/IJobFactory.cs index 84d988c2..2ec3c723 100644 --- a/src/Sa.Schedule/Engine/IJobFactory.cs +++ b/src/Sa.Schedule/Engine/IJobFactory.cs @@ -2,6 +2,5 @@ internal interface IJobFactory { - IJobController CreateJobController(IJobSettings settings); IJobScheduler CreateJobSchedule(IJobSettings settings); } diff --git a/src/Sa.Schedule/Engine/JobContext.cs b/src/Sa.Schedule/Engine/JobContext.cs index 0a4fc4ee..b6e65717 100644 --- a/src/Sa.Schedule/Engine/JobContext.cs +++ b/src/Sa.Schedule/Engine/JobContext.cs @@ -9,8 +9,6 @@ internal sealed class JobContext(IJobSettings settings) : IJobContext { public string JobName => settings.Properties.JobName ?? $"{settings.JobId}"; - public JobStatus Status { get; set; } - public IJobSettings Settings => settings; public ulong NumIterations { get; set; } @@ -29,13 +27,14 @@ internal sealed class JobContext(IJobSettings settings) : IJobContext public ulong NumRuns { get; set; } - public IServiceProvider JobServices { get; set; } = NullJobServices.Instance; - - public Queue Stack { get; private set; } = new(); + public Queue Stack { get; private set; } = []; IEnumerable IJobContext.Stack => Stack.Reverse(); - public ILogger Logger => JobServices.GetService>() ?? NullLogger.Instance; + public IServiceProvider ServiceProvider { get; set; } = NullJobServices.Instance; + + public ILogger Logger => ServiceProvider.GetService>() + ?? NullLogger.Instance; public IJobContext Clone() { @@ -47,7 +46,6 @@ public IJobContext Clone() LastError = LastError, CreatedAt = CreatedAt, ExecuteAt = ExecuteAt, - Status = Status, NumRuns = NumRuns, Stack = new Queue(Stack.Select(x => x.Clone())), }; diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index 693ad5a7..97b31e6f 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -9,68 +9,139 @@ namespace Sa.Schedule.Engine; /// job lifecycly controller with context /// internal sealed partial class JobController( + int index, IJobSettings settings, IInterceptorSettings interceptorSettings, IServiceScopeFactory scopeFactory, - TimeProvider timeProvider) : IJobController + TimeProvider timeProvider) : IJobController, IDisposable { - private readonly JobContext context = new(settings); + private readonly JobContext _context = new(settings); + private readonly SemaphoreSlim _pauseSemaphore = new(1, 1); - private JobPipeline? _job; + private volatile bool _disposed; + private volatile JobExecutor? _job; + private volatile bool _isPaused; + private readonly CancellationTokenSource _shutdownCts = new(); - public IJobContext Context => context; - public DateTimeOffset UtcNow => timeProvider.GetUtcNow(); + public int Index => index; + public bool IsPaused => _isPaused; + public async ValueTask WaitToRun(CancellationToken cancellationToken) { - if (context.NumRuns == 0 && settings.Properties.InitialDelay.HasValue && settings.Properties.InitialDelay.Value != TimeSpan.Zero) + if (_disposed) return; + + if (_context.NumRuns == 0 + && settings.Properties.InitialDelay.HasValue + && settings.Properties.InitialDelay.Value != TimeSpan.Zero) { - context.Status = JobStatus.WaitingToRun; await Task.Delay(settings.Properties.InitialDelay.Value, cancellationToken); } } - public void Running() + + public void Pause() + { + if (_disposed) return; + + if (Interlocked.CompareExchange(ref _isPaused, true, false)) + { + return; + } + + _pauseSemaphore.Wait(); // Блокируем семафор + } + + public void Resume() { - _job = new JobPipeline(settings, interceptorSettings, scopeFactory); + if (_disposed) return; + + if (!Interlocked.CompareExchange(ref _isPaused, false, true)) + { + return; + } - context.JobServices = _job.JobServices; - context.Status = JobStatus.Running; + ReleaseSemaphore(); + } - if (context.NumRuns == 0) context.CreatedAt = UtcNow; - context.NumRuns++; + private void ReleaseSemaphore() + { + try + { + _pauseSemaphore.Release(); + } + catch (ObjectDisposedException) + { + // ignore + } + catch (SemaphoreFullException) + { + // Уже разблокирован + } } - public void Stopped(TaskStatus status) + public async ValueTask WaitIfPaused(CancellationToken cancellationToken) { - switch (status) + if (_disposed || !_isPaused) return; + + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, _shutdownCts.Token); + + try { - case TaskStatus.Faulted: context.Status = JobStatus.Failed; break; - case TaskStatus.Canceled: context.Status = JobStatus.Cancelled; break; - case TaskStatus.RanToCompletion: context.Status = JobStatus.Completed; break; + await _pauseSemaphore.WaitAsync(cts.Token); } + finally + { + if (!_disposed && !_isPaused) + { + ReleaseSemaphore(); + } + } + } - context.JobServices = NullJobServices.Instance; + public void Start() + { + if (_disposed) return; + + _job = new JobExecutor(settings, interceptorSettings, scopeFactory); + + _context.ServiceProvider = _job.ServiceProvider; + + if (_context.NumRuns == 0) _context.CreatedAt = timeProvider.GetUtcNow(); + _context.NumRuns++; + } + + public void Shutdown() + { + if (_disposed) return; + + _disposed = true; + _shutdownCts.Cancel(); + + _context.ServiceProvider = NullJobServices.Instance; _job?.Dispose(); + _pauseSemaphore.Dispose(); + _shutdownCts.Dispose(); } public async ValueTask CanExecute(CancellationToken cancellationToken) { - if (settings.Properties.IsRunOnce == true && context.NumIterations > 0) + if (settings.Properties.IsRunOnce == true && _context.NumIterations > 0) return CanJobExecuteResult.Abort; - if (context.NumIterations == 0 && settings.Properties.Immediate == true) + if (_context.NumIterations == 0 && settings.Properties.Immediate == true) return CanJobExecuteResult.Ok; IJobTiming? timing = settings.Properties.Timing; if (timing != null) { - DateTimeOffset now = UtcNow; + DateTimeOffset now = timeProvider.GetUtcNow(); - DateTimeOffset? next = timing.GetNextOccurrence(now, context); + DateTimeOffset? next = timing.GetNextOccurrence(now, _context); if (!next.HasValue) return CanJobExecuteResult.Abort; @@ -87,8 +158,8 @@ public async ValueTask CanExecute(CancellationToken cancell if (stackSize > 0) { - if (context.Stack.Count == stackSize) context.Stack.Dequeue(); - context.Stack.Enqueue(context.Clone()); + if (_context.Stack.Count == stackSize) _context.Stack.Dequeue(); + _context.Stack.Enqueue(_context.Clone()); } return !cancellationToken.IsCancellationRequested @@ -98,39 +169,50 @@ public async ValueTask CanExecute(CancellationToken cancell public Task Execute(CancellationToken cancellationToken) { - context.NumIterations++; - context.ExecuteAt = UtcNow; - return _job!.Execute(Context, cancellationToken); + _context.NumIterations++; + _context.ExecuteAt = timeProvider.GetUtcNow(); + return _job!.Execute(_context, cancellationToken); } public void ExecutionCompleted() { - context.CompetedIterations++; - context.FailedRetries = 0; + _context.CompetedIterations++; + _context.FailedRetries = 0; } public void ExecutionFailed(Exception exception) { - JobException error = new(context, exception); - context.FailedIterations++; - context.LastError = error; + JobException error = new(_context, exception); + _context.FailedIterations++; + _context.LastError = error; IJobErrorHandling errorHandling = settings.ErrorHandling; - if (errorHandling.HasSuppressError && errorHandling.SuppressError?.Invoke(exception) == true) + if (errorHandling.HasSuppressError + && errorHandling.SuppressError?.Invoke(exception) == true) { - LogJobWasSuppressed(context.Logger, context.JobName, exception.GetType().Name, exception.Message); + LogJobWasSuppressed( + _context.Logger, + _context.JobName, + exception.GetType().Name, + exception.Message); return; } - if (context.FailedRetries < settings.ErrorHandling.RetryCount) + if (_context.FailedRetries < settings.ErrorHandling.RetryCount) { - context.FailedRetries++; - LogFailedRetryAttempts(context.Logger, context.JobName, context.FailedRetries, errorHandling.RetryCount, exception.GetType().Name, exception.Message); + _context.FailedRetries++; + LogFailedRetryAttempts( + _context.Logger, + _context.JobName, + _context.FailedRetries, + errorHandling.RetryCount, + exception.GetType().Name, + exception.Message); return; } - context.JobServices.GetService()?.HandleError(Context, error); + _context.ServiceProvider.GetService()?.HandleError(_context, error); } @@ -144,5 +226,8 @@ public void ExecutionFailed(Exception exception) EventId = 402, Level = LogLevel.Warning, Message = "[{JobName}] {FailedRetryAttempts} out of {RetryCount} reps when the job failed due to an error: {Type} “{Error}”")] - static partial void LogFailedRetryAttempts(ILogger logger, string jobName, int failedRetryAttempts, int retryCount, string type, string error); + static partial void LogFailedRetryAttempts( + ILogger logger, string jobName, int failedRetryAttempts, int retryCount, string type, string error); + + public void Dispose() => Shutdown(); } diff --git a/src/Sa.Schedule/Engine/JobErrorHandler.cs b/src/Sa.Schedule/Engine/JobErrorHandler.cs index f8739d80..84576e5c 100644 --- a/src/Sa.Schedule/Engine/JobErrorHandler.cs +++ b/src/Sa.Schedule/Engine/JobErrorHandler.cs @@ -29,11 +29,15 @@ private void DoHandleError(IJobContext context, Exception exception) break; case ErrorHandlingAction.CloseApplication: - CloseApplication(context.JobName, context.JobServices.GetRequiredService(), exception); + CloseApplication( + context.JobName, + context.ServiceProvider.GetRequiredService(), exception); break; case ErrorHandlingAction.StopAllJobs: - StopAllJobs(context.JobName, context.JobServices.GetRequiredService(), exception); + StopAllJobs( + context.JobName, + context.ServiceProvider.GetRequiredService(), exception); break; default: diff --git a/src/Sa.Schedule/Engine/JobPipeline.cs b/src/Sa.Schedule/Engine/JobExecutor.cs similarity index 74% rename from src/Sa.Schedule/Engine/JobPipeline.cs rename to src/Sa.Schedule/Engine/JobExecutor.cs index d17009e6..d5ba9b6e 100644 --- a/src/Sa.Schedule/Engine/JobPipeline.cs +++ b/src/Sa.Schedule/Engine/JobExecutor.cs @@ -3,7 +3,7 @@ namespace Sa.Schedule.Engine; -internal sealed class JobPipeline : IJob, IDisposable +internal sealed class JobExecutor : IJob, IDisposable { #region proxy class JobProxy(IJob job, IJobInterceptor interceptor, object? key) : IJob @@ -20,7 +20,7 @@ public Task Execute(IJobContext context, CancellationToken cancellationToken) private readonly IServiceScope _scope; private readonly IJob _job; - public JobPipeline( + public JobExecutor( IJobSettings settings, IInterceptorSettings interceptorSettings, IServiceScopeFactory scopeFactory) @@ -34,7 +34,10 @@ public JobPipeline( .Interceptors .Reverse() .Aggregate(originalJob, (job, s) - => new JobProxy(job, (IJobInterceptor)_scope.ServiceProvider.GetRequiredKeyedService(s.HandlerType, s.Key), s.Key)); + => new JobProxy( + job, + (IJobInterceptor)_scope.ServiceProvider.GetRequiredKeyedService(s.HandlerType, s.Key), + s.Key)); } else { @@ -42,9 +45,10 @@ public JobPipeline( } } - public IServiceProvider JobServices => _scope.ServiceProvider; + public IServiceProvider ServiceProvider => _scope.ServiceProvider; public void Dispose() => _scope.Dispose(); - public Task Execute(IJobContext context, CancellationToken cancellationToken) => _job.Execute(context, cancellationToken); + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => _job.Execute(context, cancellationToken); } diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index 25f8bb64..0dd03eb2 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -7,11 +7,21 @@ internal sealed class JobFactory( IServiceScopeFactory scopeFactory, IInterceptorSettings interceptorSettings, IJobRunner jobRunner, - TimeProvider timeProvider) : IJobFactory + TimeProvider? timeProvider = null) : IJobFactory { - public IJobController CreateJobController(IJobSettings settings) - => new JobController(settings, interceptorSettings, scopeFactory, timeProvider); - public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(jobRunner, CreateJobController(settings)); + => new JobScheduler( + settings, + jobRunner, + i => CreateController(i, settings)); + + private JobController CreateController(int index, IJobSettings settings) + { + return new( + index, + settings, + interceptorSettings, + scopeFactory, + timeProvider ?? TimeProvider.System); + } } diff --git a/src/Sa.Schedule/Engine/JobRunner.cs b/src/Sa.Schedule/Engine/JobRunner.cs index 9482e792..729824e2 100644 --- a/src/Sa.Schedule/Engine/JobRunner.cs +++ b/src/Sa.Schedule/Engine/JobRunner.cs @@ -2,17 +2,22 @@ namespace Sa.Schedule.Engine; - internal sealed class JobRunner() : IJobRunner { public async Task Run(IJobController controller, CancellationToken cancellationToken) { await controller.WaitToRun(cancellationToken); - controller.Running(); + controller.Start(); - await RunLoop(controller, cancellationToken) - .ContinueWith(t => controller.Stopped(t.Status), CancellationToken.None); + try + { + await RunLoop(controller, cancellationToken); + } + finally + { + controller.Shutdown(); + } } [StackTraceHidden] @@ -20,24 +25,41 @@ private static async Task RunLoop(IJobController controller, CancellationToken c { while (!cancellationToken.IsCancellationRequested) { - CanJobExecuteResult next = await controller.CanExecute(cancellationToken); + await controller.WaitIfPaused(cancellationToken); - if (next == CanJobExecuteResult.Abort) break; - if (next == CanJobExecuteResult.Skip) continue; + CanJobExecuteResult next = await controller.CanExecute(cancellationToken); - try - { - await controller.Execute(cancellationToken); - controller.ExecutionCompleted(); - } - catch (OperationCanceledException ex) when (ex.CancellationToken == cancellationToken) - { - // skip - } - catch (Exception ex) + switch (next) { - controller.ExecutionFailed(ex); + case CanJobExecuteResult.Abort: + return; + + case CanJobExecuteResult.Skip: + continue; + + case CanJobExecuteResult.Ok: + await ExecuteIteration(controller, cancellationToken); + break; } } } + + private static async Task ExecuteIteration( + IJobController controller, + CancellationToken cancellationToken) + { + try + { + await controller.Execute(cancellationToken); + controller.ExecutionCompleted(); + } + catch (OperationCanceledException ex) when (ex.CancellationToken == cancellationToken) + { + // Expected cancellation - silently exit + } + catch (Exception ex) + { + controller.ExecutionFailed(ex); + } + } } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 2e0c7d6b..47e96629 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -1,99 +1,223 @@ using Microsoft.Extensions.Primitives; +using Sa.Utils.WorkQueue; namespace Sa.Schedule.Engine; -internal sealed class JobScheduler(IJobRunner runner, IJobController controller) : IJobScheduler, IDisposable, IAsyncDisposable +internal sealed class JobScheduler : IJobScheduler { - private readonly static IChangeToken NoneChangeToken = new CancellationChangeToken(CancellationToken.None); + private readonly static IChangeToken NoneChangeToken + = new CancellationChangeToken(CancellationToken.None); - private readonly Lock _locked = new(); + private readonly Lock _lock = new(); - private TaskCompletionSource? _stoppingTask; - - private CancellationTokenSource _stoppingToken = new(); + private CancellationTokenSource _stoppingTokenSource = new(); private CancellationToken _originalToken; + private bool? _started = false; + private bool _disposed; - public IJobContext Context => controller.Context; + private readonly SaWorkQueue _jobs; + + private readonly Func _createController; + + private readonly IJobRunner _runner; + + private IReadOnlyList _jobControllers = []; + + + public JobScheduler( + IJobSettings settings, + IJobRunner runner, + Func createController) + { + + _runner = runner; + _createController = createController; + + JobId = settings.JobId; + + var concurrency = Math.Max(0, settings.Properties.ConcurrencyLimit ?? 1); + var maxConcurrency = Math.Max(1, settings.Properties.MaxConcurrency ?? concurrency); + + concurrency = Math.Clamp(concurrency, 0, maxConcurrency); + + _jobs = new SaWorkQueue(SaWorkQueueOptions.Create(CreateJob) + .WithQueueCapacity(maxConcurrency) + .WithMaxConcurrency(maxConcurrency) + .WithConcurrencyLimit(concurrency) + .WithSingleWriter(true) + ); + } + + private Task CreateJob(IJobController controller, CancellationToken ct) + => _runner.Run(controller, ct); + + public Guid JobId { get; } + + public int ConcurrencyLimit + { + get => _jobs.ConcurrencyLimit; + set + { + if (_jobs.ConcurrencyLimit == value) return; - public bool IsActive => _stoppingTask?.Task.Status == TaskStatus.WaitingForActivation; + _jobs.ConcurrencyLimit = value; + RefreshConcurrency(); + } + } - public IChangeToken GetActiveChangeToken() + public bool IsStarted { - if (_disposed) return NoneChangeToken; + get + { + lock (_lock) + { + return !_disposed && _started.GetValueOrDefault(); + } + } + } - lock (_locked) + + public int ActiveTasks => _jobs.QueueTasks; + + public IChangeToken StartChangeToken() + { + lock (_lock) { - return new CancellationChangeToken(_stoppingToken.Token); + if (_disposed) return NoneChangeToken; + return new CancellationChangeToken(_stoppingTokenSource.Token); } } - /// - /// Start all jobs - /// - public bool Start(CancellationToken cancellationToken) + + public async Task Start(CancellationToken cancellationToken) { - if (IsActive) return false; + CancellationToken stoppingToken; - lock (_locked) + lock (_lock) { - _stoppingToken.Cancel(); - _stoppingToken.Dispose(); + if (_disposed || (_started == null || _started == true)) + { + return false; + } + + _started = null; + + _stoppingTokenSource.Cancel(); + _stoppingTokenSource.Dispose(); _originalToken = cancellationToken; - _stoppingToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _stoppingTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - _stoppingTask = new TaskCompletionSource(); + stoppingToken = _stoppingTokenSource.Token; + } + + var maxCapacity = _jobs.MaxConcurrency; - _ = runner - .Run(controller, _stoppingToken.Token) - .ContinueWith(Done, CancellationToken.None); + List controllers = new(capacity: maxCapacity); - return true; + try + { + await _jobs.WaitForIdleAsync(_originalToken); + + for (var i = 0; i < maxCapacity; i++) + { + IJobController controller = _createController(i); + controller.Pause(); + controllers.Add(controller); + await _jobs.Enqueue(controller, stoppingToken); + } + } + finally + { + lock (_lock) + { + _jobControllers = controllers; + _started = true; + } } - } - public bool Restart() => Start(_originalToken); + RefreshConcurrency(); + + return true; + } - public Task Stop() + private void RefreshConcurrency() { - if (IsActive) + IReadOnlyList controllers; + + lock (_lock) { - lock (_locked) + controllers = _jobControllers; + } + + int limit = _jobs.ConcurrencyLimit; + + for (int i = 0; i < controllers.Count; i++) + { + if (i < limit) + { + controllers[i].Resume(); + } + else { - _stoppingToken?.Cancel(); + controllers[i].Pause(); } } - - return _stoppingTask?.Task ?? Task.CompletedTask; } - private void Done(Task task) + public async Task Stop() { - lock (_locked) + if (_disposed) return; + lock (_lock) { - _stoppingTask?.TrySetResult(); + if (_disposed || !_started.GetValueOrDefault()) return; + + _stoppingTokenSource.Cancel(); + _started = false; } + + await _jobs.WaitForIdleAsync(_originalToken); } public void Dispose() { - if (!_disposed) + if (_disposed) return; + CancellationTokenSource ctsStopping; + + lock (_lock) { + if (_disposed) return; _disposed = true; - _ = Stop(); - _stoppingToken.Dispose(); + ctsStopping = _stoppingTokenSource; + } + + try + { + ctsStopping.Cancel(); + } + catch (ObjectDisposedException) + { + // ignore } + + _jobs.Dispose(); + + ctsStopping.Dispose(); } public async ValueTask DisposeAsync() { - if (!_disposed) + if (_disposed) return; + lock (_lock) { + if (_disposed) return; _disposed = true; - await Stop(); - _stoppingToken.Dispose(); } + + await _jobs.DisposeAsync(); + + _stoppingTokenSource.Dispose(); } } diff --git a/src/Sa.Schedule/Engine/JobTiming.cs b/src/Sa.Schedule/Engine/JobTiming.cs index 05b986b1..a8551132 100644 --- a/src/Sa.Schedule/Engine/JobTiming.cs +++ b/src/Sa.Schedule/Engine/JobTiming.cs @@ -1,12 +1,15 @@ namespace Sa.Schedule.Engine; -internal sealed class JobTiming(Func nextTime, string name) : IJobTiming +internal sealed class JobTiming( + Func nextTime, string name) : IJobTiming { public string TimingName => name; - public DateTimeOffset? GetNextOccurrence(DateTimeOffset dateTime, IJobContext context) => nextTime(dateTime, context); + public DateTimeOffset? GetNextOccurrence( + DateTimeOffset dateTime, IJobContext context) => nextTime(dateTime, context); public static IJobTiming EveryTime(TimeSpan timeSpan, string? name = null) => new JobTiming((dateTime, _) => dateTime.Add(timeSpan), name ?? $"every {timeSpan}"); - public static IJobTiming Default { get; } = EveryTime(TimeSpan.FromSeconds(1), "default every seconds"); + public static IJobTiming Default { get; } + = EveryTime(TimeSpan.FromSeconds(1), "default every seconds"); } diff --git a/src/Sa.Schedule/Engine/ScheduleHost.cs b/src/Sa.Schedule/Engine/ScheduleHost.cs index 0920b6c0..d3c73089 100644 --- a/src/Sa.Schedule/Engine/ScheduleHost.cs +++ b/src/Sa.Schedule/Engine/ScheduleHost.cs @@ -4,10 +4,9 @@ namespace Sa.Schedule.Engine; internal sealed class ScheduleHost(IScheduler controller) : IHostedService { - public Task StartAsync(CancellationToken cancellationToken) + public async Task StartAsync(CancellationToken cancellationToken) { - controller.Start(cancellationToken); - return Task.CompletedTask; + await controller.Start(cancellationToken); } public async Task StopAsync(CancellationToken cancellationToken) diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index 437e177e..42fb5eb2 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -1,8 +1,9 @@ namespace Sa.Schedule.Engine; -internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) : IScheduler, IDisposable, IAsyncDisposable +internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) + : IScheduler, IDisposable, IAsyncDisposable { - private bool _disposed; + private volatile bool _disposed; public IScheduleSettings Settings => settings; @@ -13,20 +14,55 @@ internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) /// /// Start all jobs /// - public int Start(CancellationToken cancellationToken) => Schedules.Count(c => c.Start(cancellationToken)); + public async Task Start(CancellationToken cancellationToken) + { + var results = await Task.WhenAll(Schedules.Select(c => c.Start(cancellationToken))); + return results.Count(r => r); + } - public int Restart() => Schedules.Count(c => c.Restart()); + public async Task Restart(CancellationToken cancellationToken) + { + var results = await Task + .WhenAll(Schedules + .Where(c => c.IsStarted) + .Select(c => Task.Run(async () => + { + await c.Stop(); + await c.Start(cancellationToken); + return true; + }))); - public async Task Stop() => await Task.WhenAll(Schedules.Select(c => c.Stop())); + return results.Count(r => r); + } + + public async Task Stop() + { + await Task.WhenAll(Schedules.Select(c => c.Stop())); + } public void Dispose() { if (!_disposed) { _disposed = true; - _ = Stop(); + + foreach (var job in Schedules) + { + job.Dispose(); + } } } - public async ValueTask DisposeAsync() => await Stop(); + public async ValueTask DisposeAsync() + { + if (!_disposed) + { + _disposed = true; + await Stop(); + foreach (var job in Schedules) + { + await job.DisposeAsync(); + } + } + } } diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index a00680d7..4543323e 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -65,14 +65,22 @@ public interface IJobBuilder /// /// The number of seconds (default is 1). /// The current builder instance. - IJobBuilder EverySeconds(int seconds = 1) => EveryTime(TimeSpan.FromSeconds(seconds), $"every {seconds} seconds"); + IJobBuilder EverySeconds(int seconds = 1) + => EveryTime(TimeSpan.FromSeconds(seconds), $"every {seconds} seconds"); /// /// Configures the job to run every specified number of minutes. /// /// The number of minutes (default is 1). /// The current builder instance. - IJobBuilder EveryMinutes(int minutes = 1) => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); + IJobBuilder EveryMinutes(int minutes = 1) + => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); + + + IJobBuilder WithConcurrencyLimit(int limit); + + IJobBuilder WithMaxConcurrency(int limit); + /// /// Merges the specified job properties into the current job configuration. diff --git a/src/Sa.Schedule/IJobContext.cs b/src/Sa.Schedule/IJobContext.cs index 11b0a919..52fa5bec 100644 --- a/src/Sa.Schedule/IJobContext.cs +++ b/src/Sa.Schedule/IJobContext.cs @@ -12,9 +12,8 @@ namespace Sa.Schedule; /// public interface IJobContext { - Guid JobId => Settings.JobId; string JobName { get; } - JobStatus Status { get; } + IJobSettings Settings { get; } ulong NumIterations { get; } ulong FailedIterations { get; } @@ -23,10 +22,9 @@ public interface IJobContext DateTimeOffset CreatedAt { get; } DateTimeOffset? ExecuteAt { get; } JobException? LastError { get; } - IServiceProvider JobServices { get; } + IServiceProvider ServiceProvider { get; } IEnumerable Stack { get; } ILogger Logger { get; } IJobContext Clone(); - bool Active => Status == JobStatus.Running || Status == JobStatus.WaitingToRun; } diff --git a/src/Sa.Schedule/IJobProperties.cs b/src/Sa.Schedule/IJobProperties.cs index 47cdbce8..cd45c526 100644 --- a/src/Sa.Schedule/IJobProperties.cs +++ b/src/Sa.Schedule/IJobProperties.cs @@ -44,4 +44,17 @@ public interface IJobProperties /// Gets an optional tag associated with the job. /// object? Tag { get; } + + /// + /// Gets the current concurrency limit for the job when it starts. + /// This value cannot exceed . + /// Null means use the default limit = 1. + /// + int? ConcurrencyLimit { get; } + + /// + /// Gets the absolute maximum concurrency limit allowed for this job. + /// Null means = ConcurrencyLimit. + /// + int? MaxConcurrency { get; } } diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index b85a3d26..b9842a97 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -5,32 +5,37 @@ namespace Sa.Schedule; /// /// This individual task scheduler is responsible for managing specific tasks. /// -public interface IJobScheduler +public interface IJobScheduler: IDisposable, IAsyncDisposable { + /// + /// Gets the unique identifier of the job. + /// + Guid JobId { get; } + /// /// Gets a value indicating whether the job scheduler is currently active. /// - bool IsActive { get; } + bool IsStarted { get; } /// - /// Gets the context associated with the job scheduler. + /// Gets the number of tasks that are contained in the Job. /// - IJobContext Context { get; } + int ActiveTasks { get; } /// - /// Gets a change token that can be used to track changes to the active state of the scheduler. + /// /// - IChangeToken GetActiveChangeToken(); + int ConcurrencyLimit { get; set; } /// - /// Starts the job scheduler asynchronously. + /// Gets a change token that can be used to track changes to the active state of the scheduler. /// - bool Start(CancellationToken cancellationToken); + IChangeToken StartChangeToken(); /// - /// Restarts the job scheduler asynchronously. + /// Starts the job scheduler asynchronously. /// - bool Restart(); + Task Start(CancellationToken cancellationToken); /// /// Stops the job scheduler asynchronously. diff --git a/src/Sa.Schedule/IScheduleBuilder.cs b/src/Sa.Schedule/IScheduleBuilder.cs index 16beded6..f71e5728 100644 --- a/src/Sa.Schedule/IScheduleBuilder.cs +++ b/src/Sa.Schedule/IScheduleBuilder.cs @@ -10,7 +10,8 @@ public interface IScheduleBuilder /// The type of job to add. /// The ID of the job. If not specified, a new ID will be generated. /// A builder for the added job. - IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Guid? jobId = null) where T : class, IJob; + IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (Guid? jobId = null) where T : class, IJob; /// /// Adds a job with the specified action to the schedule. @@ -27,7 +28,8 @@ public interface IScheduleBuilder /// An action to configure the job. /// The ID of the job. If not specified, a new ID will be generated. /// The schedule builder. - IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Action configure, Guid? jobId = null) where T : class, IJob; + IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (Action configure, Guid? jobId = null) where T : class, IJob; /// /// Adds an interceptor of type to the schedule. @@ -35,7 +37,8 @@ public interface IScheduleBuilder /// The type of interceptor to add. /// The key to use for the interceptor. If not specified, a default key will be used. /// The schedule builder. - IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(object? key = null) where T : class, IJobInterceptor; + IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (object? key = null) where T : class, IJobInterceptor; /// /// Configures the schedule to use a hosted service. diff --git a/src/Sa.Schedule/IScheduler.cs b/src/Sa.Schedule/IScheduler.cs index fb3cb5ee..1002c779 100644 --- a/src/Sa.Schedule/IScheduler.cs +++ b/src/Sa.Schedule/IScheduler.cs @@ -20,17 +20,20 @@ public interface IScheduler /// /// The cancellation token. /// The number of jobs started. - int Start(CancellationToken cancellationToken); + Task Start(CancellationToken cancellationToken); /// /// Restarts the scheduler. /// /// The number of jobs restarted. - int Restart(); + Task Restart(CancellationToken cancellationToken); /// /// Stops the scheduler. /// /// A task representing the asynchronous operation. Task Stop(); + + + IJobScheduler? GetSchedule(Guid jobId) => Schedules.FirstOrDefault(c => c.JobId == jobId); } diff --git a/src/Sa.Schedule/JobStatus.cs b/src/Sa.Schedule/JobStatus.cs deleted file mode 100644 index bb67c901..00000000 --- a/src/Sa.Schedule/JobStatus.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Sa.Schedule; - -public enum JobStatus -{ - WaitingToRun = 2, - Running = 3, - Completed = 5, - Cancelled = 6, - Failed = 7 -} diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index fbc50cd7..4b1f3d11 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -1,27 +1,35 @@  - + - - 0.8.0 - Execute jobs on a schedule - + + 0.9.0 + Execute jobs on a schedule + - - 1701;1702;CS8602; - + + 1701;1702;CS8602; + - - 1701;1702;CS8602; - + + 1701;1702;CS8602; + - - - + + + - - - - + + + + + + + + + + + + diff --git a/src/Sa.Schedule/Settings/JobBuilder.cs b/src/Sa.Schedule/Settings/JobBuilder.cs index 7a0aaa40..fcce2385 100644 --- a/src/Sa.Schedule/Settings/JobBuilder.cs +++ b/src/Sa.Schedule/Settings/JobBuilder.cs @@ -62,6 +62,18 @@ public IJobBuilder WithContextStackSize(int size) return this; } + public IJobBuilder WithConcurrencyLimit(int limit) + { + settings.Properties.WithConcurrencyLimit(limit); + return this; + } + + public IJobBuilder WithMaxConcurrency(int limit) + { + settings.Properties.WithMaxConcurrencyLimit(limit); + return this; + } + public IJobBuilder Disabled() { settings.Properties.SetDisabled(); diff --git a/src/Sa.Schedule/Settings/JobProperies.cs b/src/Sa.Schedule/Settings/JobProperies.cs index f637b8cf..6c483650 100644 --- a/src/Sa.Schedule/Settings/JobProperies.cs +++ b/src/Sa.Schedule/Settings/JobProperies.cs @@ -12,18 +12,77 @@ internal sealed class JobProperies : IJobProperties public IJobTiming? Timing { get; private set; } public object? Tag { get; private set; } public int? ContextStackSize { get; private set; } + public int? ConcurrencyLimit { get; private set; } + public int? MaxConcurrency { get; private set; } - public void WithName(string name) => JobName = name; - public void RunOnce() => IsRunOnce = true; - public void StartImmediate() => Immediate = true; - public void WithInitialDelay(TimeSpan time) => InitialDelay = time; - public void WithTiming(IJobTiming timing) => Timing = timing; - public void SetDisabled() => Disabled = true; - public void WithContextStackSize(int size) => ContextStackSize = size; - public void WithTag(object tag) => Tag = tag; + public JobProperies WithName(string name) + { + JobName = name; + return this; + } + + public JobProperies RunOnce() + { + IsRunOnce = true; + return this; + } + + public JobProperies StartImmediate() + { + Immediate = true; + return this; + } + + public JobProperies WithInitialDelay(TimeSpan time) + { + InitialDelay = time; + return this; + } + + public JobProperies WithTiming(IJobTiming timing) + { + Timing = timing; + return this; + } + + public JobProperies SetDisabled() + { + Disabled = true; + return this; + } + + public JobProperies WithContextStackSize(int size) + { + ContextStackSize = size; + return this; + } + + public JobProperies WithTag(object tag) + { + Tag = tag; + return this; + } + + public JobProperies EveryTime(TimeSpan timeSpan, string? name = null) + { + Timing = JobTiming.EveryTime(timeSpan, name); + return this; + } + + public JobProperies WithConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + ConcurrencyLimit = limit; + return this; + } + + public JobProperies WithMaxConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + MaxConcurrency = limit; + return this; + } - public void EveryTime(TimeSpan timeSpan, string? name = null) - => Timing = JobTiming.EveryTime(timeSpan, name); internal JobProperies Merge(IJobProperties props) { @@ -35,6 +94,10 @@ internal JobProperies Merge(IJobProperties props) InitialDelay ??= props.InitialDelay; ContextStackSize ??= props.ContextStackSize; Tag ??= props.Tag; + + ConcurrencyLimit ??= props.ConcurrencyLimit; + MaxConcurrency ??= props.MaxConcurrency; + return this; } } diff --git a/src/Sa.Schedule/Settings/JobSettings.cs b/src/Sa.Schedule/Settings/JobSettings.cs index 86c1f09d..15f52876 100644 --- a/src/Sa.Schedule/Settings/JobSettings.cs +++ b/src/Sa.Schedule/Settings/JobSettings.cs @@ -5,7 +5,7 @@ internal sealed class JobSettings(Type jobType, Guid jobId) : IJobSettings /// /// handler id /// - public Guid JobId { get; } = jobId; + public Guid JobId => jobId; public Type JobType => jobType; diff --git a/src/Sa.Schedule/Settings/ScheduleBuilder.cs b/src/Sa.Schedule/Settings/ScheduleBuilder.cs index d7e29720..3954aeaa 100644 --- a/src/Sa.Schedule/Settings/ScheduleBuilder.cs +++ b/src/Sa.Schedule/Settings/ScheduleBuilder.cs @@ -20,7 +20,12 @@ public ScheduleBuilder(IServiceCollection services) _services.TryAddSingleton(sp => { IEnumerable jobSettings = sp.GetServices(); - ScheduleSettings settings = ScheduleSettings.Create(jobSettings, _isHostedService, _handleError); + + ScheduleSettings settings = ScheduleSettings.Create( + jobSettings, + _isHostedService, + _handleError); + return settings; }); @@ -32,8 +37,9 @@ public ScheduleBuilder(IServiceCollection services) } - public IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Guid? jobId = null) - where T : class, IJob + public IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + Guid? jobId = null) + where T : class, IJob { Guid id = GetId(jobId); _services.TryAddKeyedScoped(id); @@ -44,8 +50,10 @@ public ScheduleBuilder(IServiceCollection services) return new JobBuilder(jobSettings); } - public IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Action configure, Guid? jobId = null) - where T : class, IJob + public IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + Action configure, + Guid? jobId = null) + where T : class, IJob { Guid id = GetId(jobId); _services.TryAddKeyedScoped(id); @@ -88,8 +96,9 @@ public IScheduleBuilder UseHostedService() return this; } - public IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(object? key = null) - where T : class, IJobInterceptor + public IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + object? key = null) + where T : class, IJobInterceptor { _services.AddSingleton(new JobInterceptorSettings(typeof(T), key)); _services.TryAddKeyedScoped(key); diff --git a/src/Sa.Schedule/Settings/ScheduleSettings.cs b/src/Sa.Schedule/Settings/ScheduleSettings.cs index 24b08406..9f5cbd9f 100644 --- a/src/Sa.Schedule/Settings/ScheduleSettings.cs +++ b/src/Sa.Schedule/Settings/ScheduleSettings.cs @@ -2,28 +2,41 @@ internal sealed class ScheduleSettings : IScheduleSettings { - private Dictionary _storage = []; + private readonly IReadOnlyDictionary _storage; - public bool IsHostedService { get; private set; } + private ScheduleSettings( + IReadOnlyDictionary storage, + Func? handleError, + bool isHostedService) + { + _storage = storage; + IsHostedService = isHostedService; + HandleError = handleError; + } - public Func? HandleError { get; private set; } + public bool IsHostedService { get; } - public IEnumerable GetJobSettings() => _storage.Values.Where(c => c.Properties.Disabled != true); + public Func? HandleError { get; } - public void UseHostedService() => IsHostedService = true; + public IEnumerable GetJobSettings() + => _storage.Values.Where(c => c.Properties.Disabled != true); - internal static ScheduleSettings Create(IEnumerable jobSettings, bool isHostedService, Func? handleError) + internal static ScheduleSettings Create( + IEnumerable jobSettings, + bool isHostedService, + Func? handleError) { IEnumerable items = jobSettings.GroupBy( c => (c.JobId, c.JobType) - , (k, items) => items.Aggregate(seed: new JobSettings(k.JobType, k.JobId), (s1, s2) => s1.Merge(s2)) + , (k, items) => items.Aggregate( + seed: new JobSettings(k.JobType, k.JobId), + (s1, s2) => s1.Merge(s2)) ); - return new ScheduleSettings - { - HandleError = handleError, - IsHostedService = isHostedService, - _storage = items.ToDictionary(c => c.JobId) - }; + return new ScheduleSettings( + storage: items.ToDictionary(c => c.JobId), + handleError: handleError, + isHostedService: isHostedService + ); } } diff --git a/src/Sa.Schedule/Setup.cs b/src/Sa.Schedule/Setup.cs index 4c378176..0613c46f 100644 --- a/src/Sa.Schedule/Setup.cs +++ b/src/Sa.Schedule/Setup.cs @@ -16,7 +16,9 @@ public static class Setup /// The service collection to add the scheduling system to. /// An action to configure the scheduling system. /// The service collection with the scheduling system added. - public static IServiceCollection AddSaSchedule(this IServiceCollection services, Action configure) + public static IServiceCollection AddSaSchedule( + this IServiceCollection services, + Action configure) { services.TryAddSingleton(TimeProvider.System); services.TryAddSingleton(); diff --git a/src/Sa.Utils.WorkQueue/ISaWork.cs b/src/Sa.Utils.WorkQueue/ISaWork.cs new file mode 100644 index 00000000..4400270b --- /dev/null +++ b/src/Sa.Utils.WorkQueue/ISaWork.cs @@ -0,0 +1,10 @@ +namespace Sa.Utils.WorkQueue; + + +/// +/// BLL interface. +/// +public interface ISaWork +{ + Task Execute(TInput input, CancellationToken cancellationToken); +} diff --git a/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs b/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs new file mode 100644 index 00000000..08f20212 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs @@ -0,0 +1,21 @@ +namespace Sa.Utils.WorkQueue; + +/// +/// Asynchronous task queue with limited parallelism. +/// +public interface ISaWorkQueue : IDisposable, IAsyncDisposable +{ + bool IsEnabled { get; } + int QueueTasks { get; } + bool IsIdle(); + int ConcurrencyLimit { get; set; } + int MaxConcurrency { get; } + int QueueCapacity { get; } + + Exception? ShutdownError { get; } + + ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default); + Task WaitForIdleAsync(CancellationToken cancellationToken = default); + Task ShutdownAsync(); + void ForceCancelReaders(); +} diff --git a/src/Sa.Utils.WorkQueue/Readme.md b/src/Sa.Utils.WorkQueue/Readme.md new file mode 100644 index 00000000..3d1e4aff --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Readme.md @@ -0,0 +1,127 @@ +# SaWorkQueue — Async Queue with Concurrency Limiting + +> High-performance task queue for .NET 10 with dynamic scaling, DI integration, and a type-safe API. + +--- + +## Features + +| Feature | Description | +|---------|-------------| +| **Concurrency limiting** | Control the number of simultaneously executing tasks via `ConcurrencyLimit` | +| **Dynamic scaling** | Change the limit at runtime: `queue.ConcurrencyLimit = newLimit` | +| **Scaling strategies** | `Lifo` • `Fifo` • `RoundRobin` • `Random` — choose the one that fits your scenario | +| **DI integration** | Registration via `AddWorkQueue` with configuration support | +| **Zero-allocation logging** | `[LoggerMessage]` generation for `ILogger` | +| **Safe shutdown** | `DisposeAsync`, `ShutdownAsync`, `WaitForIdleAsync` — idempotent and thread‑safe | + +--- + +## 🚀 Quick Start + +### 1️⃣ Implement your task processor + +```csharp +public sealed class OrderWork : ISaWork +{ + private readonly ILogger _logger; + public OrderWork(ILogger logger) => _logger = logger; + + public async Task Execute(OrderInput input, CancellationToken ct) + { + _logger.LogInformation("Processing order {OrderId}", input.OrderId); + await ProcessAsync(input, ct); // Your business logic + } +} +``` + +### 2️⃣ Register with DI + +```csharp +builder.Services.AddSaWorkQueue((sp, opts) => + opts + .WithConcurrencyLimit(4) + .WithQueueCapacity(100) + .WithReaderScalingStrategy(SaReaderScalingStrategy.RoundRobin) + .WithStatusChanged((input, status, ex) => + { + // logger.LogDebug("Order {Id} → {Status}", input.OrderId, status); + })); +``` + +### 3️⃣ Use via injection + +```csharp +public class OrderService(ISaWorkQueue queue) +{ + public async Task SubmitAsync(OrderInput order, CancellationToken ct) + { + await queue.Enqueue(order, ct); // Does not block the caller + } + + public bool IsIdle() => queue.IsIdle(); + public int Pending => queue.QueueTasks; +} +``` + +--- + +## ⚙️ `WorkQueueOptions` Configuration + +```csharp +SaWorkQueueOptions.Create(processor) + .WithConcurrencyLimit(int) // Concurrency limit (default: CPU count) + .WithQueueCapacity(int) // Queue capacity (default: equals the limit) + .WithMaxConcurrency(int) // Absolute maximum number of readers + .WithReaderScalingStrategy(enum) // Lifo | Fifo | RoundRobin | Random + .WithSingleWriter(bool) // Optimisation for a single write source + .WithFullMode(enum) // Wait | DropWrite | DropOldest + .WithStatusCallback(Action<...>) // Callback on task status change + .WithHandleItemFaulted(Func<...>) // Continue | StopReader | ShutdownQueue + .WithItemDisplayName(Func<...>) // Display name for each work item (e.g., for logging) +``` + +--- + +## Reader Scaling Strategies + +| Strategy | Behaviour | Best for | +|----------|-----------|----------| +| `Lifo` (default) | Cancels the most recently created | CPU‑bound tasks, cache locality | +| `Fifo` | Cancels the earliest created | Connection pools, databases, rotation | +| `RoundRobin` 🔄 | Cycles through the queue | Load balancing, stable workers | +| `Random` 🎲 | Random selection | Testing, avoiding patterns | + +--- + +## 🔑 Key Methods of `ISaWorkQueue` + +```csharp +// Enqueue a task (does not block if there is room in the queue) +await queue.Enqueue(input, cancellationToken); + +// Wait until all tasks have been fully processed +await queue.WaitForIdleAsync(ct); + +// Graceful shutdown: finish active tasks + clear the queue +await queue.ShutdownAsync(); + +// Emergency stop of readers (without waiting for completion) +queue.ForceCancelReaders(); + +// Monitoring +bool idle = queue.IsIdle(); // true if no active or pending tasks +int pending = queue.QueueTasks; // total tasks in progress + in the queue +int limit = queue.ConcurrencyLimit; // current concurrency limit +``` + +--- + +## ⚠️ Important Notes + +1. **Lifecycle**: the queue is registered as `Singleton`. Do not use `Scoped`/`Transient`. +2. **`StatusChanged`**: invoked on a thread‑pool thread. Avoid long synchronous operations inside. +3. **Cancellation**: tasks receive a `CancellationToken`. Handle `OperationCanceledException` properly. +4. **`ForceCancelReaders`**: use only in emergencies — may leave the queue in an inconsistent state. +5. **Reusability**: all shutdown/cleanup methods (`Shutdown`, `Dispose`) are idempotent — safe to call multiple times. + diff --git a/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj new file mode 100644 index 00000000..de70e49e --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj @@ -0,0 +1,22 @@ + + + + + + 0.9.0 + WorkQueue wrapper for Channels + + + + 1701;1702;CS8602; + + + + 1701;1702;CS8602; + + + + + + + diff --git a/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs b/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs new file mode 100644 index 00000000..64f5cb45 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs @@ -0,0 +1,13 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaExecutionErrorStrategy +{ + /// Mark the item as Faulted and continue processing (default). + Continue, + + /// Mark the item as Faulted and stop the current reader (it will be replaced). + StopReader, + + /// Mark the item as Faulted and initiate a shutdown of the entire queue. + ShutdownQueue +} diff --git a/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs b/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs new file mode 100644 index 00000000..d6a8eb27 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs @@ -0,0 +1,26 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaReaderScalingStrategy +{ + /// + /// Cancel the most recent readers (LIFO). + /// + /// Older readers live longer — useful for caches, connection pools. + Lifo = 0, + + /// + /// Cancel the oldest readers (FIFO). + /// + /// Even lifetime distribution — useful for resource rotation. + Fifo = 1, + + /// + /// Round‑robin: cancel readers in cyclic order. + /// + RoundRobin = 2, + + /// + /// Random: cancel random readers (uniform distribution). + /// + Random = 3 +} diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs new file mode 100644 index 00000000..e4f460c9 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs @@ -0,0 +1,617 @@ +namespace Sa.Utils.WorkQueue; + +using Microsoft.Extensions.Logging; +using System.Threading; +using System.Threading.Channels; + +public sealed partial class SaWorkQueue : ISaWorkQueue +{ + private sealed record WorkItem(TInput Input, CancellationToken CancellationToken); + + private enum QueueState + { + Active = 0, + Shutdown = 1, + Disposed = 2 + } + + private readonly Channel _queue; + + private readonly Lock _wiSync = new(); + private readonly Lock _readersSync = new(); + private readonly Lock _cbSync = new(); + + private readonly ILogger? _logger; + private readonly ISaWork _processor; + + private readonly Action? _statusChanged; + private readonly Func _handleItemFaulted; + private readonly Func _getItemDisplayName; + + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly int _maxConcurrency; + private readonly int _queueCapacity; + + private volatile int _concurrency; + /// + /// 0: Active, 1: Shutdown, 2: Disposed + /// + private volatile QueueState _state; + + private volatile Exception? _shutdownError = null; + + // список ожидающих задач + private volatile int _taskCount; + private TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private volatile int _readerCount; + + private readonly List _ctsReaders = []; + private readonly List _taskReaders = []; + + private readonly SaReaderScalingStrategy _scalingStrategy; + private int _lastRemovedIndex = -1; // Для RoundRobin + + + public SaWorkQueue(SaWorkQueueOptions options, ILogger>? logger = null) + { + ArgumentNullException.ThrowIfNull(options.Processor); + + _logger = logger; + _processor = options.Processor; + + _statusChanged = options.StatusChanged; + + _maxConcurrency = options.MaxConcurrency > 0 ? options.MaxConcurrency.Value : Environment.ProcessorCount; + _concurrency = Math.Clamp(options.ConcurrencyLimit ?? Environment.ProcessorCount, 0, _maxConcurrency); + _queueCapacity = options.QueueCapacity ?? _maxConcurrency; + + _scalingStrategy = options.ReaderScalingStrategy; + _getItemDisplayName = options.GetItemDisplayName ?? (item => $"{item}"); + _handleItemFaulted = options.HandleItemFaulted ?? ((_, ex) => SaExecutionErrorStrategy.ShutdownQueue); + + _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) + { + AllowSynchronousContinuations = false, + SingleReader = false, + SingleWriter = options.SingleWriter ?? false, + FullMode = options.FullMode + }); + + SpawnReaders(_concurrency); + } + + public bool IsEnabled => _state == QueueState.Active; + + public int QueueTasks => _taskCount; + + public bool IsIdle() => QueueTasks == 0 && _queue.Reader.Count == 0; + + public int MaxConcurrency => _maxConcurrency; + public int QueueCapacity => _queueCapacity; + public Exception? ShutdownError => _shutdownError; + + public int ConcurrencyLimit + { + get => _concurrency; + set + { + var newLimit = Math.Clamp(value, 0, _maxConcurrency); + var oldLimit = Interlocked.Exchange(ref _concurrency, newLimit); + if (newLimit != oldLimit && IsEnabled) + { + AdjustReaders(newLimit); + } + } + } + + public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); + if (!IsEnabled) ThrowHelper.QueueStopped(); + + var wi = new WorkItem(input, cancellationToken); + MarkActive(); + + try + { + await _queue.Writer.WriteAsync(wi, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + MarkInactive(); + + if (ex is ChannelClosedException) + { + ThrowHelper.QueueStopped(); + } + } + } + + public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + _shutdownCts.Token, + cancellationToken); + + while (!IsIdle()) + { + TaskCompletionSource tcs; + lock (_wiSync) { tcs = _idleTcs; } + try + { + await tcs.Task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + // ignore + } + } + } + + + /// + /// Forcefully cancels all readers without waiting. + /// Use only in emergency situations (e.g., when hanging). + /// + public void ForceCancelReaders() + { + if (!IsEnabled) return; + + CancellationTokenSource[] readers; + + lock (_readersSync) + { + readers = [.. _ctsReaders.Where(c => !c.IsCancellationRequested)]; + } + + foreach (var reader in readers) + { + try { reader.Cancel(); } catch (ObjectDisposedException) { /* ignored */ } + } + + lock (_wiSync) + { + _idleTcs.TrySetResult(); + } + } + + public async Task ForceCancelReadersAsync() + { + if (!IsEnabled) return; + + CancellationTokenSource[] readers; + + lock (_readersSync) + { + readers = [.. _ctsReaders.Where(c => !c.IsCancellationRequested)]; + } + + foreach (var reader in readers) + { + try { await reader.CancelAsync(); } catch (ObjectDisposedException) { /* ignored */ } + } + + lock (_wiSync) + { + _idleTcs.TrySetResult(); + } + } + + private void SpawnReaders(int count) + { + lock (_readersSync) + { + for (var i = 0; i < count; i++) + { + StartReaderUnderLock(); + } + } + } + + private void StartReaderUnderLock() + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(_shutdownCts.Token); + var task = ReaderLoopAsync(cts); + + _ctsReaders.Add(cts); + _taskReaders.Add(task); + _readerCount++; + } + + private void AdjustReaders(int target) + { + if (!IsEnabled) return; + + lock (_readersSync) + { + int current = _readerCount; + var delta = target - current; + if (delta > 0) + { + for (int i = 0; i < delta; i++) + { + StartReaderUnderLock(); + } + } + else if (delta < 0) + { + var total = _ctsReaders.Count; + if (total == 0) return; + var toCancel = Math.Min(-delta, total); + + var cancellationSources = SelectCancellationSources(toCancel, total); + foreach (var cts in cancellationSources) + { + cts.Cancel(); + } + } + } + } + + private List SelectCancellationSources(int toCancel, int total) + { + var list = new List(toCancel); + + IEnumerable indices = _scalingStrategy switch + { + SaReaderScalingStrategy.Lifo => Enumerable.Range(total - toCancel, toCancel), + SaReaderScalingStrategy.Fifo => Enumerable.Range(0, toCancel), + SaReaderScalingStrategy.RoundRobin => GetRoundRobinIndices(total, toCancel), + SaReaderScalingStrategy.Random => GetRandomIndices(total, toCancel), + _ => Enumerable.Range(total - toCancel, toCancel) + }; + + foreach (var idx in indices.Where(c => c < total)) + { + list.Add(_ctsReaders[idx]); + } + + return list; + } + + private IEnumerable GetRoundRobinIndices(int totalCount, int toCancel) + { + if (totalCount == 0) yield break; + var start = (_lastRemovedIndex + 1) % totalCount; + for (var i = 0; i < toCancel; i++) + { + yield return (start + i) % totalCount; + } + _lastRemovedIndex = (start + toCancel - 1) % totalCount; + } + + private static IEnumerable GetRandomIndices(int totalCount, int toCancel) + { + if (toCancel >= totalCount) + { + for (var i = 0; i < totalCount; i++) yield return i; + yield break; + } + + var selected = new HashSet(toCancel); + while (selected.Count < toCancel) + { + selected.Add(Random.Shared.Next(0, totalCount)); + } + + foreach (var idx in selected) yield return idx; + } + + private async Task ReaderLoopAsync(CancellationTokenSource cts) + { + try + { + while (!cts.IsCancellationRequested) + { + WorkItem item; + try + { + item = await _queue.Reader.ReadAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (ChannelClosedException) + { + break; + } + + bool isContinue = false; + + using var ctsExec = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, item.CancellationToken); + + + isContinue = await ExecuteItemAsync(item, ctsExec.Token).ConfigureAwait(false); + if (!isContinue) break; + + } + } + catch (Exception ex) + { + LogReaderError(_logger, ex); + await ShutdownAsync(); + } + finally + { + RemoveReader(cts); + } + } + + private void RemoveReader(CancellationTokenSource cts) + { + lock (_readersSync) + { + _ctsReaders.Remove(cts); + _readerCount--; + _concurrency = _readerCount; + } + } + + private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) + { + try + { + ct.ThrowIfCancellationRequested(); + OnStatusChanged(item.Input, SaWorkStatus.Running); + await _processor.Execute(item.Input, ct).ConfigureAwait(false); + OnStatusChanged(item.Input, SaWorkStatus.Completed); + return true; + } + catch (OperationCanceledException ex) when (item.CancellationToken.IsCancellationRequested) + { + OnStatusChanged(item.Input, SaWorkStatus.Aborted); + LogItemAborted(_logger, _getItemDisplayName(item.Input), ex); + return true; + } + catch (OperationCanceledException ex) when (ct.IsCancellationRequested) + { + OnStatusChanged(item.Input, SaWorkStatus.Cancelled); + LogItemCancelled(_logger, _getItemDisplayName(item.Input), ex); + return false; + } + catch (Exception ex) + { + var dislpayItem = _getItemDisplayName(item.Input); + + SaExecutionErrorStrategy errorStrategy = SaExecutionErrorStrategy.ShutdownQueue; + try + { + OnStatusChanged(item.Input, SaWorkStatus.Faulted, ex); + LogItemExecutionFailed(_logger, dislpayItem, ex); + + errorStrategy = _handleItemFaulted(item.Input, ex); + } + catch (Exception callbackEx) + { + LogItemHandlerFailed(_logger, dislpayItem, callbackEx); + } + + return errorStrategy switch + { + SaExecutionErrorStrategy.Continue => true, + SaExecutionErrorStrategy.StopReader => false, + SaExecutionErrorStrategy.ShutdownQueue => HandleShutdownOnError(ex), + _ => true + }; + } + finally + { + MarkInactive(); + } + } + + private bool HandleShutdownOnError(Exception ex) + { + _shutdownError = ex; + _ = ShutdownAsync(); + return false; + } + + private void MarkActive() + { + TaskCompletionSource? tcs = null; + lock (_wiSync) + { + if (_taskCount++ == 0) + { + tcs = _idleTcs; + _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + tcs?.TrySetResult(); + } + + private void MarkInactive() + { + TaskCompletionSource? tcs = null; + lock (_wiSync) + { + if (--_taskCount == 0) + { + tcs = _idleTcs; + } + } + tcs?.TrySetResult(); + } + + private void OnStatusChanged(TInput item, SaWorkStatus status, Exception? error = null) + { + if (_statusChanged is null) return; + + Exception? handlerEx = null; + lock (_cbSync) + { + try + { + _statusChanged(item, status, error); + } + catch (Exception ex) + { + handlerEx = ex; + } + } + + if (handlerEx is not null) + { + LogItemHandlerFailed(_logger, _getItemDisplayName(item), handlerEx); + } + } + + public async Task ShutdownAsync() + { + if (Interlocked.CompareExchange(ref _state, QueueState.Shutdown, QueueState.Active) != QueueState.Active) + return; + + try + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + _queue.Writer.TryComplete(); + await WaitForReadersToCompleteAsync(); + ClearRemainingItems(); + } + catch (Exception ex) + { + LogShutdownError(_logger, ex); + } + } + + private async Task WaitForReadersToCompleteAsync() + { + Task[] tasks; + lock (_readersSync) + { + tasks = [.. _taskReaders]; + } + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + public void Shutdown() + { + if (Interlocked.CompareExchange(ref _state, QueueState.Shutdown, QueueState.Active) != QueueState.Active) + return; + + try + { + _shutdownCts.Cancel(); + _queue.Writer.TryComplete(); + + Task[] tasks; + lock (_readersSync) + { + tasks = [.. _taskReaders]; + } + + Task.WhenAll(tasks).GetAwaiter().GetResult(); + + ClearRemainingItems(); + } + catch (Exception ex) + { + LogShutdownError(_logger, ex); + } + } + + private void ClearRemainingItems() + { + OperationCanceledException? err = null; + + while (_queue.Reader.TryRead(out var item)) + { + err ??= ThrowHelper.QueueShutdownException; + OnStatusChanged(item.Input, SaWorkStatus.Faulted, err); + MarkInactive(); + } + } + + private void CompleteDispose() + { + if (Interlocked.Exchange(ref _state, QueueState.Disposed) != QueueState.Disposed) + { + _shutdownCts.Dispose(); + } + } + + public void Dispose() + { + Shutdown(); + CompleteDispose(); + } + + public async ValueTask DisposeAsync() + { + if (IsEnabled) + { + await ShutdownAsync().ConfigureAwait(false); + } + CompleteDispose(); + } + + + #region Logging Definitions (Source Generator) + + private static partial class LogMessages + { + [LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = "[{Item}] processing was cancelled")] + public static partial void ItemCancelled(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = "[{Item}] processing was aborted")] + public static partial void ItemAborted(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 3, Level = LogLevel.Error, Message = "[{Item}] execution failed")] + public static partial void ItemExecutionFailed(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 4, Level = LogLevel.Error, Message = "[{Item}] event handler failed for work item")] + public static partial void ItemHandlerFailed(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 5, Level = LogLevel.Error, Message = "ReaderTask error")] + public static partial void ReaderError(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 6, Level = LogLevel.Error, Message = "Error during shutdown")] + public static partial void ShutdownError(ILogger logger, Exception exception); + } + + private static void LogItemCancelled(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemCancelled(logger, item, ex); + } + + private static void LogItemAborted(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemAborted(logger, item, ex); + } + + private static void LogItemExecutionFailed(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemExecutionFailed(logger, item, ex); + } + + private static void LogItemHandlerFailed(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemHandlerFailed(logger, item, ex); + } + + private static void LogReaderError(ILogger? logger, Exception ex) + { + if (logger is not null) LogMessages.ReaderError(logger, ex); + } + + private static void LogShutdownError(ILogger? logger, Exception ex) + { + if (logger is not null) LogMessages.ShutdownError(logger, ex); + } + + #endregion +} + + +internal static class ThrowHelper +{ + public static void QueueStopped() => throw new InvalidOperationException("Queue has been stopped."); + public static OperationCanceledException QueueShutdownException { get; } + = new OperationCanceledException("Queue shutdown"); +} diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs b/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs new file mode 100644 index 00000000..6576df25 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs @@ -0,0 +1,78 @@ +namespace Sa.Utils.WorkQueue; + +using System.Threading.Channels; + +public sealed record SaWorkQueueOptions( + ISaWork Processor, + int? QueueCapacity = null, + int? ConcurrencyLimit = null, + int? MaxConcurrency = null, + bool? SingleWriter = false, + Func? HandleItemFaulted = null, + Action? StatusChanged = null, + BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait, + SaReaderScalingStrategy ReaderScalingStrategy = SaReaderScalingStrategy.Lifo, + Func? GetItemDisplayName = null) +{ + /// Creates a new options instance with the specified queue capacity. + /// Must be at least 1. + public SaWorkQueueOptions WithQueueCapacity(int capacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + return this with { QueueCapacity = capacity }; + } + + /// Sets the concurrency limit (number of parallel processors). + /// 0 means unlimited, otherwise must be positive. + public SaWorkQueueOptions WithConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + return this with { ConcurrencyLimit = limit }; + } + + /// Sets the absolute maximum number of reader tasks. + /// If less than 1, defaults to CPU count. + public SaWorkQueueOptions WithMaxConcurrency(int limit) + => this with { MaxConcurrency = limit < 1 ? Environment.ProcessorCount : limit }; + + /// Optimises for a single writer source. + public SaWorkQueueOptions WithSingleWriter(bool sw) + => this with { SingleWriter = sw }; + + /// Registers a callback for status changes of work items. + public SaWorkQueueOptions WithStatusCallback(Action cb) + => this with { StatusChanged = cb }; + + /// Sets a callback that decides the error handling strategy when an item fails. + public SaWorkQueueOptions WithHandleItemFaulted(Func cb) + => this with { HandleItemFaulted = cb }; + + /// Sets the behaviour when the queue is full. + public SaWorkQueueOptions WithFullMode(BoundedChannelFullMode mode) + => this with { FullMode = mode }; + + /// Sets the strategy for assigning items to readers. + public SaWorkQueueOptions WithReaderScalingStrategy(SaReaderScalingStrategy strategy) + => this with { ReaderScalingStrategy = strategy }; + + /// Sets a function to obtain a display name for each work item (e.g., for logging). + public SaWorkQueueOptions WithItemDisplayName(Func toString) + => this with { GetItemDisplayName = toString }; + + + /// Creates options from a delegate that processes a single item. + /// Async delegate that receives the item and a cancellation token. + public static SaWorkQueueOptions Create(Func process) + => new(new DelegatingWork(process)); + + /// Creates options from an processor. + public static SaWorkQueueOptions Create(ISaWork processor) => new(processor); + + // Helper adapter from delegate to ISaWork + private sealed class DelegatingWork(Func process) : ISaWork + { + public Task Execute(TInput input, CancellationToken cancellationToken) + => process(input, cancellationToken); + } +} + diff --git a/src/Sa.Utils.WorkQueue/SaWorkStatus.cs b/src/Sa.Utils.WorkQueue/SaWorkStatus.cs new file mode 100644 index 00000000..796f6bdd --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkStatus.cs @@ -0,0 +1,27 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaWorkStatus +{ + /// + /// in processing + /// + Running, + /// + /// it`s ok + /// + Completed, + /// + /// Unhandled technical error, may or may not be retried depending on policy. + /// + Faulted, + /// + /// Operation was cancelled by the system (timeout, shutdown, external trigger). + /// Automatic retry is permissible. + /// + Cancelled, + /// + /// Operation was intentionally aborted by user or orchestrator. + /// Automatic retry MUST NOT be performed. + /// + Aborted +} diff --git a/src/Sa.Utils.WorkQueue/Setup.cs b/src/Sa.Utils.WorkQueue/Setup.cs new file mode 100644 index 00000000..7f57585b --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Setup.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using System.Diagnostics.CodeAnalysis; + +namespace Sa.Utils.WorkQueue; + +public static class Setup +{ + public static IServiceCollection AddSaWorkQueue( + this IServiceCollection services, + Func> configureOptions, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + + services.Add(new ServiceDescriptor( + typeof(ISaWorkQueue), + sp => + { + var options = configureOptions(sp); + var logger = sp.GetService>>(); + return new SaWorkQueue(options, logger); + }, + lifetime)); + + + return services; + } + + public static IServiceCollection AddSaWorkQueue<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TProcessor, TInput>( + this IServiceCollection services, + Func, SaWorkQueueOptions>? configureOptions = null) + where TProcessor : class, ISaWork + { + services.TryAddSingleton(); + + configureOptions ??= (_, opts) => opts; + + services.TryAddSingleton>( + sp => + { + var processor = sp.GetRequiredService(); + var options = configureOptions(sp, SaWorkQueueOptions.Create(processor)); + var logger = sp.GetService>>(); + return new SaWorkQueue(options, logger); + }); + + return services; + } + + + public static ISaWorkQueue CreateSimple( + this ISaWork processor, + int concurrency = -1) + { + var options = SaWorkQueueOptions.Create(processor) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new SaWorkQueue(options); + } + + public static ISaWorkQueue CreateSimple( + Func process, + int concurrency = -1) + { + var options = SaWorkQueueOptions.Create(process) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new SaWorkQueue(options); + } +} diff --git a/src/Sa.slnx b/src/Sa.slnx index 1866a29e..da46cab0 100644 --- a/src/Sa.slnx +++ b/src/Sa.slnx @@ -30,7 +30,7 @@ - + @@ -62,6 +62,7 @@ + @@ -69,5 +70,8 @@ + + + diff --git a/src/Sa/Classes/IProcessExecutor.cs b/src/Sa/Classes/IProcessExecutor.cs index 97d369d2..8f6a3c6b 100644 --- a/src/Sa/Classes/IProcessExecutor.cs +++ b/src/Sa/Classes/IProcessExecutor.cs @@ -172,7 +172,7 @@ private static async Task Run( process.BeginErrorReadLine(); } - using var timeoutCts = timeout.HasValue + using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero ? new CancellationTokenSource(timeout.Value) : null; diff --git a/src/Sa/Classes/LockRenewer.cs b/src/Sa/Classes/LockRenewer.cs index 05089426..c601e0da 100644 --- a/src/Sa/Classes/LockRenewer.cs +++ b/src/Sa/Classes/LockRenewer.cs @@ -4,7 +4,11 @@ namespace Sa.Classes; internal static class LockRenewer { - public static IDisposable KeepLocked(TimeSpan lockExpiration, Func extendLocked, bool blockImmediately = false, CancellationToken cancellationToken = default) + public static IDisposable KeepLocked( + TimeSpan lockExpiration, + Func extendLocked, + bool blockImmediately = false, + CancellationToken cancellationToken = default) { var timer = new PeriodicTimer(lockExpiration); var task = Task.Run(async () => @@ -55,8 +59,11 @@ public static async Task WaitForConditionAsync( { while (sw.Elapsed < timeout) { - if (!cancellationToken.IsCancellationRequested && await predicate(cancellationToken).ConfigureAwait(false)) + if (!cancellationToken.IsCancellationRequested + && await predicate(cancellationToken).ConfigureAwait(false)) + { return true; + } await Task.Delay(interval, cancellationToken); } diff --git a/src/Sa/Classes/ResetLazy.cs b/src/Sa/Classes/ResetLazy.cs index 8c7c017f..06deb567 100644 --- a/src/Sa/Classes/ResetLazy.cs +++ b/src/Sa/Classes/ResetLazy.cs @@ -1,120 +1,79 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; namespace Sa.Classes; -internal interface IResetLazy -{ - object? Value { get; } - void Reset(); - void Load(); -} - /// /// Provides support for lazy initialization with reset /// /// The type of object that is being lazily initialized. [DebuggerStepThrough] -internal sealed class ResetLazy(Func valueFactory, LazyThreadSafetyMode mode = LazyThreadSafetyMode.ExecutionAndPublication, Action? valueReset = null) : IResetLazy +internal sealed class ResetLazy( + Func valueFactory, + LazyThreadSafetyMode mode = LazyThreadSafetyMode.ExecutionAndPublication, + Action? valueReset = null) { - record Box(T Value); + private sealed record Box(T Value); - private readonly Func _valueFactory = valueFactory ?? throw new ArgumentNullException(nameof(valueFactory)); + private readonly Func _valueFactory = valueFactory + ?? throw new ArgumentNullException(nameof(valueFactory)); private readonly Lock _syncLock = new(); - private Box? _box; + private volatile Box? _box; public T Value { - [DebuggerStepThrough] - get - { - Box? b1 = _box; - if (b1 != null) - return b1.Value; - - if (mode == LazyThreadSafetyMode.ExecutionAndPublication) - { - return LockExecutionAndPublication(); - } - else if (mode == LazyThreadSafetyMode.PublicationOnly) - { - return LockPublicationOnly(); - } - else - { - return CreateAndStoreValuw(); - } - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (_box ?? Initialize()).Value; } - private T CreateAndStoreValuw() - { - Box? b = new(CreateValue()); - _box = b; - return b.Value; - } + public bool IsValueCreated => _box != null; - private T LockPublicationOnly() + private Box Initialize() { - T newValue = CreateValue(); - - lock (_syncLock) + return mode switch { - Box? b2 = _box; - if (b2 != null) - return b2.Value; + LazyThreadSafetyMode.None => CreateAndStore(), + LazyThreadSafetyMode.PublicationOnly => CreatePublicationOnly(), + LazyThreadSafetyMode.ExecutionAndPublication => CreateExecutionAndPublication(), + _ => throw new InvalidOperationException($"Unsupported thread safety mode: {mode}") + }; + } - _box = new Box(newValue); + private Box CreateAndStore() => _box = new(CreateValue()); - return _box.Value; - } + private Box CreatePublicationOnly() + { + var newBox = new Box(_valueFactory()); + var existing = Interlocked.CompareExchange(ref _box, newBox, null); + return (existing ?? newBox); } - private T LockExecutionAndPublication() + private Box CreateExecutionAndPublication() { lock (_syncLock) { - Box? b2 = _box; - if (b2 != null) - return b2.Value; - - _box = new Box(CreateValue()); - - return _box.Value; + return _box ?? CreateAndStore(); } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private T CreateValue() => _valueFactory(); public void Load() => _ = Value; - public bool IsValueCreated => _box != null; - object? IResetLazy.Value => Value; public void Reset() { - if (mode != LazyThreadSafetyMode.None) - { - lock (_syncLock) - { - ResetBox(); - } - } - else - { - ResetBox(); - } - } + Box? oldBox = Interlocked.Exchange(ref _box, null); - private void ResetBox() - { - if (IsValueCreated) + if (oldBox != null && valueReset != null) { - valueReset?.Invoke(_box!.Value); - _box = null; + valueReset(oldBox.Value); } } } diff --git a/src/Sa/Classes/Retry.cs b/src/Sa/Classes/Retry.cs index 657e448e..191efddd 100644 --- a/src/Sa/Classes/Retry.cs +++ b/src/Sa/Classes/Retry.cs @@ -243,7 +243,8 @@ public static IEnumerable GenerateConstant(TimeSpan delay, int retryCo } [DebuggerStepThrough] - public static IEnumerable GenerateLinear(TimeSpan initialDelay, int retryCount, double factor = 1.0, bool fastFirst = true) + public static IEnumerable GenerateLinear( + TimeSpan initialDelay, int retryCount, double factor = 1.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); if (factor < 0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 0"); @@ -252,7 +253,8 @@ public static IEnumerable GenerateLinear(TimeSpan initialDelay, int re } [DebuggerStepThrough] - public static IEnumerable GenerateExponential(TimeSpan initialDelay, int retryCount, double factor = 2.0, bool fastFirst = true) + public static IEnumerable GenerateExponential( + TimeSpan initialDelay, int retryCount, double factor = 2.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); if (factor < 1.0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 1.0"); @@ -261,7 +263,8 @@ public static IEnumerable GenerateExponential(TimeSpan initialDelay, i } [DebuggerStepThrough] - public static IEnumerable GenerateJitter(TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst = true) + public static IEnumerable GenerateJitter( + TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst = true) { ValidateParameters(medianFirstRetryDelay, retryCount, nameof(medianFirstRetryDelay)); return retryCount == 0 ? Empty() : Generator.GenJitter(medianFirstRetryDelay, retryCount, fastFirst); @@ -284,7 +287,8 @@ public static IEnumerable GenConstant(TimeSpan delay, int retryCount, } } - public static IEnumerable GenLinear(TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) + public static IEnumerable GenLinear( + TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) { @@ -300,7 +304,8 @@ public static IEnumerable GenLinear(TimeSpan initialDelay, int retryCo } } - public static IEnumerable GenExponential(TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) + public static IEnumerable GenExponential( + TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) { @@ -335,7 +340,8 @@ public static IEnumerable GenJitter(TimeSpan medianFirstRetryDelay, in double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(pFactor * t)); double formulaIntrinsicValue = next - prev; - yield return TimeSpan.FromTicks((long)Math.Min(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); + yield return TimeSpan.FromTicks( + (long)Math.Min(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); prev = next; } } diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs deleted file mode 100644 index eb73be00..00000000 --- a/src/Sa/Classes/WorkQueue.cs +++ /dev/null @@ -1,285 +0,0 @@ -using System.Threading.Channels; - -namespace Sa.Classes; - - -internal interface IWork -{ - Task Execute(TModel model, CancellationToken cancellationToken); -} - -internal interface IWorkObserver -{ - Task HandleChanges(TModel model, WorkInfo work, CancellationToken cancellationToken); -} - -internal record struct WorkInfo( - long Id, - WorkStatus Status, - DateTimeOffset EnqueuedTime, - DateTimeOffset? StartedTime = null, - DateTimeOffset? EndedTime = null, - Exception? LastError = null -) -{ - public readonly bool IsEmpty => Id == 0; -} - -internal enum WorkStatus -{ - Queued, - Running, - Completed, - Faulted, - Cancelled -} - -internal interface IWorkQueue : IDisposable, IAsyncDisposable -{ - bool IsEnabled { get; } - int ActiveTasks { get; } - bool IsIdle(); - int ConcurrencyLimit { get; set; } - - ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default); - - Task WaitForIdleAsync(CancellationToken cancellationToken = default); - Task ShutdownAsync(); - - event EventHandler? StatusChanged; -} - -internal sealed class WorkQueue : IWorkQueue -{ - private readonly Channel _queue; - - private readonly List _activeItems = []; - private readonly Lock _rootSync = new(); - - private int _maxConcurrency = Environment.ProcessorCount; - private long _taskIdCounter = 0; - private bool _isEnabled = true; - private bool _disposed = false; - - private readonly Task _processingTask; - private readonly TimeProvider _timeProvider; - - public readonly IWork _processor; - public readonly IWorkObserver? _watcher; - - private readonly CancellationTokenSource _shutdownCts = new(); - - public event EventHandler? StatusChanged; - - public WorkQueue( - IWork processor, - IWorkObserver? observer = null, - TimeProvider? timeProvider = null) - { - _processor = processor; - _watcher = observer ?? processor as IWorkObserver; - _timeProvider = timeProvider ?? TimeProvider.System; - - _queue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - AllowSynchronousContinuations = true, - SingleReader = false, - SingleWriter = false, - }); - - _processingTask = LoopAsync(); - } - - public bool IsEnabled => _isEnabled; - public int ActiveTasks => _activeItems.Count; - public int QueuedTasks => _queue.Reader.Count; - - public bool IsIdle() - { - lock (_rootSync) - { - return _activeItems.Count == 0 && _queue.Reader.Count == 0; - } - } - - public int ConcurrencyLimit - { - get => _maxConcurrency; - set - { - ObjectDisposedException.ThrowIf(_disposed, this); - if (value < 1) throw new ArgumentOutOfRangeException(nameof(value), "Must be at least 1."); - _maxConcurrency = value; - } - } - - public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default) - { - ObjectDisposedException.ThrowIf(_disposed, this); - if (!_isEnabled) throw new InvalidOperationException("Task scheduler has been stopped."); - - var shapshot = new WorkItemSnapshot(model, cancellationToken) - { - Id = Interlocked.Increment(ref _taskIdCounter), - Status = WorkStatus.Queued, - EnqueuedTime = _timeProvider.GetUtcNow(), - }; - - await OnStatusChanged(shapshot); - - await _queue.Writer.WriteAsync(shapshot, cancellationToken); - } - - public async Task ShutdownAsync() - { - if (!_isEnabled || _disposed) return; - - _isEnabled = false; - await _shutdownCts.CancelAsync(); - await _processingTask; - - } - - public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) - { - ObjectDisposedException.ThrowIf(_disposed, this); - while (!IsIdle()) - { - await Task.Delay(100, cancellationToken); - } - } - - private async Task LoopAsync() - { - var ct = _shutdownCts.Token; - try - { - while (!ct.IsCancellationRequested && await _queue.Reader.WaitToReadAsync(ct)) - { - while (_isEnabled - && ActiveTasks < _maxConcurrency - && _queue.Reader.TryRead(out var snapshot)) - { - _ = Execute(snapshot); // Не ждём, запускаем параллельно - } - } - } - catch (OperationCanceledException) - { - /* ignore */ - } - } - - private async Task Execute(WorkItemSnapshot snapshot) - { - ActivateSnapshot(snapshot); - - using var cts = CancellationTokenSource.CreateLinkedTokenSource(_shutdownCts.Token, snapshot.CancellationToken); - var ct = cts.Token; - try - { - ct.ThrowIfCancellationRequested(); - - snapshot.StartedTime = _timeProvider.GetUtcNow(); - snapshot.Status = WorkStatus.Running; - await OnStatusChanged(snapshot); - - await _processor.Execute(snapshot.Model, ct); - snapshot.Status = WorkStatus.Completed; - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - snapshot.Status = WorkStatus.Cancelled; - } - catch (Exception ex) - { - snapshot.Status = WorkStatus.Faulted; - snapshot.LastError = ex; - } - finally - { - snapshot.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChanged(snapshot); - - DeactiveSnapshot(snapshot); - } - } - - private void ActivateSnapshot(WorkItemSnapshot snapshot) - { - lock (_rootSync) - { - _activeItems.Add(snapshot); - } - } - - private void DeactiveSnapshot(WorkItemSnapshot snapshot) - { - lock (_rootSync) - { - _activeItems.Remove(snapshot); - } - } - - private async Task OnStatusChanged(WorkItemSnapshot snapshot) - { - var info = snapshot.ToInfo(); - - if (_watcher != null) - { - try - { - await _watcher.HandleChanges(snapshot.Model, info, _shutdownCts.Token); - } - catch { /* ignore */ } - } - - try - { - StatusChanged?.Invoke(this, info); - } - catch { /* ignore */ } - } - - public void Dispose() - { - if (_disposed) return; - - _disposed = true; - _isEnabled = false; - _queue.Writer.Complete(); - - _shutdownCts.Cancel(); - _processingTask.Wait(5000); - - _shutdownCts.Dispose(); - } - - public async ValueTask DisposeAsync() - { - if (_disposed) return; - - _disposed = true; - _isEnabled = false; - _queue.Writer.Complete(); - - await _shutdownCts.CancelAsync(); - await _processingTask; - - _shutdownCts.Dispose(); - } - - private sealed class WorkItemSnapshot(TModel item, CancellationToken token) - { - public long Id { get; init; } - public TModel Model => item; - public CancellationToken CancellationToken => token; - public WorkStatus Status { get; set; } - public DateTimeOffset EnqueuedTime { get; init; } - public DateTimeOffset? StartedTime { get; set; } = default; - public DateTimeOffset? EndedTime { get; set; } = default; - public Exception? LastError { get; set; } = null; - - public WorkInfo ToInfo() => new(Id, Status, EnqueuedTime, StartedTime, EndedTime, LastError); - } -} diff --git a/src/Samples/Configuration.Web/Program.cs b/src/Samples/Configuration.Web/Program.cs index 7d17ca15..fd2c4f5d 100644 --- a/src/Samples/Configuration.Web/Program.cs +++ b/src/Samples/Configuration.Web/Program.cs @@ -12,7 +12,7 @@ string connectionString = builder.Configuration[PG_KEY] ?? throw new ArgumentException(PG_KEY); -using var ds = new PgDataSource(new(connectionString)); +using var ds = IPgDataSource.Create(connectionString); ds.ExecuteScalar(""" CREATE TABLE IF NOT EXISTS settings ( diff --git a/src/Samples/HybridFileStorage.Console/Program.cs b/src/Samples/HybridFileStorage.Console/Program.cs index f94fd464..25ecd680 100644 --- a/src/Samples/HybridFileStorage.Console/Program.cs +++ b/src/Samples/HybridFileStorage.Console/Program.cs @@ -46,8 +46,8 @@ public async Task Run(CancellationToken cancellationToken = default) using var stream = expected.ToStream(); var result = await storage.UploadAsync( - new UploadFileInput { FileName = "file.txt" }, string.Empty, + new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); @@ -55,7 +55,6 @@ public async Task Run(CancellationToken cancellationToken = default) var isDowload = await storage.DownloadAsync( result.FileId, - string.Empty, async (fs, t) => actual = await fs.ToStrAsync(t), cancellationToken); diff --git a/src/Samples/Schedule.Console/Program.cs b/src/Samples/Schedule.Console/Program.cs index 0aed5e0a..3285c3d1 100644 --- a/src/Samples/Schedule.Console/Program.cs +++ b/src/Samples/Schedule.Console/Program.cs @@ -46,16 +46,17 @@ } else { - controller.Start(cts.Token); + await controller.Start(cts.Token); } _ = Task.Run(async () => { await Task.Delay(5000); await controller.Stop(); - Console.WriteLine($"*** stopped & restart after 2 sec"); + Console.WriteLine($"*** stopped & start after 2 sec"); + await Task.Delay(2000); - controller.Restart(); + await controller.Start(cts.Token); }); _ = Task.Run(async () => @@ -93,7 +94,11 @@ public async Task Execute(IJobContext context, CancellationToken cancellationTok public class SomeInterceptor : IJobInterceptor { - public async Task OnHandle(IJobContext context, Func next, object? key, CancellationToken cancellationToken) + public async Task OnHandle( + IJobContext context, + Func next, + object? key, + CancellationToken cancellationToken) { System.Console.WriteLine($""); await next(); diff --git a/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs b/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs index 42267c45..281c7b20 100644 --- a/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs +++ b/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs @@ -43,7 +43,8 @@ protected SaFixture() public async virtual ValueTask DisposeAsync() { - if (_serviceProvider.IsValueCreated) await _serviceProvider.Value.DisposeAsync(); + if (_serviceProvider.IsValueCreated) + await _serviceProvider.Value.DisposeAsync(); } } diff --git a/src/Tests/Host.Test.Properties.xml b/src/Tests/Host.Test.Properties.xml index 0a13bcdb..361d328c 100644 --- a/src/Tests/Host.Test.Properties.xml +++ b/src/Tests/Host.Test.Properties.xml @@ -7,6 +7,10 @@ false + + 1591;1701;1702;IDE1006;S3776 + + diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs index badd43e5..f53a0b15 100644 --- a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs @@ -43,6 +43,10 @@ public async Task Crud() bool canProcessed = Storage.CanProcess(result.FileId); Assert.True(canProcessed); + var meta = await Storage.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); + + var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); @@ -77,7 +81,12 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy [Fact] public async Task CrudEx() { - var metadata = new UploadFileInput { FileName = "/api/files/download/file/var/www/uploads/image.bin", TenantId = 1 }; + var metadata = new UploadFileInput + { + FileName = "/api/files/download/file/var/www/uploads/image.bin", + TenantId = 1 + }; + using MemoryStream fileContent = FixtureHelper.GetByteStream(); var result = await Storage.UploadAsync(metadata, fileContent, fixture.CancellationToken); @@ -88,6 +97,9 @@ public async Task CrudEx() bool canProcessed = Storage.CanProcess(result.FileId); Assert.True(canProcessed); + var meta = await Storage.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); + var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs index 7f500762..c7663cc8 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs @@ -8,14 +8,16 @@ public class FileIdParserTests public void ParseFromFileId_ValidFileId_ReturnsTenantIdAndTimestamp() { string fileId = "pg://files/123/1773210911/foo/some.txt"; - var result = FileIdParser.TryParseFileIdWithFilename( + var result = FileIdParser.TryParse( fileId, + out string scopeName, out int tenantId, out long timestamp, out string filename); Assert.True(result); Assert.Equal(123, tenantId); + Assert.Equal("files", scopeName); Assert.Equal(1773210911, timestamp); Assert.Equal("foo/some.txt", filename); } @@ -24,10 +26,11 @@ public void ParseFromFileId_ValidFileId_ReturnsTenantIdAndTimestamp() public void ParseFromFileId_InvalidFileId() { string fileId = "invalid_file_id"; - var result = FileIdParser.TryParseFileIdWithFilename( + var result = FileIdParser.TryParse( fileId, out _, out _, + out _, out _); Assert.False(result); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index 5396ffa8..0164ff4c 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -21,23 +21,29 @@ public async Task UploadFileAsync() Console.WriteLine(fixture.ConnectionString); // Arrange - var metadata = new UploadFileInput { FileName = "test.txt", TenantId = 1 }; + var input = new UploadFileInput { FileName = "test.txt", TenantId = 1 }; using MemoryStream fileContent = await CreateStream(fixture.CancellationToken); // Act - var result = await Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); + var result = await Sub.UploadAsync(input, fileContent, fixture.CancellationToken); // Assert Assert.NotNull(result); Assert.NotEmpty(result.FileId); - Assert.StartsWith("pg://files/1/", result.FileId); + Assert.StartsWith("pg://share/1/", result.FileId); object? v = await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM public.files WHERE id = @id", cmd => cmd.Parameters.Add(new("id", result.FileId)), fixture.CancellationToken); var count = (long)v!; Assert.Equal(1, count); + + + Assert.True(Sub.CanProcess(result.FileId)); + + var meta = Sub.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); } @@ -88,9 +94,10 @@ public async Task DownloadFileAsync() [Fact] public async Task GetMetadataAsync() { - var metadata = await Sub.GetMetadataAsync("pg://files/7/1773210911/some/data.bin", CancellationToken.None); + var metadata = await Sub.GetMetadataAsync("pg://share/7/1773210911/some/data.bin", CancellationToken.None); Assert.NotNull(metadata); + Assert.Equal("share", metadata.Basket); Assert.Equal(7, metadata.TenantId); Assert.Equal("some/data.bin", metadata.FileName); } @@ -108,20 +115,37 @@ public async Task Upload2FileAsync() var t1 = Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); var t2 = Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); - await Task.WhenAll([t1, t2]); + await Task.WhenAll(t1, t2); var fileId1 = (await t1).FileId; var fileId2 = (await t2).FileId; Assert.Equal(fileId1, fileId2); - long count = await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM public.files WHERE id = @id", + long count = await fixture.DataSource.ExecuteScalar( + "SELECT COUNT(*) FROM public.files WHERE id = @id", cmd => cmd.Parameters.Add(new("id", fileId1)), fixture.CancellationToken); Assert.Equal(1, count); } + [Theory] + [InlineData("./data/12345.wav")] + public async Task UploadWavFileAsync(string filePath) + { + Console.WriteLine(fixture.ConnectionString); + + // Arrange + var metadata = new UploadFileInput { FileName = filePath, TenantId = 1 }; + using var fileContent = File.OpenRead(filePath); + + var r = await Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); + + Assert.NotEmpty(r.FileId); + } + + private static async Task CreateStream(CancellationToken cancellationToken) { var fileContent = new MemoryStream(); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj b/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj index 265c4575..3b4c9d52 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj @@ -1,4 +1,4 @@ - + @@ -11,4 +11,10 @@ + + + PreserveNewest + + + diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav b/src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav new file mode 100644 index 00000000..5045628e Binary files /dev/null and b/src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav differ diff --git a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs index f33a711f..3af6bda2 100644 --- a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs @@ -66,7 +66,7 @@ public async Task Crud() var input = new UploadFileInput { FileName = "test.bin", TenantId = 2 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, string.Empty, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(string.Empty, input, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.NotEmpty(result.FileId); @@ -77,7 +77,7 @@ public async Task Crud() var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, string.Empty, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); Assert.True(isDeleted); @@ -92,7 +92,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var input = new UploadFileInput { FileName = "some.bin", TenantId = 1 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, string.Empty, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(string.Empty, input, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.Equal(InMemoryFileStorage.DefaultStorageType, result.StorageType); @@ -100,7 +100,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, string.Empty, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); Assert.True(isDeleted); } @@ -115,7 +115,7 @@ public async Task WhenStorageIsEmptyThrowsInvalidOperationException() using MemoryStream fileContent = FixtureHelper.GetByteStream(); await Assert.ThrowsAsync(() => sp.GetRequiredService().UploadAsync( - new UploadFileInput { FileName = "", TenantId = 1 }, string.Empty, fileContent, fixture.CancellationToken)); + string.Empty, new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); } @@ -133,8 +133,8 @@ public async Task WhenStorageIsReadOnlyThrowsInvalidOperationException() await Assert.ThrowsAsync(() => sp.GetRequiredService().UploadAsync( - new UploadFileInput { FileName = "", TenantId = 1 }, string.Empty, + new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); } @@ -147,7 +147,6 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy using MemoryStream memoryStream = new(); var isDownloaded = await Storage.DownloadAsync(fileName - , scopeName: string.Empty , (stream, ct) => stream.CopyToAsync(memoryStream, ct) , fixture.CancellationToken); diff --git a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs index 734341ac..6cb97691 100644 --- a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs +++ b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs @@ -1,5 +1,5 @@ -using Sa.Classes; -using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpegTests; @@ -48,6 +48,7 @@ await Processor.ConvertToPcmS16Le( [Theory] [InlineData("./data/input.mp3")] [InlineData("./data/gsm.wav")] + [InlineData("./data/12345.wav")] public async Task ConvertToPcm16Wav_CallsFFmpegAsStream(string testFilePath) { var ext = Path.GetExtension(testFilePath).TrimStart('.'); diff --git a/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj b/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj index 1a908a16..cfc1d536 100644 --- a/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj +++ b/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj @@ -11,6 +11,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Tests/Sa.Media.FFmpegTests/data/12345.wav b/src/Tests/Sa.Media.FFmpegTests/data/12345.wav new file mode 100644 index 00000000..5045628e Binary files /dev/null and b/src/Tests/Sa.Media.FFmpegTests/data/12345.wav differ diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs index e6159588..db2dd4da 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs @@ -52,10 +52,18 @@ public async Task Deliver_Process_MustBe_Work() Console.Write(fixture.ConnectionString); - var cnt = await fixture.Publisher.PublishSingle(new TestMessage { PayloadId = "11", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); + var cnt = await fixture.Publisher.PublishSingle( + new TestMessage { PayloadId = "11", Content = "Message 1", TenantId = 1 }, + tenantId: 1, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(cnt > 0); - cnt = await fixture.Publisher.PublishSingle(new TestMessage { PayloadId = "12", Content = "Message 2", TenantId = 2 }, 2, TestContext.Current.CancellationToken); + cnt = await fixture.Publisher.PublishSingle( + new TestMessage { PayloadId = "12", Content = "Message 2", TenantId = 2 }, + tenantId: 2, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(cnt > 0); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs index de33a9aa..580a1399 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs @@ -130,7 +130,7 @@ public async Task ParallelMessaging_MustBeProcessed() // start cron schedules IScheduler scheduler = ServiceProvider.GetRequiredService(); - int i = scheduler.Start(CancellationToken.None); + int i = await scheduler.Start(CancellationToken.None); Assert.True(i > 0); // start delivery message diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs index 9678be1b..166d06b9 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs @@ -142,7 +142,7 @@ public async Task Outbox_TenantParallelism_ShouldProcessTenantsConcurrently() ParallelTestConsumer.Clear(); var scheduler = ServiceProvider.GetRequiredService(); - int startedSchedules = scheduler.Start(CancellationToken.None); + int startedSchedules = await scheduler.Start(CancellationToken.None); Assert.True(startedSchedules >= 1); var publisher = ServiceProvider.GetRequiredService(); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs index 6639554f..b854e8d0 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs @@ -79,7 +79,7 @@ public async Task OutBoxTest() // start cron schedules var scheduler = ServiceProvider.GetRequiredService(); - int i = scheduler.Start(CancellationToken.None); + int i = await scheduler.Start(CancellationToken.None); Assert.True(i > 0); // start delivery message diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs index 17f441cb..d3d2d8ea 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs @@ -99,7 +99,7 @@ public async Task OutBox_TwoGroups_ShouldProcessSeparately() SomeMessageConsumerGr2.Counter = 0; var scheduler = ServiceProvider.GetRequiredService(); - int startedSchedules = scheduler.Start(CancellationToken.None); + int startedSchedules = await scheduler.Start(CancellationToken.None); Assert.True(startedSchedules >= 2, "Ожидалось как минимум 2 расписания (по одному на группу)"); var publisher = ServiceProvider.GetRequiredService(); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs index d2c2fb1b..7c8087dd 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs @@ -19,14 +19,18 @@ public async Task Publish_MultipleMessages_ReturnsExpectedResult() { Console.Write(fixture.ConnectionString); - ulong result = await Sub.PublishSingle(new TestMessage { PayloadId = "1", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); + ulong result = await Sub.PublishSingle( + new TestMessage { PayloadId = "1", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); - result += await Sub.PublishSingle(new TestMessage { PayloadId = "2", Content = "Message 3", TenantId = 2 }, 2, TestContext.Current.CancellationToken); + result += await Sub.PublishSingle( + new TestMessage { PayloadId = "2", Content = "Message 3", TenantId = 2 }, 2, TestContext.Current.CancellationToken); // Assert Assert.Equal(2, (int)result); - var sql = $"select count(*) from {PgOutboxTableSettings.Defaults.DatabaseTableName}{PgOutboxTableSettings.MessageTable.Suffix}"; + var sql = @$"select count(*) from + {PgOutboxTableSettings.Defaults.DatabaseTableName}{PgOutboxTableSettings.MessageTable.Suffix}"; + int count = await fixture.DataSource.ExecuteReaderFirst(sql, TestContext.Current.CancellationToken); Assert.Equal(2, count); } diff --git a/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs b/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs index 86930118..e56d2fb8 100644 --- a/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs +++ b/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs @@ -4,8 +4,6 @@ namespace Sa.Partitional.PostgreSqlTests.Configuration; - - public class ConfigurationPartTests(ConfigurationPartTests.Fixture fixture) : IClassFixture { diff --git a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs index 7e4651c6..79bddf0e 100644 --- a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs +++ b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs @@ -6,10 +6,11 @@ namespace Sa.Partitional.PostgreSqlTests; -public class PartitionAsJobTests(PartitionAsJobTests.Fixture fixture) : IClassFixture +public class PartitionAsJobTests(PartitionAsJobTests.Fixture fixture) + : IClassFixture { - public class Fixture : PgDataSourceFixture + public sealed class Fixture : PgDataSourceFixture { public Fixture() { @@ -53,12 +54,17 @@ public async Task MigrateAsJobTest() { Console.WriteLine(fixture.ConnectionString); - int i = fixture.ServiceProvider.GetRequiredService().Start(CancellationToken.None); + int i = await fixture.ServiceProvider + .GetRequiredService().Start(CancellationToken.None); Assert.True(i > 0); await Task.Delay(800, TestContext.Current.CancellationToken); - var list = await Sub.GetPartsFromDate("customer", StartOfDay(DateTimeOffset.Now), TestContext.Current.CancellationToken); + var list = await Sub.GetPartsFromDate( + "customer", + StartOfDay(DateTimeOffset.Now), + TestContext.Current.CancellationToken); + Assert.NotEmpty(list); } } diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs new file mode 100644 index 00000000..c512cc35 --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -0,0 +1,163 @@ +using Sa.Schedule; +using Sa.Schedule.Engine; +using Sa.Schedule.Settings; +using System.Collections.Concurrent; + +namespace Sa.ScheduleTests; + +public class JobSchedulerTests +{ + + [Fact] + public async Task Start_ConcurrentCalls_OnlyOneSucceeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + var results = new ConcurrentBag(); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => results.Add( + await scheduler.Start(TestContext.Current.CancellationToken)) + )); + + await Task.WhenAll(tasks); + + Assert.Single(results, r => r); + } + + + [Fact] + public async Task Stop_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => + await scheduler.Stop() + )); + + await Task.WhenAll(tasks); + + + Assert.False(scheduler.IsStarted); + Assert.Equal(0, scheduler.ActiveTasks); + } + + + [Fact] + public async Task Dispose_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => await scheduler.DisposeAsync())); + + await Task.WhenAll(tasks); + + Assert.False(scheduler.IsStarted); + Assert.Equal(0, scheduler.ActiveTasks); + } + + + [Fact] + public async Task ConcurrencyLimit_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + settings.Properties + .WithMaxConcurrencyLimit(30) + .WithConcurrencyLimit(1); + + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(() => scheduler.ConcurrencyLimit = Random.Shared.Next(2, 45))); + + await Task.WhenAll(tasks); + + Assert.True(scheduler.IsStarted); + Assert.InRange(scheduler.ConcurrencyLimit, 2, 30); + + await scheduler.DisposeAsync(); + } + + + + class TestJob : IJob + { + public async Task Execute(IJobContext context, CancellationToken cancellationToken) + => await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken); + } + + + class TestJobRunner : IJobRunner + { + public Task Run(IJobController controller, CancellationToken cancellationToken) => Task.CompletedTask; + } + + + class TestJobController(int index) : IJobController + { + public bool IsPaused => false; + + public int Index => index; + + public ValueTask CanExecute(CancellationToken cancellationToken) + { + return ValueTask.FromResult(CanJobExecuteResult.Ok); + } + + public Task Execute(CancellationToken cancellationToken) => Task.CompletedTask; + + public void ExecutionCompleted() + { + // + } + + public void ExecutionFailed(Exception exception) + { + // + } + + public void Pause() + { + + } + + public void Resume() + { + + } + + public void Start() + { + // + } + + public void Shutdown() + { + // + } + + public ValueTask WaitIfPaused(CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public ValueTask WaitToRun(CancellationToken cancellationToken) => ValueTask.CompletedTask; + } +} diff --git a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj index 47dc9d6f..7a9e09e3 100644 --- a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj +++ b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj @@ -1,15 +1,14 @@ - + - - true - - - - - - + + true + + + + + diff --git a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs new file mode 100644 index 00000000..257a4ef2 --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs @@ -0,0 +1,107 @@ +using Sa.Fixture; +using Sa.Schedule; + +namespace Sa.ScheduleTests; + + + +public sealed class ScheduleConcurrencyTests(ScheduleConcurrencyTests.Fixture fixture) + : IClassFixture +{ + public class Fixture : SaFixture + { + static class Counter + { + private static readonly Lock @lock = new(); + + private readonly static HashSet jobs = []; + public static int Total + { + get + { + lock (@lock) + { + return jobs.Count; + } + } + } + public static void Inc(SomeJob job) + { + lock (@lock) + { + jobs.Add(job); + } + } + + public static void Clear() + { + lock (@lock) { jobs.Clear(); } + } + } + + class SomeJob : IJob + { + public async Task Execute(IJobContext context, CancellationToken cancellationToken) + { + Counter.Inc(this); + await Task.Delay(10, cancellationToken); + } + } + + public Fixture() + { + Services.AddSaSchedule(b => + { + b + .AddJob(JobId) + .EveryTime(TimeSpan.FromMilliseconds(10)) + .StartImmediate() + .WithConcurrencyLimit(1) + .WithMaxConcurrency(10) + ; + }); + } + + public static int Count => Counter.Total; + + public readonly static Guid JobId = Guid.NewGuid(); + public static void Reset() => Counter.Clear(); + } + + private IScheduler Sub => fixture.Sub; + + + + [Fact] + public async Task Check_ExecuteCounterJob() + { + Fixture.Reset(); + + int i = await Sub.Start(CancellationToken.None); + + Assert.NotEqual(0, i); + + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Equal(1, Fixture.Count); + Fixture.Reset(); + + var j = Sub.GetSchedule(Fixture.JobId); + Assert.NotNull(j); + + j.ConcurrencyLimit = 10; + + await Task.Delay(250, TestContext.Current.CancellationToken); + + Assert.Equal(10, Fixture.Count); + + j.ConcurrencyLimit = 4; + + await Task.Delay(200, TestContext.Current.CancellationToken); + Fixture.Reset(); + + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Equal(4, Fixture.Count); + } +} diff --git a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs index 2fe992cf..ed6e5253 100644 --- a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs @@ -4,7 +4,8 @@ namespace Sa.ScheduleTests; -public class SchedulePostSetupTests(SchedulePostSetupTests.Fixture fixture) : IClassFixture +public class SchedulePostSetupTests(SchedulePostSetupTests.Fixture fixture) + : IClassFixture { public class Fixture : SaFixture { @@ -28,12 +29,13 @@ public Fixture() { Services.AddSaSchedule(b => { - b.AddJob((sp, builder) => + b.AddJob((_, builder) => { builder .EveryTime(TimeSpan.FromMilliseconds(100)) .RunOnce() .StartImmediate() + //.WithMaxConcurrencyLimit(1) ; }); @@ -41,12 +43,13 @@ public Fixture() Services.AddSaSchedule(b => { - b.AddJob((sp, builder) => + b.AddJob((_, builder) => { builder .EveryTime(TimeSpan.FromMilliseconds(100)) .RunOnce() .StartImmediate() + //.WithMaxConcurrencyLimit(1) ; }); @@ -62,7 +65,7 @@ public Fixture() [Fact] public async Task Check_Executing_RunOnce_ForMultiJobs() { - int i = Sub.Start(CancellationToken.None); + int i = await Sub.Start(CancellationToken.None); Assert.Equal(2, i); diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs index e1c19114..544d9be1 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs @@ -4,7 +4,8 @@ namespace Sa.ScheduleTests; -public class ScheduleSetupTests(ScheduleSetupTests.Fixture fixture) : IClassFixture +public sealed class ScheduleSetupTests(ScheduleSetupTests.Fixture fixture) + : IClassFixture { public class Fixture : SaFixture { @@ -45,7 +46,7 @@ public Fixture() [Fact] public async Task Check_ExecuteCounterJob() { - int i = Sub.Start(CancellationToken.None); + int i = await Sub.Start(CancellationToken.None); Assert.NotEqual(0, i); diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj b/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj new file mode 100644 index 00000000..7ef1de06 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj @@ -0,0 +1,13 @@ + + + + + + true + + + + + + + diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs new file mode 100644 index 00000000..01091d51 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs @@ -0,0 +1,518 @@ +using System.Collections.Concurrent; + +namespace Sa.Utils.WorkQueue.Tests; + + +public sealed class WorkQueueConcurrencyTests : IAsyncLifetime +{ + static CancellationToken TestToken => TestContext.Current.CancellationToken; + + internal sealed class TrackingWork( + Func? executeFunc = null) : ISaWork + { + private readonly Func? _executeFunc = executeFunc; + + // Thread-safe + private int _maxParallelObserved; + private int _currentParallel; + private long _completedCount; + private long _failedCount; + private long _cancelledCount; + + public int MaxParallelObserved => Volatile.Read(ref _maxParallelObserved); + public long CompletedCount => Interlocked.Read(ref _completedCount); + public long FailedCount => Interlocked.Read(ref _failedCount); + public long CancelledCount => Interlocked.Read(ref _cancelledCount); + public long TotalProcessed => CompletedCount + FailedCount + CancelledCount; + + public async Task Execute(BlockingTaskModel model, CancellationToken cancellationToken) + { + int current = Interlocked.Increment(ref _currentParallel); + try + { + int max = Volatile.Read(ref _maxParallelObserved); + if (current > max) + { + Interlocked.CompareExchange(ref _maxParallelObserved, current, max); + } + + await model.AllowCompletion.Task; + + + if (_executeFunc != null) + await _executeFunc(model, cancellationToken); + else + await Task.Delay(50, cancellationToken); // Default delay + } + finally + { + Interlocked.Decrement(ref _currentParallel); + } + } + + public void HandleChanges( + BlockingTaskModel _, + SaWorkStatus status, + Exception? __) + { + + switch (status) + { + case SaWorkStatus.Completed: + Interlocked.Increment(ref _completedCount); + break; + case SaWorkStatus.Faulted: + Interlocked.Increment(ref _failedCount); + break; + case SaWorkStatus.Cancelled: + Interlocked.Increment(ref _cancelledCount); + break; + } + } + + public void Reset() + { + Volatile.Write(ref _maxParallelObserved, 0); + Volatile.Write(ref _currentParallel, 0); + Interlocked.Exchange(ref _completedCount, 0); + Interlocked.Exchange(ref _failedCount, 0); + Interlocked.Exchange(ref _cancelledCount, 0); + } + } + + internal sealed record BlockingTaskModel( + TimeSpan? Delay = null, + bool ShouldFail = false, + string? Id = null + ) + { + public TaskCompletionSource AllowCompletion { get; } + = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public BlockingTaskModel Unlock() + { + AllowCompletion.SetResult(); + return this; + } + }; + + + private readonly List _disposables = []; + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public async ValueTask DisposeAsync() + { + foreach (var d in _disposables) + { + if (d is IAsyncDisposable ad) + await ad.DisposeAsync(); + else + d.Dispose(); + } + _disposables.Clear(); + } + + private SaWorkQueue CreateQueue( + TrackingWork work, + int concurrencyLimit, + int? queueCapacity = null, + int? maxConcurrency = null) + { + var options = new SaWorkQueueOptions( + Processor: work, + ConcurrencyLimit: concurrencyLimit, + MaxConcurrency: maxConcurrency ?? concurrencyLimit * 2, + QueueCapacity: queueCapacity ?? 100, + StatusChanged: work.HandleChanges + ); + + var queue = new SaWorkQueue(options); + _disposables.Add(queue); + return queue; + } + + + + [Fact] + public async Task ConcurrencyLimit_Respected_AtStartup() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3); + + var blockers = new List(); + + // Act: enqueue 10 tasks that block until we allow them + for (int i = 0; i < 10; i++) + { + var model = new BlockingTaskModel(); + blockers.Add(model.AllowCompletion); + await queue.Enqueue(model, TestToken); + } + + await Task.Delay(200, TestToken); + + + Assert.True(work.MaxParallelObserved <= 3, + $"Expected max parallel <= 3, but observed {work.MaxParallelObserved}"); + + Assert.True(work.MaxParallelObserved >= 1, "At least one task should have started"); + + + foreach (var tcs in blockers) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestToken); + + Assert.Equal(10, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_Zero_StopsProcessing() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 0); + + // Enqueue tasks + for (int i = 0; i < 5; i++) + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromMilliseconds(10)); + await queue.Enqueue(model, TestToken); + model.Unlock(); // Сразу разрешаем завершение + } + + Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); + + await Task.Delay(300, TestToken); + + Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); + Assert.Equal(0, work.TotalProcessed); + + // Восстанавливаем лимит и ждём завершения + queue.ConcurrencyLimit = 2; + await queue.WaitForIdleAsync(TestToken); + Assert.Equal(5, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_DynamicIncrease_Works() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 2); + + var phase1Blockers = new List(); + var phase2Blockers = new List(); + + // Act Phase 1: enqueue 4 tasks with limit=2 + for (int i = 0; i < 4; i++) + { + var model = new BlockingTaskModel(Id: $"P1-{i}"); + phase1Blockers.Add(model.AllowCompletion); + await queue.Enqueue(model, TestToken); + } + await Task.Delay(150, TestToken); + + int maxAtPhase1 = work.MaxParallelObserved; + Assert.True(maxAtPhase1 <= 2, $"Phase 1: expected <=2, got {maxAtPhase1}"); + + // Act Phase 2: increase limit to 4 and enqueue more + queue.ConcurrencyLimit = 4; + + for (int i = 0; i < 4; i++) + { + var model2 = new BlockingTaskModel(Id: $"P2-{i}"); + phase2Blockers.Add(model2.AllowCompletion); + await queue.Enqueue(model2, TestToken); + } + await Task.Delay(150, TestToken); + + // Assert: после увеличения лимита должно быть возможно до 4 параллельных + Assert.True(work.MaxParallelObserved <= 4, + $"Expected max parallel <= 4, but observed {work.MaxParallelObserved}"); + Assert.True(work.MaxParallelObserved > maxAtPhase1, + "Max parallel should have increased after raising limit"); + + // Завершаем все задачи + foreach (var tcs in phase1Blockers.Concat(phase2Blockers)) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestToken); + Assert.Equal(8, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_DynamicDecrease_ThrottlesReaders() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 4); + + var blockers = new List(); + var startedEvents = new List(); + + // Act: enqueue 8 tasks + for (int i = 0; i < 8; i++) + { + var model = new BlockingTaskModel(Id: $"T-{i}"); + var startedTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + blockers.Add(model.AllowCompletion); + startedEvents.Add(startedTcs); + + await queue.Enqueue(model, TestToken); + } + + + for (int i = 0; i < 4; i++) + blockers[i].SetResult(); + + await Task.Delay(200, TestToken); + int beforeDecrease = work.MaxParallelObserved; + Assert.True(beforeDecrease <= 4, $"Before decrease: expected <=4, got {beforeDecrease}"); + + Assert.Equal(4, queue.QueueTasks); + + // Act: decrease limit to 2 + queue.ConcurrencyLimit = 2; + await Task.Delay(200, TestToken); + + Assert.True(work.MaxParallelObserved <= 4, + $"After decrease: observed spike to {work.MaxParallelObserved}"); + + // Завершаем остальные + for (int i = 4; i < 8; i++) + blockers[i].SetResult(); + + + await queue.WaitForIdleAsync(TestToken); + + Assert.Equal(8, work.TotalProcessed); + Assert.Equal(0, queue.QueueTasks); + } + + + + [Theory] + [InlineData(1, 10)] + [InlineData(5, 25)] + [InlineData(10, 50)] + public async Task ConcurrencyLimit_NeverExceeded_UnderLoad(int limit, int taskCount) + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: limit, maxConcurrency: limit * 2); + + var blockers = new List(); + + // Act: rapid enqueue + var enqueueTasks = Enumerable.Range(0, taskCount).Select(async i => + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromMilliseconds(20)); + blockers.Add(model.AllowCompletion); + await queue.Enqueue(model); + }); + + await Task.WhenAll(enqueueTasks); + + await Task.Delay(300, TestToken); + + Assert.True(work.MaxParallelObserved <= limit); + + foreach (var tcs in blockers) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestToken); + Assert.Equal(taskCount, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimitRapidChangesDoesNotBreak() + { + int totals = 30; + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3, maxConcurrency: 6, queueCapacity: totals); + + using var cts = new CancellationTokenSource(); + var blockers = new ConcurrentQueue(); + + // Background enqueuer + var enqueueTask = Task.Run(async () => + { + int i = 0; + while (!cts.Token.IsCancellationRequested && i < totals) + { + var model = new BlockingTaskModel(Id: $"T-{i++}"); + blockers.Enqueue(model.AllowCompletion); + await queue.Enqueue(model, cts.Token); + await Task.Delay(20, cts.Token); + } + }, cts.Token); + + // Limiter changer + var changeTask = Task.Run(async () => + { + int[] limits = [2, 5, 1, 4, 3, 6, 2]; + foreach (var limit in limits) + { + if (cts.Token.IsCancellationRequested) break; + queue.ConcurrencyLimit = limit; + await Task.Delay(100, cts.Token); + } + // cts.Cancel(); // Stop enqueuer + }, cts.Token); + Task processTask = CreateProcess(totals, cts, blockers); + + await Task.WhenAll(enqueueTask, changeTask, processTask); + await queue.WaitForIdleAsync(TestToken); + + // Assert: no exceptions, all tasks processed, limit respected at peaks + Assert.True(work.MaxParallelObserved <= 6, + $"Max observed {work.MaxParallelObserved} exceeded max configured limit 6"); + + Assert.Equal(totals, work.TotalProcessed); // All accounted for + } + + private static Task CreateProcess( + int totals, CancellationTokenSource cts, ConcurrentQueue blockers) + { + + // Processor: complete tasks with small delay + return Task.Run(async () => + { + + int i = 0; + + try + { + while (!cts.Token.IsCancellationRequested || !blockers.IsEmpty) + { + if (blockers.TryDequeue(out var tcs)) + { + i++; + await Task.Delay(10); + + tcs.TrySetResult(); + } + else + { + await Task.Delay(100, cts.Token); + if (i >= totals) + { + return; + } + } + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + }, cts.Token); + } + + [Fact] + public async Task ConcurrencyLimit_SetToMaxAllowed_ClampsCorrectly() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 2, maxConcurrency: 5); + + // Act & Assert + queue.ConcurrencyLimit = 10; // Above max + Assert.Equal(5, queue.ConcurrencyLimit); // Should be clamped + + queue.ConcurrencyLimit = -1; // Below min + Assert.Equal(0, queue.ConcurrencyLimit); // Should be clamped to 0 + + queue.ConcurrencyLimit = 3; // Valid + Assert.Equal(3, queue.ConcurrencyLimit); + } + + + [Fact] + public async Task ConcurrencyLimit_UpChanges_DontLoseTasks() + { + // Arrange + var completedIds = new ConcurrentBag(); + var work = new TrackingWork( + executeFunc: async (model, ct) => + { + await Task.Delay(10, ct); + if (!string.IsNullOrEmpty(model.Id)) + completedIds.Add(model.Id); + }); + + var queue = CreateQueue(work, concurrencyLimit: 2); + + + List models = []; + + // Act: enqueue with interleaved limit changes + var tasks = new List(); + for (int i = 0; i < 20; i++) + { + if (i == 5) queue.ConcurrencyLimit = 3; + if (i == 10) queue.ConcurrencyLimit = 4; + if (i == 15) queue.ConcurrencyLimit = 5; + + var model = new BlockingTaskModel(Id: $"T-{i:D3}"); + models.Add(model); + var t = queue.Enqueue(model, TestToken).AsTask(); + + tasks.Add(t); + } + + await Task.WhenAll(tasks); + + foreach (var model in models) model.Unlock(); + + + await queue.WaitForIdleAsync(TestToken); + + // Assert: all tasks completed, no duplicates, no losses + Assert.Equal(20, completedIds.Count); + Assert.Equal(20, completedIds.Distinct().Count()); + Assert.All(Enumerable.Range(0, 20), i => + Assert.Contains($"T-{i:D3}", completedIds)); + } + + [Fact] + public async Task ConcurrencyLimit_WithCancellation_HandlesGracefully() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3); + + using var cts = new CancellationTokenSource(); + + // Act: enqueue cancellable tasks + var enqueueTasks = Enumerable.Range(0, 10).Select(i => + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromSeconds(20)); + model.Unlock(); + return queue.Enqueue(model, cts.Token).AsTask(); + }); + + await Task.WhenAll(enqueueTasks); + + await Task.Delay(100, TestToken); + + await cts.CancelAsync(); + + // Change limit during cancellation + queue.ConcurrencyLimit = 1; + queue.ConcurrencyLimit = 5; + + // Shutdown should complete without hanging + await queue.ShutdownAsync(); + + await queue.WaitForIdleAsync(TestToken); + + Assert.True(work.TotalProcessed > 0); + } + +} diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs new file mode 100644 index 00000000..95b20529 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs @@ -0,0 +1,63 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace Sa.Utils.WorkQueue.Tests; + + + +public sealed record TestInput(string Id, int Value); + + +public sealed class TestWorkProcessor : ISaWork +{ + public int ExecuteCount { get; private set; } + public TestInput? LastInput { get; private set; } + + public Task Execute(TestInput input, CancellationToken cancellationToken) + { + ExecuteCount++; + LastInput = input; + return Task.CompletedTask; + } +} + + +public sealed class WorkQueueTestFixture : IDisposable +{ + public ServiceProvider ServiceProvider { get; } + + public TestWorkProcessor Processor => ServiceProvider.GetRequiredService(); + + public WorkQueueTestFixture() + { + var services = new ServiceCollection(); + services.AddSaWorkQueue(); + ServiceProvider = services.BuildServiceProvider(); + } + + public void Dispose() => ServiceProvider.Dispose(); +} + + +public class WorkQueueDiTests(WorkQueueTestFixture fixture) : IClassFixture +{ + + + [Fact] + public async Task WorkQueue_With_Di() + { + var queue = fixture.ServiceProvider.GetRequiredService>(); + Assert.NotNull(queue); + + + var input = new TestInput("test-1", 42); + await queue.Enqueue(input, TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + // Assert + + var processor = fixture.Processor; + + Assert.Equal(1, processor.ExecuteCount); + Assert.Equal(input, processor.LastInput); + } +} diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs new file mode 100644 index 00000000..a0b5bfd5 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs @@ -0,0 +1,149 @@ +namespace Sa.Utils.WorkQueue.Tests; + +public class SaWorkQueueRecoveryTests +{ + static CancellationToken TestToken => TestContext.Current.CancellationToken; + + private class TestProcessor(int failOnValue = -1) : ISaWork + { + public int ProcessedCount; + + public Task Execute(int input, CancellationToken _) + { + if (input == failOnValue) throw new InvalidOperationException("Simulated processing failure"); + Interlocked.Increment(ref ProcessedCount); + return Task.CompletedTask; + } + } + + [Fact] + public async Task ForceCancelReaders_ThenResetConcurrency_ReadersRecoverAndProcessItems() + { + // Arrange + var options = new SaWorkQueueOptions(new TestProcessor()) + { + ConcurrencyLimit = 2, + MaxConcurrency = 4 + }; + + using var queue = new SaWorkQueue(options); + + // Добавим несколько задач + await queue.Enqueue(1, TestToken); + await queue.Enqueue(2, TestToken); + await queue.WaitForIdleAsync(TestToken); + + Assert.Equal(2, ((TestProcessor)options.Processor).ProcessedCount); + + await queue.ForceCancelReadersAsync(); + await Task.Delay(100, TestToken); + + Assert.True(queue.IsIdle()); + + + queue.ConcurrencyLimit = 2; + + await Task.Delay(100, TestToken); + + await queue.Enqueue(3, TestToken); + await queue.WaitForIdleAsync(TestToken); + Assert.Equal(3, ((TestProcessor)options.Processor).ProcessedCount); + } + + [Fact] + public async Task FaultedItem_DoesNotBreakReader_QueueContinuesProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 1, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.Continue + }; + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.Enqueue(2, TestToken); // OK после fail + + await queue.WaitForIdleAsync(TestToken); + + // Assert + Assert.Equal(2, processor.ProcessedCount); + Assert.Equal(new[] { SaWorkStatus.Running, SaWorkStatus.Completed, // item 1 + SaWorkStatus.Running, SaWorkStatus.Faulted, // item -1 + SaWorkStatus.Running, SaWorkStatus.Completed }, // item 2 + statuses); + + Assert.True(queue.IsIdle()); + } + + + [Fact] + public async Task FaultedItem_BreakReader_QueueContinuesProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 2, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.StopReader + }; + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.WaitForIdleAsync(TestToken); + + await queue.Enqueue(1, TestToken); // OK + + await queue.WaitForIdleAsync(TestToken); + + // Assert + Assert.Equal(2, processor.ProcessedCount); + + Assert.Equal(1, queue.ConcurrencyLimit); + } + + [Fact] + public async Task FaultedItem_ShutdownQueue_QueueFailedProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 2, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.ShutdownQueue + }; + + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.WaitForIdleAsync(TestToken); + + + await Assert.ThrowsAsync(async () => + { + await queue.Enqueue(2, cancellationToken: CancellationToken.None); + }); + + Assert.False(queue.IsEnabled); + Assert.NotNull(queue.ShutdownError); + } +} diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs similarity index 51% rename from src/Tests/SaTests/Classes/WorkQueueTests.cs rename to src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs index 40bd5ed7..40ad9b92 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs @@ -1,6 +1,4 @@ -using Sa.Classes; - -namespace SaTests.Classes; +namespace Sa.Utils.WorkQueue.Tests; public class WorkQueueTests { @@ -10,7 +8,7 @@ private sealed class TestModel public bool WasProcessed { get; set; } } - private sealed class TestWork : IWork + private sealed class TestWork : ISaWork { public Task Execute(TestModel model, CancellationToken cancellationToken) { @@ -19,18 +17,21 @@ public Task Execute(TestModel model, CancellationToken cancellationToken) } } - private sealed class TestWorkWithDelay(TimeSpan delay) : IWork + private sealed class TestWorkWithDelay(TimeSpan delay) : ISaWork { private readonly TimeSpan _delay = delay; public async Task Execute(TestModel model, CancellationToken cancellationToken) { await Task.Delay(_delay, cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + model.WasProcessed = true; } } - private sealed class TestWorkThatThrows : IWork + private sealed class TestWorkThatThrows : ISaWork { public Task Execute(TestModel model, CancellationToken cancellationToken) { @@ -38,46 +39,53 @@ public Task Execute(TestModel model, CancellationToken cancellationToken) } } - private sealed class TestObserver : IWorkObserver + static CancellationToken TestToken => TestContext.Current.CancellationToken; + + [Fact] + public async Task Enqueue_SingleTask_ExecutesAndCompletes() { - public readonly List Changes = []; + // Arrange + var model = new TestModel(); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(50)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); + + // Act + await queue.Enqueue(model, cancellationToken: TestToken); + await queue.WaitForIdleAsync(cancellationToken: TestToken); + + // Assert + Assert.True(model.WasProcessed); + Assert.Equal(0, queue.QueueTasks); - public Task HandleChanges(TestModel model, WorkInfo work, CancellationToken cancellationToken) - { - lock (Changes) - { - Changes.Add(work); - } - return Task.CompletedTask; - } } - static CancellationToken TestToken => TestContext.Current.CancellationToken; [Fact] - public async Task Enqueue_SingleTask_ExecutesAndCompletes() + public async Task Enqueue_WaitForIdle_Completed() { // Arrange var model = new TestModel(); - var processor = new TestWork(); - using var queue = new WorkQueue(processor); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(10)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); + await queue.WaitForIdleAsync(cancellationToken: TestToken); // Act await queue.Enqueue(model, cancellationToken: TestToken); await queue.WaitForIdleAsync(cancellationToken: TestToken); + await queue.WaitForIdleAsync(cancellationToken: TestToken); + // Assert Assert.True(model.WasProcessed); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueTasks); } [Fact] public async Task Enqueue_MultipleTasks_AllExecuted() { // Arrange - var processor = new TestWork(); - using var queue = new WorkQueue(processor); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(30)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); var models = new List { new(), new(), new() @@ -91,21 +99,20 @@ public async Task Enqueue_MultipleTasks_AllExecuted() // Assert Assert.All(models, m => Assert.True(m.WasProcessed)); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueTasks); } [Fact] public async Task ConcurrencyLimit_Respected() { // Arrange - var concurrencyLimit = 3; var delay = TimeSpan.FromMilliseconds(120); var processor = new TestWorkWithDelay(delay); - using var queue = new WorkQueue(processor) - { - ConcurrencyLimit = concurrencyLimit - }; + + var concurrencyLimit = 3; + using var queue = new SaWorkQueue(SaWorkQueueOptions + .Create(processor) + .WithConcurrencyLimit(concurrencyLimit)); var models = new List { new(), new(), new(), new(), new(), new(), new() }; @@ -116,23 +123,27 @@ public async Task ConcurrencyLimit_Respected() await Task.Delay(50, TestToken); // Assert - Assert.True(queue.ActiveTasks <= concurrencyLimit, "Concurrency limit violated"); + Assert.True(queue.ConcurrencyLimit <= concurrencyLimit, "Concurrency limit violated"); + + Assert.False(queue.IsIdle()); await queue.WaitForIdleAsync(cancellationToken: TestToken); - Console.WriteLine(queue.ActiveTasks); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueTasks); } [Fact] public async Task FaultedTask_StatusIsFaulted() { + List<(SaWorkStatus Status, Exception? LastError)> changes = []; + // Arrange var processor = new TestWorkThatThrows(); - var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + + using var queue = new SaWorkQueue(SaWorkQueueOptions + .Create(processor) + .WithStatusCallback((_, s, e) => changes.Add((s, e)))); var model = new TestModel(); @@ -141,20 +152,26 @@ public async Task FaultedTask_StatusIsFaulted() await Task.Delay(100, TestToken); // Assert - var completed = observer.Changes.Find(x => x.Status == WorkStatus.Faulted); - Assert.IsType(completed.LastError); - Assert.Contains("Test exception", completed.LastError?.Message); + Assert.Contains(changes, x => x.Status == SaWorkStatus.Faulted); + + Assert.Contains(changes, x => x.LastError is InvalidOperationException); + Assert.Contains(changes, x => x.LastError?.Message == "Test exception"); } [Fact] - public async Task CancelledTask_StatusIsCancelled() + public async Task CancelledTask_StatusIsAborted() { + List statuses = []; + + // Arrange using var cts = new CancellationTokenSource(); var processor = new TestWorkWithDelay(TimeSpan.FromSeconds(5)); - var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + + using var queue = new SaWorkQueue(SaWorkQueueOptions + .Create(processor) + .WithStatusCallback((_, s, _) => statuses.Add(s))); var model = new TestModel(); @@ -165,27 +182,38 @@ public async Task CancelledTask_StatusIsCancelled() await queue.WaitForIdleAsync(TestToken); // Assert - var cancelled = observer.Changes.Find(x => x.Status == WorkStatus.Cancelled); - Assert.NotEqual(0, cancelled.Id); + + Assert.Contains(SaWorkStatus.Aborted, statuses); } [Fact] public async Task ShutdownAsync_StopsProcessing() { + List errors = []; + // Arrange - var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(500)); - var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(300)); + + using var queue = new SaWorkQueue(SaWorkQueueOptions + .Create(processor) + .WithStatusCallback((m, s, e) => + { + if (s == SaWorkStatus.Cancelled) + errors.Add(s); + })); for (int i = 0; i < 5; i++) await queue.Enqueue(new TestModel(), cancellationToken: TestToken); + + await Task.Delay(50, TestToken); + // Act await queue.ShutdownAsync(); // Assert Assert.False(queue.IsEnabled); - Assert.True(queue.ActiveTasks <= 5); + Assert.True(queue.QueueTasks <= 5); await Assert.ThrowsAsync(async () => { @@ -193,46 +221,26 @@ await Assert.ThrowsAsync(async () => }); await queue.WaitForIdleAsync(TestToken); - Assert.Equal(0, queue.ActiveTasks); + Assert.Equal(0, queue.QueueTasks); - var canceled = observer.Changes.Count(c => c.Status == WorkStatus.Cancelled); - Assert.Equal(5, canceled); + Assert.Equal(5, errors.Count); } - [Fact] - public async Task StatusChanged_Event_FiresOnStateChange() - { - // Arrange - var processor = new TestWork(); - WorkInfo? lastInfo = null; - var eventTcs = new TaskCompletionSource(); - using var queue = new WorkQueue(processor); - - queue.StatusChanged += (sender, info) => - { - lastInfo = info; - if (info.Status == WorkStatus.Completed) - eventTcs.TrySetResult(info); - }; - - var model = new TestModel(); - - // Act - await queue.Enqueue(model, cancellationToken: TestToken); - var result = await eventTcs.Task; - - // Assert - Assert.Equal(WorkStatus.Completed, result.Status); - } [Fact] public async Task Observer_InvokedOnStateChange() { + + List statuses = []; + + // Arrange - var observer = new TestObserver(); var processor = new TestWork(); - using var queue = new WorkQueue(processor, observer); + using var queue = new SaWorkQueue( + SaWorkQueueOptions + .Create(processor) + .WithStatusCallback((_, s, _) => statuses.Add(s))); var model = new TestModel(); @@ -241,10 +249,9 @@ public async Task Observer_InvokedOnStateChange() await queue.WaitForIdleAsync(cancellationToken: TestToken); // Assert - Assert.True(observer.Changes.Count >= 3); // Queued → Running → Completed - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Queued); - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Running); - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Completed); + Assert.True(statuses.Count >= 2); // Running → Completed + Assert.Contains(statuses, x => x == SaWorkStatus.Running); + Assert.Contains(statuses, x => x == SaWorkStatus.Completed); } [Fact] @@ -252,25 +259,26 @@ public async Task DisposeAsync_CleansUpResources() { // Arrange var processor = new TestWork(); - var queue = new WorkQueue(processor); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(processor)); await queue.Enqueue(new TestModel(), cancellationToken: TestToken); await queue.DisposeAsync(); - // Act & Assert - Assert.False(queue.IsEnabled); - Assert.Equal(0, queue.QueuedTasks); // Writer completed await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); } [Fact] - public void ConcurrencyLimit_InvalidValue_Throws() + public void ConcurrencyLimit_InvalidValue_Clamp() { - var queue = new WorkQueue(new TestWork()); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(new TestWork())) + { + ConcurrencyLimit = -1 + }; - Assert.Throws(() => queue.ConcurrencyLimit = 0); - Assert.Throws(() => queue.ConcurrencyLimit = -1); + Assert.Equal(0, queue.ConcurrencyLimit); queue.Dispose(); } @@ -278,9 +286,26 @@ public void ConcurrencyLimit_InvalidValue_Throws() [Fact] public async Task Enqueue_Disposed_Throws() { - var queue = new WorkQueue(new TestWork()); - await queue.DisposeAsync(); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(new TestWork())); + queue.Dispose(); + + await Assert.ThrowsAsync(() + => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); + } + + [Fact] + public async Task ShutdownAsync_CleansUpResources() + { + // Arrange + var processor = new TestWork(); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(processor)); - await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); + await queue.Enqueue(new TestModel(), cancellationToken: TestToken); + await queue.ShutdownAsync(); + + await Assert.ThrowsAsync(() + => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); } } diff --git a/src/Tests/SaTests/Classes/ResetLazyTests.cs b/src/Tests/SaTests/Classes/ResetLazyTests.cs new file mode 100644 index 00000000..cbe77930 --- /dev/null +++ b/src/Tests/SaTests/Classes/ResetLazyTests.cs @@ -0,0 +1,101 @@ +using Sa.Classes; +using System.Collections.Concurrent; + +namespace SaTests.Classes; + +public class ResetLazyTests +{ + [Fact] + public void Value_ShouldBeInitializedLazy() + { + // Arrange + int counter = 0; + var lazy = new ResetLazy(() => Interlocked.Increment(ref counter)); + + // Assert + Assert.Equal(0, counter); + Assert.False(lazy.IsValueCreated); + + // Act + var val = lazy.Value; + + // Assert + Assert.Equal(1, val); + Assert.Equal(1, counter); + Assert.True(lazy.IsValueCreated); + } + + [Fact] + public void Reset_ShouldClearValue_AndCallCleanup() + { + // Arrange + int counter = 0; + int cleanupValue = -1; + var lazy = new ResetLazy>( + () => [++counter], + valueReset: list => cleanupValue = list[0] + ); + + // Act + var firstList = lazy.Value; + lazy.Reset(); + + // Assert + Assert.False(lazy.IsValueCreated); + Assert.Equal(1, cleanupValue); // Cleanup вызван для первого списка + + var secondList = lazy.Value; + Assert.Equal(2, secondList[0]); // Создан новый список + } + + [Fact] + public async Task MultithreadedAccess_ShouldCreateValueOnlyOnce() + { + // Arrange + int factoryCalls = 0; + var lazy = new ResetLazy(() => + { + Interlocked.Increment(ref factoryCalls); + Thread.Sleep(50); // Симуляция работы + return Guid.NewGuid().ToString(); + }); + + // Act + var tasks = Enumerable.Range(0, 50) + .Select(_ => Task.Run(() => lazy.Value)) + .ToList(); + + var results = await Task.WhenAll(tasks); + + // Assert + Assert.Equal(1, factoryCalls); // Фабрика вызвана ровно 1 раз + Assert.All(results, r => Assert.Equal(results[0], r)); // Все потоки получили одну строку + } + + [Fact] + public async Task MultithreadedReset_ShouldBeSafe() + { + // Arrange + var disposedItems = new ConcurrentBag(); + int factoryCalls = 0; + var lazy = new ResetLazy( + () => Interlocked.Increment(ref factoryCalls), + valueReset: val => disposedItems.Add(val) + ); + + // Act + // Интенсивно читаем и сбрасываем из разных потоков + var tasks = Enumerable.Range(0, 100).Select(i => Task.Run(() => + { + if (i % 2 == 0) _ = lazy.Value; + else lazy.Reset(); + })); + + await Task.WhenAll(tasks); + + // Assert + // Проверяем, что сумма созданных объектов совпадает с суммой удаленных + 1 (если текущий остался) + int currentExist = lazy.IsValueCreated ? 1 : 0; + Assert.Equal(factoryCalls, disposedItems.Count + currentExist); + } +}