From df1ae9eae1df656a3702baf92af11ac6346b8cc6 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 25 Mar 2026 17:06:25 +0300 Subject: [PATCH 01/28] outbox bump v0.9.0 --- .../DatabaseConfigurationProvider.cs | 2 +- .../Sa.Configuration.PostgreSql.csproj | 2 +- src/Sa.Configuration/Sa.Configuration.csproj | 2 +- .../IPgDataSourceSettingsBuilder.cs | 5 +- .../Configuration/PgDataSourceSettings.cs | 34 +++++++++- .../PgDataSourceSettingsBuilder.cs | 16 ++--- src/Sa.Data.PostgreSql/DbCommandExtensions.cs | 4 +- src/Sa.Data.PostgreSql/IPgDataSource.cs | 65 ++++++++++++++----- src/Sa.Data.PostgreSql/IPgDistributedLock.cs | 5 +- src/Sa.Data.PostgreSql/PgDataSource.cs | 31 +++++++-- src/Sa.Data.PostgreSql/PgDistributedLock.cs | 4 +- src/Sa.Data.PostgreSql/PgRetryStrategy.cs | 4 +- .../Sa.Data.PostgreSql.csproj | 2 +- src/Sa.Data.PostgreSql/Setup.cs | 21 +++++- src/Sa.Data.S3/S3BucketSettings.cs | 3 +- src/Sa.Data.S3/Sa.Data.S3.csproj | 2 +- src/Sa.Data.S3/Setup.cs | 8 +-- .../FileSystemStorage.cs | 31 ++++++--- .../FileSystemStorageSettings.cs | 2 +- .../Sa.HybridFileStorage.FileSystem.csproj | 2 +- src/Sa.HybridFileStorage.FileSystem/Setup.cs | 2 +- .../FileIdParser.cs | 32 +++++---- .../IPostgresFileStorageConfiguration.cs | 1 - .../PostgresFileStorage.cs | 35 ++++++++-- .../PostgresFileStorageConfiguration.cs | 35 +++++----- .../PostgresFileStorageOptions.cs | 2 +- .../Sa.HybridFileStorage.Postgres.csproj | 2 +- src/Sa.HybridFileStorage.S3/S3FileStorage.cs | 3 +- .../Sa.HybridFileStorage.S3.csproj | 30 ++++----- .../Domain/IFileStorage.cs | 5 +- src/Sa.HybridFileStorage/FileMetadata.cs | 2 + src/Sa.HybridFileStorage/HybridFileStorage.cs | 24 ++++--- .../HybridFileStorageExtensions.cs | 37 ++++++++--- .../IHybridFileStorage.cs | 4 +- .../Sa.HybridFileStorage.csproj | 2 +- .../Commands/ErrorDeliveryCommand.cs | 5 +- .../Commands/InsertMsgTypeCommand.cs | 4 +- .../Commands/SelectTenantCommand.cs | 16 +++-- .../Configuration/IPgOutboxConfiguration.cs | 6 +- .../Configuration/PgOutboxConfiguration.cs | 10 ++- .../PgOutboxTableSettingsExtensions.cs | 4 +- .../Partitional/MsgMigrationSupport.cs | 3 +- .../Sa.Outbox.PostgreSql.csproj | 2 +- .../Services/IOutboxTaskLoader.cs | 3 +- .../Services/OutboxPartRepository.cs | 16 +++-- .../SqlBuilder/SqlOutboxBuilder.cs | 4 +- src/Sa.Outbox/Delivery/DeliveryBuilder.cs | 9 ++- src/Sa.Outbox/Delivery/DeliveryTenant.cs | 15 +++-- src/Sa.Outbox/Delivery/Job/Setup.cs | 10 +-- src/Sa.Outbox/Delivery/OutboxContext.cs | 12 ++-- .../Delivery/OutboxContextFactory.cs | 2 +- src/Sa.Outbox/Delivery/Setup.cs | 5 +- .../Exceptions/DeliveryPermanentException.cs | 6 +- src/Sa.Outbox/IOutboxBuilder.cs | 16 +++++ src/Sa.Outbox/IOutboxPublishable.cs | 14 ++++ .../Metadata/IOutboxMessageMetadataBuilder.cs | 10 ++- .../Metadata/MetadataConfiguration.cs | 4 +- .../Metadata/OutboxMessageMetadata.cs | 8 ++- src/Sa.Outbox/Metadata/Setup.cs | 14 ++-- src/Sa.Outbox/OutboxBuilder.cs | 17 ++--- src/Sa.Outbox/Partitional/ITenantSource.cs | 3 +- .../Partitional/OutboxPartitionalSupport.cs | 3 +- src/Sa.Outbox/Partitional/Setup.cs | 8 ++- src/Sa.Outbox/Partitional/TenantSettings.cs | 10 ++- src/Sa.Outbox/PlugServices/IOutboxPlugin.cs | 22 +++---- .../PlugServices/IOutboxTenantDetector.cs | 4 +- .../Publication/IOutboxMessagePublisher.cs | 35 +++++++--- .../Publication/OutboxMessagePublisher.cs | 7 +- src/Sa.Outbox/Sa.Outbox.csproj | 2 +- .../Cache/IPartCache.cs | 12 +++- .../Cache/PartCache.cs | 16 +++-- .../Cleaning/Setup.cs | 4 +- .../Configuration/Builder/ISettingsBuilder.cs | 2 + .../Configuration/Builder/SettingsBuilder.cs | 4 +- .../Configuration/Builder/Setup.cs | 20 ++---- .../Configuration/PartConfiguration.cs | 2 +- .../IPartitionManager.cs | 10 ++- .../Migration/PartMigrationService.cs | 4 +- src/Sa.Partitional.PostgreSql/Part.cs | 3 +- .../PartitionManager.cs | 6 +- .../Partitional/IPartRepository.cs | 28 ++++++-- .../Partitional/PartRepository.cs | 13 ++-- .../Sa.Partitional.PostgreSql.csproj | 2 +- .../SqlBuilder/SqlBuilder.cs | 12 ++-- src/Samples/Configuration.Web/Program.cs | 2 +- .../HybridFileStorage.Console/Program.cs | 1 - .../FileSystemStorageTests.cs | 14 +++- .../FileIdParserTests.cs | 7 +- .../PostgresFileStorageTests.cs | 18 +++-- .../HybridFileStorageTests.cs | 5 +- .../Delivery/DeliveryBatchingWindowTests.cs | 12 +++- .../Publisher/OutboxPublisherTests.cs | 10 ++- .../Configuration/ConfigurationPartTests.cs | 2 - 93 files changed, 656 insertions(+), 323 deletions(-) create mode 100644 src/Sa.Outbox/IOutboxPublishable.cs diff --git a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs index 80576f00..29a1974a 100644 --- a/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs +++ b/src/Sa.Configuration.PostgreSql/DatabaseConfigurationProvider.cs @@ -19,7 +19,7 @@ private async Task LoadAsync(PostgreSqlConfigurationOptions options) { try { - using var dataSource = new PgDataSource(new(options.ConnectionString)); + using var dataSource = IPgDataSource.Create(options.ConnectionString); return await dataSource.ExecuteReader(options.SelectSql, (reader, _) => { diff --git a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj index 5e15b30f..9fa80b67 100644 --- a/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj +++ b/src/Sa.Configuration.PostgreSql/Sa.Configuration.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 add a PostgreSQL-based configuration source to IConfiguration diff --git a/src/Sa.Configuration/Sa.Configuration.csproj b/src/Sa.Configuration/Sa.Configuration.csproj index 02f9ca02..937ac6e3 100644 --- a/src/Sa.Configuration/Sa.Configuration.csproj +++ b/src/Sa.Configuration/Sa.Configuration.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 extensions for Configuration diff --git a/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs b/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs index e4377287..0b022182 100644 --- a/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs +++ b/src/Sa.Data.PostgreSql/Configuration/IPgDataSourceSettingsBuilder.cs @@ -1,10 +1,7 @@ - -namespace Sa.Data.PostgreSql; +namespace Sa.Data.PostgreSql; public interface IPgDataSourceSettingsBuilder { void WithConnectionString(string connectionString); void WithConnectionString(Func implementationFactory); - void WithSettings(PgDataSourceSettings settings); - void WithSettings(Func implementationFactory); } diff --git a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs index 98039b3e..6fa2dd79 100644 --- a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs +++ b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettings.cs @@ -1,6 +1,34 @@ -namespace Sa.Data.PostgreSql; +using Npgsql; -public sealed class PgDataSourceSettings(string connectionString) +namespace Sa.Data.PostgreSql; + +internal sealed class PgDataSourceSettings(string connectionString) { - public string ConnectionString { get; } = connectionString; + private string? _searchPath; + + public string ConnectionString => connectionString; + + public string GetSearchPath() + { + if (_searchPath is not null) + return _searchPath; + + try + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + return _searchPath = builder.SearchPath ?? "public"; + } + catch + { + return _searchPath = "public"; + } + } + + public void Validate() + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new InvalidOperationException("Connection string cannot be null or empty."); + } + } } diff --git a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs index db56cba1..0f376739 100644 --- a/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs +++ b/src/Sa.Data.PostgreSql/Configuration/PgDataSourceSettingsBuilder.cs @@ -6,22 +6,14 @@ namespace Sa.Data.PostgreSql.Configuration; internal sealed class PgDataSourceSettingsBuilder(IServiceCollection services) : IPgDataSourceSettingsBuilder { public void WithConnectionString(string connectionString) - { - services.TryAddSingleton(new PgDataSourceSettings(connectionString)); - } + => services.TryAddSingleton(new PgDataSourceSettings(connectionString)); public void WithConnectionString(Func implementationFactory) - { - services.TryAddSingleton(sp => new PgDataSourceSettings(implementationFactory(sp))); - } + => services.TryAddSingleton(sp => new PgDataSourceSettings(implementationFactory(sp))); public void WithSettings(Func implementationFactory) - { - services.TryAddSingleton(implementationFactory); - } + => services.TryAddSingleton(implementationFactory); public void WithSettings(PgDataSourceSettings settings) - { - services.TryAddSingleton(settings); - } + => services.TryAddSingleton(settings); } diff --git a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs index e63c57bf..cf1e0731 100644 --- a/src/Sa.Data.PostgreSql/DbCommandExtensions.cs +++ b/src/Sa.Data.PostgreSql/DbCommandExtensions.cs @@ -1,6 +1,6 @@ -using System.Collections.ObjectModel; +using Npgsql; +using System.Collections.ObjectModel; using System.Runtime.CompilerServices; -using Npgsql; namespace Sa.Data.PostgreSql; diff --git a/src/Sa.Data.PostgreSql/IPgDataSource.cs b/src/Sa.Data.PostgreSql/IPgDataSource.cs index 8427f7c2..11978fc2 100644 --- a/src/Sa.Data.PostgreSql/IPgDataSource.cs +++ b/src/Sa.Data.PostgreSql/IPgDataSource.cs @@ -2,46 +2,71 @@ namespace Sa.Data.PostgreSql; -public interface IPgDataSource +public interface IPgDataSource : IDisposable, IAsyncDisposable { - public static IPgDataSource Create(string connectionString) => new PgDataSource(new PgDataSourceSettings(connectionString)); + public static IPgDataSource Create(string connectionString) + => new PgDataSource(new PgDataSourceSettings(connectionString)); + + string GetSearchPath(); ValueTask OpenDbConnection(CancellationToken cancellationToken); - Task ExecuteNonQuery(string sql, Action? initCommand, CancellationToken cancellationToken = default); + Task ExecuteNonQuery( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default); - Task ExecuteNonQuery(string sql, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) - => ExecuteNonQuery(sql, cmd => FillParams(cmd, parameters), cancellationToken); + Task ExecuteNonQuery( + string sql, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) + => ExecuteNonQuery(sql, cmd => FillParams(cmd, parameters), cancellationToken); Task ExecuteNonQuery(string sql, CancellationToken cancellationToken = default) => ExecuteNonQuery(sql, [], cancellationToken); - Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default); + Task ExecuteScalar( + string sql, Action? initCommand, CancellationToken cancellationToken = default); - async Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default) - => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); + async Task ExecuteScalar( + string sql, Action? initCommand, CancellationToken cancellationToken = default) + => ((T)(await ExecuteScalar(sql, initCommand, cancellationToken))!); // ExecuteReader - Task ExecuteReader(string sql, Action read, Action? initCommand, CancellationToken cancellationToken = default); - - Task ExecuteReader(string sql, Action read, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + Task ExecuteReader( + string sql, + Action read, + Action? initCommand, + CancellationToken cancellationToken = default); + + Task ExecuteReader( + string sql, + Action read, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) => ExecuteReader(sql, read, cmd => FillParams(cmd, parameters), cancellationToken); - async Task ExecuteReader(string sql, Action read, CancellationToken cancellationToken = default) + async Task ExecuteReader( + string sql, Action read, CancellationToken cancellationToken = default) => await ExecuteReader(sql, read, [], cancellationToken); // ExecuteReaderList - async Task> ExecuteReaderList(string sql, Func read, CancellationToken cancellationToken = default) + async Task> ExecuteReaderList( + string sql, Func read, CancellationToken cancellationToken = default) { List list = []; await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), cancellationToken); return list; } - async Task> ExecuteReaderList(string sql, Func read, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + async Task> ExecuteReaderList( + string sql, + Func read, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) { List list = []; await ExecuteReader(sql, (reader, _) => list.Add(read(reader)), parameters, cancellationToken); @@ -57,7 +82,10 @@ Task ExecuteReaderFirst(string sql, CancellationToken cancellationToken = return ExecuteReaderFirst(sql, [], cancellationToken); } - async Task ExecuteReaderFirst(string sql, IReadOnlyCollection parameters, CancellationToken cancellationToken = default) + async Task ExecuteReaderFirst( + string sql, + IReadOnlyCollection parameters, + CancellationToken cancellationToken = default) { T value = default!; @@ -85,9 +113,10 @@ await ExecuteReader(sql, (reader, _) => // BeginBinaryImport - - - ValueTask BeginBinaryImport(string sql, Func> write, CancellationToken cancellationToken = default); + ValueTask BeginBinaryImport( + string sql, + Func> write, + CancellationToken cancellationToken = default); void FillParams(NpgsqlCommand cmd, IReadOnlyCollection parameters) { diff --git a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs b/src/Sa.Data.PostgreSql/IPgDistributedLock.cs index 5ebf8989..1d4eec50 100644 --- a/src/Sa.Data.PostgreSql/IPgDistributedLock.cs +++ b/src/Sa.Data.PostgreSql/IPgDistributedLock.cs @@ -2,5 +2,8 @@ public interface IPgDistributedLock { - Task TryExecuteInDistributedLock(long lockId, Func exclusiveLockTask, CancellationToken cancellationToken); + Task TryExecuteInDistributedLock( + long lockId, + Func exclusiveLockTask, + CancellationToken cancellationToken); } diff --git a/src/Sa.Data.PostgreSql/PgDataSource.cs b/src/Sa.Data.PostgreSql/PgDataSource.cs index 96f4a311..40fe0980 100644 --- a/src/Sa.Data.PostgreSql/PgDataSource.cs +++ b/src/Sa.Data.PostgreSql/PgDataSource.cs @@ -6,11 +6,15 @@ namespace Sa.Data.PostgreSql; /// NpgsqlDataSource lite /// /// connection string -public sealed class PgDataSource(PgDataSourceSettings settings) : IPgDataSource, IDisposable, IAsyncDisposable +internal sealed class PgDataSource(PgDataSourceSettings settings) : IPgDataSource { - private readonly Lazy _dataSource = new(() => NpgsqlDataSource.Create(settings.ConnectionString)); + private readonly Lazy _dataSource + = new(() => NpgsqlDataSource.Create(settings.ConnectionString)); - public ValueTask OpenDbConnection(CancellationToken cancellationToken) => _dataSource.Value.OpenConnectionAsync(cancellationToken); + public string GetSearchPath() => settings.GetSearchPath(); + + public ValueTask OpenDbConnection(CancellationToken cancellationToken) + => _dataSource.Value.OpenConnectionAsync(cancellationToken); public void Dispose() { @@ -28,7 +32,10 @@ public async ValueTask DisposeAsync() } } - public async ValueTask BeginBinaryImport(string sql, Func> write, CancellationToken cancellationToken = default) + public async ValueTask BeginBinaryImport( + string sql, + Func> write, + CancellationToken cancellationToken = default) { using NpgsqlConnection db = await OpenDbConnection(cancellationToken); using NpgsqlBinaryImporter writer = await db.BeginBinaryImportAsync(sql, cancellationToken); @@ -36,7 +43,10 @@ public async ValueTask BeginBinaryImport(string sql, Func ExecuteNonQuery(string sql, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteNonQuery( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default) { using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); using NpgsqlCommand cmd = new(sql, connection); @@ -44,7 +54,10 @@ public async Task ExecuteNonQuery(string sql, Action? initCo return await cmd.ExecuteNonQueryAsync(cancellationToken); } - public async Task ExecuteScalar(string sql, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteScalar( + string sql, + Action? initCommand, + CancellationToken cancellationToken = default) { using NpgsqlConnection connection = await OpenDbConnection(cancellationToken); using NpgsqlCommand cmd = new(sql, connection); @@ -52,7 +65,11 @@ public async Task ExecuteNonQuery(string sql, Action? initCo return await cmd.ExecuteScalarAsync(cancellationToken); } - public async Task ExecuteReader(string sql, Action read, Action? initCommand, CancellationToken cancellationToken = default) + public async Task ExecuteReader( + string sql, + Action read, + Action? initCommand, + CancellationToken cancellationToken = default) { int rowCount = 0; diff --git a/src/Sa.Data.PostgreSql/PgDistributedLock.cs b/src/Sa.Data.PostgreSql/PgDistributedLock.cs index 84eaab17..3b49dd87 100644 --- a/src/Sa.Data.PostgreSql/PgDistributedLock.cs +++ b/src/Sa.Data.PostgreSql/PgDistributedLock.cs @@ -9,7 +9,9 @@ namespace Sa.Data.PostgreSql; /// /// /// -internal sealed partial class PgDistributedLock(PgDataSourceSettings settings, ILogger? logger = null) : IPgDistributedLock +internal sealed partial class PgDistributedLock( + PgDataSourceSettings settings, + ILogger? logger = null) : IPgDistributedLock { private readonly ILogger _logger = logger ?? NullLogger.Instance; diff --git a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs index 4f38ad17..1e36b9ac 100644 --- a/src/Sa.Data.PostgreSql/PgRetryStrategy.cs +++ b/src/Sa.Data.PostgreSql/PgRetryStrategy.cs @@ -15,7 +15,9 @@ public static ValueTask ExecuteWithRetry( fun: fun, retryCount: retryCount, initialDelay: initialDelay - , next: (ex, i) => next != null ? next(ex, i) : (ex is NpgsqlException exception) && exception.IsTransient + , next: (ex, i) => next != null + ? next(ex, i) + : (ex is NpgsqlException exception) && exception.IsTransient , cancellationToken: cancellationToken); } } diff --git a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj index 88bc49b8..1862684d 100644 --- a/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj +++ b/src/Sa.Data.PostgreSql/Sa.Data.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple client for Npqsql diff --git a/src/Sa.Data.PostgreSql/Setup.cs b/src/Sa.Data.PostgreSql/Setup.cs index 8cb25480..66e5ce5c 100644 --- a/src/Sa.Data.PostgreSql/Setup.cs +++ b/src/Sa.Data.PostgreSql/Setup.cs @@ -1,16 +1,33 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Npgsql; using Sa.Data.PostgreSql.Configuration; namespace Sa.Data.PostgreSql; public static class Setup { - public static IServiceCollection AddSaPostgreSqlDataSource(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddSaPostgreSqlDataSource( + this IServiceCollection services, + Action? configure = null) { PgDataSourceSettingsBuilder builder = new(services); configure?.Invoke(builder); - services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + PgDataSourceSettings? settings = sp.GetService(); + + if (settings is null) + { + var connection = sp.GetService()?.ConnectionString + ?? throw new InvalidOperationException("Empty connection string"); + settings = new(connection); + } + + settings.Validate(); + + return new PgDataSource(settings); + }); services.TryAddSingleton(); return services; } diff --git a/src/Sa.Data.S3/S3BucketSettings.cs b/src/Sa.Data.S3/S3BucketSettings.cs index b48aec41..9fae5e49 100644 --- a/src/Sa.Data.S3/S3BucketSettings.cs +++ b/src/Sa.Data.S3/S3BucketSettings.cs @@ -6,7 +6,7 @@ public class S3BucketSettings { public required string AccessKey { get; init; } - + public required string SecretKey { get; init; } public required string Bucket { get; init; } /// @@ -17,7 +17,6 @@ public class S3BucketSettings public string Region { get; init; } = "us-east-1"; - public required string SecretKey { get; init; } public string Service { get; init; } = "s3"; diff --git a/src/Sa.Data.S3/Sa.Data.S3.csproj b/src/Sa.Data.S3/Sa.Data.S3.csproj index 13c1c00e..8bb96edb 100644 --- a/src/Sa.Data.S3/Sa.Data.S3.csproj +++ b/src/Sa.Data.S3/Sa.Data.S3.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.9.0 Sa.Data.S3 Simple client for S3 (Sa.Data.S3) s3 diff --git a/src/Sa.Data.S3/Setup.cs b/src/Sa.Data.S3/Setup.cs index 3078f053..224c1834 100644 --- a/src/Sa.Data.S3/Setup.cs +++ b/src/Sa.Data.S3/Setup.cs @@ -11,19 +11,15 @@ public static IServiceCollection AddSaS3BucketClient( { services.TryAddSingleton(settings); - // https://www.milanjovanovic.tech/blog/the-right-way-to-use-httpclient-in-dotnet services .AddHttpClient((sp, client) => { client.Timeout = settings.TotalRequestTimeout; client.BaseAddress = new Uri(settings.Endpoint); }) - .ConfigurePrimaryHttpMessageHandler(() => + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler() { - return new SocketsHttpHandler() - { - PooledConnectionLifetime = TimeSpan.FromMinutes(15) - }; + PooledConnectionLifetime = TimeSpan.FromMinutes(15) }) .SetHandlerLifetime(Timeout.InfiniteTimeSpan) .AddStandardResilienceHandler(options => diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index bb38ba59..7baae69b 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -4,6 +4,9 @@ using System.Globalization; using System.Runtime.CompilerServices; +/// +/// fs://share/tenant/filename +/// internal sealed class FileSystemStorage( FileSystemStorageSettings settings, TimeProvider? timeProvider = null) : IFileStorage @@ -11,6 +14,9 @@ internal sealed class FileSystemStorage( private const string SchemeSeparator = "://"; private readonly string _basePath = Path.TrimEndingDirectorySeparator( + Path.GetFullPath(settings.BasePath)); + + private readonly string _basePathScope = Path.TrimEndingDirectorySeparator( Path.GetFullPath(Path.Combine(settings.BasePath, settings.ScopeName))); private readonly string _schemePrefix = $"{settings.StorageType}{SchemeSeparator}"; @@ -53,7 +59,7 @@ public async Task UploadAsync( string filename = PathSanitizer.SanitizeRelativePath(metadata.FileName); - string relativePath = string.Concat(metadata.TenantId.ToString(), "/", filename); + string relativePath = string.Concat(ScopeName, "/", metadata.TenantId.ToString(), "/", filename); string filePath = Path.Combine(_basePath, relativePath); EnsurePathWithinBase(filePath); @@ -75,10 +81,10 @@ public async Task UploadAsync( await fileStream.CopyToAsync(fileStreamOutput, cancellationToken).ConfigureAwait(false); return new StorageResult( - string.Concat(_schemePrefix, relativePath), - Path.GetFullPath(filePath), - _storageType, - _timeProvider.GetUtcNow()); + FileId: string.Concat(_schemePrefix, relativePath), + AbsoluteUrl: Path.GetFullPath(filePath), + StorageType: _storageType, + UploadedAt: _timeProvider.GetUtcNow()); } public async Task DownloadAsync( @@ -187,7 +193,7 @@ private bool IsPathWithinBase(string path) { return Path.GetFullPath(path) .AsSpan() - .StartsWith(_basePath.AsSpan(), StringComparison.Ordinal); + .StartsWith(_basePathScope.AsSpan(), StringComparison.Ordinal); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -206,7 +212,7 @@ private void EnsurePathWithinBase(string path) if (!CanProcess(fileId)) return Task.FromResult(null); - // Парсинг формата: "storageType://tenantId/filename" + // Парсинг формата: "storageType://scope/tenant/filename" ReadOnlySpan span = fileId.AsSpan(); int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); if (schemeEnd == -1) @@ -219,14 +225,21 @@ private void EnsurePathWithinBase(string path) if (slashIndex == -1) return Task.FromResult(null); - var tenantSpan = pathPart[..slashIndex]; - var fileNameSpan = pathPart[(slashIndex + 1)..]; + var scopeSpan = pathPart[..slashIndex]; + + var nextSpan = pathPart[(slashIndex + 1)..]; + slashIndex = nextSpan.IndexOf('/'); + + var tenantSpan = nextSpan[..slashIndex]; + var fileNameSpan = nextSpan[(slashIndex + 1)..]; if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) return Task.FromResult(null); var metadata = new FileMetadata { + StorageType = StorageType, + ScopeName = scopeSpan.ToString(), FileName = fileNameSpan.ToString(), TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs index c4a0e4c6..02db3ffc 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs @@ -13,7 +13,7 @@ public sealed record FileSystemStorageSettings [StringLength(100, MinimumLength = 1)] public bool IsReadOnly { get; init; } = false; - public string ScopeName { get; init; } = string.Empty; + public string ScopeName { get; init; } = "share"; public int BufferSize { get; init; } = 256 * 1024; diff --git a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj index a458697d..5e476585 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj +++ b/src/Sa.HybridFileStorage.FileSystem/Sa.HybridFileStorage.FileSystem.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management diff --git a/src/Sa.HybridFileStorage.FileSystem/Setup.cs b/src/Sa.HybridFileStorage.FileSystem/Setup.cs index 7bfc7000..e8f1d203 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Setup.cs +++ b/src/Sa.HybridFileStorage.FileSystem/Setup.cs @@ -28,7 +28,7 @@ public static IServiceCollection AddSaFileSystemFileStorage( { BasePath = options.BasePath, IsReadOnly = options.IsReadOnly, - ScopeName = options.ScopeName ?? string.Empty, + ScopeName = string.IsNullOrWhiteSpace(options.ScopeName) ? "share" : options.ScopeName, StorageType = options.StorageType, }, sp.GetService()); }); diff --git a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs index 909c0ef0..d9304e5b 100644 --- a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs +++ b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs @@ -6,8 +6,9 @@ internal static class FileIdParser { public const string SchemeSeparator = "://"; - public static bool TryParseFileIdWithFilename( + public static bool TryParse( string fileId, + out string scopeName, out int tenantId, out long timestamp, out string fileName) @@ -15,6 +16,7 @@ public static bool TryParseFileIdWithFilename( tenantId = default; timestamp = default; fileName = string.Empty; + scopeName = string.Empty; if (string.IsNullOrEmpty(fileId)) return false; @@ -22,19 +24,23 @@ public static bool TryParseFileIdWithFilename( int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); if (schemeEnd == -1) return false; - var afterScheme = span[(schemeEnd + SchemeSeparator.Length)..]; - int tableEnd = afterScheme.IndexOf('/'); - if (tableEnd == -1) return false; - afterScheme = afterScheme[(tableEnd + 1)..]; + var afterSpan = span[(schemeEnd + SchemeSeparator.Length)..]; + int scopeEnd = afterSpan.IndexOf('/'); + if (scopeEnd == -1) return false; - int tenantEnd = afterScheme.IndexOf('/'); + scopeName = afterSpan[..scopeEnd].ToString(); + + afterSpan = afterSpan[(scopeEnd + 1)..]; + + + int tenantEnd = afterSpan.IndexOf('/'); if (tenantEnd == -1) return false; - var tenantSpan = afterScheme[..tenantEnd]; + var tenantSpan = afterSpan[..tenantEnd]; if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out tenantId)) return false; - var afterTenant = afterScheme[(tenantEnd + 1)..]; + var afterTenant = afterSpan[(tenantEnd + 1)..]; int timestampEnd = afterTenant.IndexOf('/'); if (timestampEnd == -1) return false; @@ -49,13 +55,15 @@ public static bool TryParseFileIdWithFilename( public static string FormatToFileId( string storageType, - string tableName, + string scopeName, int tenantId, DateTimeOffset date, string fileName) - => $"{storageType}://{tableName}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; + => $"{storageType}://{scopeName}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; - public static string NormalizeFileName(string fileName) => fileName.TrimStart('\\', '/').Replace('\\', '/'); + public static string NormalizeFileName(string fileName) + => fileName.TrimStart('\\', '/').Replace('\\', '/'); - public static string GetFileExtension(string fileName) => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); + public static string GetFileExtension(string fileName) + => Path.GetExtension(fileName ?? string.Empty).ToLower().TrimStart('.'); } diff --git a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs index 1f2f1f90..bdf74c52 100644 --- a/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/IPostgresFileStorageConfiguration.cs @@ -6,7 +6,6 @@ public interface IPostgresFileStorageConfiguration { IPostgresFileStorageConfiguration AddDataSource(Action? configure = null); IPostgresFileStorageConfiguration ConfigureOptions(Action configure); - IPostgresFileStorageConfiguration ConfigureOptions(Action configure); IPostgresFileStorageConfiguration WithStorageType(string storageType); IPostgresFileStorageConfiguration WithSchemaName(string schemaName); IPostgresFileStorageConfiguration WithTableName(string tableName); diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index 98a2d0e2..e3ba18a9 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -7,6 +7,9 @@ namespace Sa.HybridFileStorage.Postgres; +/// +/// pg://scope/tenant/timestamp/filename +/// internal sealed class PostgresFileStorage( IPgDataSource dataSource, IPartitionManager partManager, @@ -57,6 +60,7 @@ private readonly string _schemePrefix public string ScopeName => scopeName; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void EnsureWritable() { @@ -67,9 +71,24 @@ private void EnsureWritable() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CanProcess(string? fileId) => - !string.IsNullOrEmpty(fileId) && - fileId.AsSpan().StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal); + public bool CanProcess(string? fileId) + { + var fileSpan = fileId.AsSpan(); + + if (!string.IsNullOrWhiteSpace(fileId) + && fileSpan.StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal)) return false; + + int schemeEnd = fileSpan.IndexOf(FileIdParser.SchemeSeparator.AsSpan()); + if (schemeEnd == -1) return false; + + var afterSpan = fileSpan[(schemeEnd + FileIdParser.SchemeSeparator.Length)..]; + int scopeEnd = afterSpan.IndexOf('/'); + if (scopeEnd == -1) return false; + + var scopeName = afterSpan[..scopeEnd]; + + return scopeName.Equals(_partName, StringComparison.Ordinal); + } public async Task UploadAsync( UploadFileInput metadata, @@ -89,7 +108,7 @@ await partManager.EnsureParts( cancellationToken); string fileId = FileIdParser.FormatToFileId( - StorageType, options.TableName, metadata.TenantId, createdAtDay, metadata.FileName); + StorageType, _partName, metadata.TenantId, createdAtDay, metadata.FileName); string fileExtension = FileIdParser.GetFileExtension(metadata.FileName); @@ -136,7 +155,7 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio { EnsureWritable(); - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out int tenantId, out long timestamp, out _)) + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; } @@ -160,7 +179,7 @@ public async Task DownloadAsync( Func loadStream, CancellationToken cancellationToken) { - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out int tenantId, out long timestamp, out _)) + if (!FileIdParser.TryParse(fileId, out _, out int tenantId, out long timestamp, out _)) { return false; } @@ -207,11 +226,13 @@ private static string Sanitize(ReadOnlySpan input) if (!CanProcess(fileId)) return Task.FromResult(null); - if (!FileIdParser.TryParseFileIdWithFilename(fileId, out var tenantId, out _, out var fileName)) + if (!FileIdParser.TryParse(fileId, out _, out var tenantId, out _, out var fileName)) return Task.FromResult(null); var metadata = new FileMetadata { + ScopeName = _partName, + StorageType = StorageType, FileName = fileName, TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs index d7146669..5fe90bf4 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs @@ -19,6 +19,12 @@ public PostgresFileStorageConfiguration(IServiceCollection services) _partConfiguration = services.AddSaPartitional((sp, builder) => { + var dataSource = sp.GetRequiredService(); + _options.StorageOptions.SchemaName = dataSource.GetSearchPath(); + _configure?.Invoke(sp, _options); + _options.StorageOptions.TableName = _options.StorageOptions.TableName.Trim('"'); + + builder.AddSchema(_options.StorageOptions.SchemaName, schema => { schema.AddTable(_options.StorageOptions.TableName, @@ -49,20 +55,19 @@ public PostgresFileStorageConfiguration(IServiceCollection services) services.AddSingleton(sp => { - _configure?.Invoke(sp, _options); - _configure = null; - - var pm = sp.GetRequiredService(); var dataSource = sp.GetRequiredService(); + var pm = sp.GetRequiredService(); var time = sp.GetService(); var sm = sp.GetRequiredService(); - StorageOptions options = _options.StorageOptions with - { - TableName = _options.StorageOptions.TableName.Trim('"') - }; + var storage = new PostgresFileStorage( + dataSource: dataSource, + partManager: pm, + streamManager: sm, + options: _options.StorageOptions, + scopeName: _options.ScopeName, + timeProvider: time); - var storage = new PostgresFileStorage(dataSource, pm, sm, options, _options.ScopeName, time); return storage; }); } @@ -91,19 +96,15 @@ public IPostgresFileStorageConfiguration AsReadOnly() return this; } - public IPostgresFileStorageConfiguration ConfigureOptions(Action configure) - { - configure?.Invoke(_options); - return this; - } - - public IPostgresFileStorageConfiguration ConfigureOptions(Action configure) + public IPostgresFileStorageConfiguration ConfigureOptions( + Action configure) { _configure = configure; return this; } - public IPostgresFileStorageConfiguration AddDataSource(Action? configure = null) + public IPostgresFileStorageConfiguration AddDataSource( + Action? configure = null) { _partConfiguration.AddDataSource(configure); return this; diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs index 23acbee8..a4834bbb 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs @@ -26,5 +26,5 @@ public sealed class PostgresFileStorageOptions public StorageOptions StorageOptions { get; } = new(); public PartOptions PartOptions { get; } = new(); public CleanupOptions CleanupOptions { get; } = new(); - public string ScopeName { get; set; } = string.Empty; + public string ScopeName { get; set; } = "share"; } diff --git a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj index e09e3e72..3108c13f 100644 --- a/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj +++ b/src/Sa.HybridFileStorage.Postgres/Sa.HybridFileStorage.Postgres.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management in Pg diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index c10a16af..17330289 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -38,7 +38,6 @@ private void EnsureWritable() public bool CanProcess(string? fileId) { if (string.IsNullOrEmpty(fileId)) return false; - // Ordinal быстрее OrdinalIgnoreCase, т.к. схема всегда в нижнем регистре return fileId.AsSpan().StartsWith(_schemePrefix.AsSpan(), StringComparison.Ordinal); } @@ -143,6 +142,8 @@ private static string GetFilePath(string fileId) return new FileMetadata { + ScopeName = ScopeName, + StorageType = StorageType, FileName = fileNameSpan.ToString(), TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj index 4bcd60a5..6b5c60a1 100644 --- a/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj +++ b/src/Sa.HybridFileStorage.S3/Sa.HybridFileStorage.S3.csproj @@ -1,23 +1,23 @@  - + - - 0.8.3 - File storage management in S3 - + + 0.9.0 + File storage management in S3 + - - - + + + - - - - + + + + - - - + + + diff --git a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs index 3f39f486..145ae034 100644 --- a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs +++ b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs @@ -34,7 +34,10 @@ public interface IFileStorage /// Stream of the file to upload. /// Cancellation token. /// The result of the file upload. - Task UploadAsync(UploadFileInput metadata, Stream fileStream, CancellationToken cancellationToken); + Task UploadAsync( + UploadFileInput metadata, + Stream fileStream, + CancellationToken cancellationToken); /// /// Downloads a file from the storage by its ID. diff --git a/src/Sa.HybridFileStorage/FileMetadata.cs b/src/Sa.HybridFileStorage/FileMetadata.cs index 37bcf909..b0290bf7 100644 --- a/src/Sa.HybridFileStorage/FileMetadata.cs +++ b/src/Sa.HybridFileStorage/FileMetadata.cs @@ -2,6 +2,8 @@ public sealed class FileMetadata { + public required string StorageType { get; init; } + public required string ScopeName { get; init; } public required string FileName { get; init; } public int TenantId { get; init; } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorage.cs b/src/Sa.HybridFileStorage/HybridFileStorage.cs index 69f86d84..f42e885a 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorage.cs @@ -48,14 +48,13 @@ public async Task UploadAsync( public async Task DownloadAsync( string fileId, - string scopeName, Func loadStream, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId, scopeName), + CanStorages(fileId), async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct), async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct), @@ -64,14 +63,14 @@ public async Task DownloadAsync( ); } - public async Task DeleteAsync(string fileId, string scopeName, CancellationToken cancellationToken = default) + public async Task DeleteAsync( + string fileId, + CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); - - EnsureWritable(scopeName); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId, scopeName), + CanStorages(fileId).Where(c => !c.IsReadOnly), async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct), async (storage, ct) => await storage.DeleteAsync(fileId, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct), @@ -117,12 +116,11 @@ private static async Task ExecuteStorageOperationAsync( public async Task GetMetadataAsync( string fileId, - string scopeName, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); - foreach (var fs in CanStorages(fileId, scopeName)) + foreach (var fs in CanStorages(fileId)) { var meta = await fs.GetMetadataAsync(fileId, cancellationToken); if (meta != null) return meta; @@ -132,6 +130,6 @@ private static async Task ExecuteStorageOperationAsync( } - internal IEnumerable CanStorages(string fileId, string scopeName) - => container.Storages.Where(c => c.ScopeName == scopeName && c.CanProcess(fileId)); + internal IEnumerable CanStorages(string fileId) + => container.Storages.Where(c => c.CanProcess(fileId)); } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index 1ee20ad5..5aadcb24 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -5,25 +5,48 @@ namespace Sa.HybridFileStorage; public static class HybridFileStorageExtensions { + + public static async Task CopyFromFileAsync( + this IHybridFileStorage storage, + string filePath, + UploadFileInput input, + string scopeName, + CancellationToken ct = default) + { + // копируем файл в хранилище + await using var fs = new FileStream(filePath, new FileStreamOptions + { + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + BufferSize = 4096, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + }); + + return await storage.UploadAsync( + input: input, + scopeName: scopeName, + fileStream: fs, + cancellationToken: ct); + } + public static async Task CopyToScopeAsync( this IHybridFileStorage storage, string fileId, - string sourceScopeName, string targetScopeName, int? targetTenantId = default, CancellationToken ct = default) { - var metadata = await storage.GetMetadataAsync(fileId, sourceScopeName, ct); + var metadata = await storage.GetMetadataAsync(fileId, ct); if (metadata is null) { ThrowSourceFileNotFound(fileId); } - int tenantId = targetTenantId ?? metadata.TenantId; if (tenantId == metadata.TenantId - && string.Equals(sourceScopeName, targetScopeName, StringComparison.Ordinal)) + && string.Equals(metadata.ScopeName, targetScopeName, StringComparison.Ordinal)) { ThrowFileAlreadyInTargetScope(); } @@ -37,7 +60,6 @@ public static async Task CopyToScopeAsync( StorageResult result = default!; bool downloaded = await storage.DownloadAsync( fileId, - sourceScopeName, async (sourceStream, downloadCt) => { result = await storage.UploadAsync( @@ -66,12 +88,9 @@ static void ThrowSourceFileNotFound(string fileId) => throw new FileNotFoundException($"Source file not found: {fileId}"); - - public static async Task> CopyToScopeBatchAsync( this IHybridFileStorage storage, IEnumerable fileIds, - string sourceScopeName, string targetScopeName, int? targetTenantId = default, BatchOptions? options = default, @@ -108,7 +127,7 @@ public static async Task> CopyToScopeBatchAsync( var ct = cts?.Token ?? cancellationToken; - return await storage.CopyToScopeAsync(fileId, sourceScopeName, targetScopeName, targetTenantId, ct); + return await storage.CopyToScopeAsync(fileId, targetScopeName, targetTenantId, ct); } catch (Exception ex) { diff --git a/src/Sa.HybridFileStorage/IHybridFileStorage.cs b/src/Sa.HybridFileStorage/IHybridFileStorage.cs index f01585dd..91179adf 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorage.cs @@ -20,7 +20,7 @@ public interface IHybridFileStorage /// The unique identifier for the file to be deleted. /// A cancellation token to cancel the operation if needed. /// True if the file was successfully deleted; otherwise, false. - Task DeleteAsync(string fileId, string scopeName, CancellationToken cancellationToken = default); + Task DeleteAsync(string fileId, CancellationToken cancellationToken = default); /// /// Downloads the file associated with the specified file ID asynchronously. @@ -31,7 +31,6 @@ public interface IHybridFileStorage /// True if the file was successfully downloaded; otherwise, false. Task DownloadAsync( string fileId, - string scopeName, Func loadStream, CancellationToken cancellationToken = default); @@ -51,6 +50,5 @@ Task UploadAsync( Task GetMetadataAsync( string fileId, - string scopeName, CancellationToken cancellationToken = default); } diff --git a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj index 67354ffd..c8630cf9 100644 --- a/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj +++ b/src/Sa.HybridFileStorage/Sa.HybridFileStorage.csproj @@ -3,7 +3,7 @@ - 0.8.3 + 0.9.0 File storage management diff --git a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs index 3585656b..fcb7fea0 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/ErrorDeliveryCommand.cs @@ -5,8 +5,9 @@ namespace Sa.Outbox.PostgreSql.Commands; -internal sealed class ErrorDeliveryCommand(IPgDataSource dataSource, SqlOutboxBuilder sqlTemplate) - : IErrorDeliveryCommand +internal sealed class ErrorDeliveryCommand( + IPgDataSource dataSource, + SqlOutboxBuilder sqlTemplate): IErrorDeliveryCommand { private readonly SqlCacheSplitter sqlCache = new(len => sqlTemplate.SqlError(len)); diff --git a/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs index d967cbe7..327ff756 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/InsertMsgTypeCommand.cs @@ -3,7 +3,9 @@ namespace Sa.Outbox.PostgreSql.Commands; -internal sealed class InsertMsgTypeCommand(IPgDataSource dataSource, SqlOutboxBuilder template) : IInsertMsgTypeCommand +internal sealed class InsertMsgTypeCommand( + IPgDataSource dataSource, + SqlOutboxBuilder template) : IInsertMsgTypeCommand { public Task Execute(long id, string typeName, CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs index 4b93c4dc..97703cc4 100644 --- a/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs +++ b/src/Sa.Outbox.PostgreSql/Commands/SelectTenantCommand.cs @@ -1,4 +1,5 @@ -using Sa.Data.PostgreSql; +using Npgsql; +using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.SqlBuilder; namespace Sa.Outbox.PostgreSql.Commands; @@ -11,8 +12,15 @@ NpqsqlOutboxReader outboxReader { public async Task> Execute(CancellationToken cancellationToken) { - return await dataSource.ExecuteReaderList(sql.SqlSelectTetant, - reader => outboxReader.Message.GetTenantId(reader), - cancellationToken); + try + { + return await dataSource.ExecuteReaderList(sql.SqlSelectTetant, + reader => outboxReader.Message.GetTenantId(reader), + cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.UndefinedTable) + { + return []; + } } } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs index 9735dcea..f7f6796c 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs @@ -5,8 +5,10 @@ namespace Sa.Outbox.PostgreSql.Configuration; public interface IPgOutboxConfiguration { - IPgOutboxConfiguration WithMessageSerializer(Func messageSerializerFactory); - IPgOutboxConfiguration WithMessageSerializer(TService instance) where TService : class, IOutboxMessageSerializer; + IPgOutboxConfiguration WithMessageSerializer( + Func messageSerializerFactory); + IPgOutboxConfiguration WithMessageSerializer(TService instance) + where TService : class, IOutboxMessageSerializer; IPgOutboxConfiguration WithOutboxSettings(Action? configure = null); IPgOutboxConfiguration WithDataSource(Action? configure = null); } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs index 7f583cbb..26831f11 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs @@ -25,7 +25,8 @@ public IPgOutboxConfiguration WithDataSource(Action messageSerializerFactory) + public IPgOutboxConfiguration WithMessageSerializer( + Func messageSerializerFactory) { services.RemoveAll(); services.TryAddSingleton(messageSerializerFactory); @@ -52,6 +53,13 @@ private void RegisterOutboxSettings() { PgOutboxSettings settings = new(); + var connectionSchema = sp.GetService()?.GetSearchPath(); + + if (!string.IsNullOrWhiteSpace(connectionSchema)) + { + settings.TableSettings.WithSchema(connectionSchema); + } + var configureActions = sp.GetServices>(); foreach (var configureAction in configureActions) diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs index 40c85a8d..df4ac86a 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxTableSettingsExtensions.cs @@ -72,8 +72,8 @@ public static PgOutboxTableSettings UseBaseTableName( /// public static PgOutboxTableSettings UseBaseTableName( this PgOutboxTableSettings settings, - string baseTableName, - string schemaName) + string schemaName, + string baseTableName) { if (string.IsNullOrWhiteSpace(schemaName)) throw new ArgumentException("Schema name cannot be null or empty", nameof(schemaName)); diff --git a/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs b/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs index 1402f854..89153283 100644 --- a/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs +++ b/src/Sa.Outbox.PostgreSql/Partitional/MsgMigrationSupport.cs @@ -4,7 +4,8 @@ namespace Sa.Outbox.PostgreSql.Partitional; -internal sealed class MsgMigrationSupport(IOutboxPartitionalSupport? partitionalSupport = null) : IPartTableMigrationSupport +internal sealed class MsgMigrationSupport(IOutboxPartitionalSupport? partitionalSupport = null) + : IPartTableMigrationSupport { public async Task GetParts(CancellationToken cancellationToken) { diff --git a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj index c592b68c..cd3c74a6 100644 --- a/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj +++ b/src/Sa.Outbox.PostgreSql/Sa.Outbox.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple Outbox for Pg (publishing and using messages) diff --git a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs index 228b520f..3f069bf0 100644 --- a/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs +++ b/src/Sa.Outbox.PostgreSql/Services/IOutboxTaskLoader.cs @@ -11,5 +11,6 @@ public sealed record LoadGroupResult(int CopiedRows, Guid NewOffset) internal interface IOutboxTaskLoader { - Task LoadNewTasks(OutboxMessageFilter filter, int batchSize, CancellationToken cancellationToken = default); + Task LoadNewTasks( + OutboxMessageFilter filter, int batchSize, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs b/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs index 62c3e8fe..985a9e1d 100644 --- a/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs +++ b/src/Sa.Outbox.PostgreSql/Services/OutboxPartRepository.cs @@ -4,8 +4,9 @@ namespace Sa.Outbox.PostgreSql.Services; -internal sealed class OutboxPartRepository(IPartitionManager partManager, PgOutboxTableSettings tableSettings) - : IOutboxPartRepository +internal sealed class OutboxPartRepository( + IPartitionManager partManager, + PgOutboxTableSettings tableSettings): IOutboxPartRepository { public Task EnsureMsgParts(IEnumerable outboxParts, CancellationToken cancellationToken) => EnsureParts(tableSettings.Message.TableName, outboxParts, cancellationToken); @@ -32,13 +33,20 @@ public async Task EnsureErrorParts(IEnumerable dates, Cance public Task Migrate() => partManager.Migrate(CancellationToken.None); - private async Task EnsureParts(string databaseTableName, IEnumerable outboxParts, CancellationToken cancellationToken) + private async Task EnsureParts( + string databaseTableName, + IEnumerable outboxParts, + CancellationToken cancellationToken) { int i = 0; foreach (OutboxPartInfo part in outboxParts.Distinct()) { i++; - await partManager.EnsureParts(databaseTableName, part.CreatedAt, [part.TenantId, part.Part], cancellationToken); + await partManager.EnsureParts( + databaseTableName, + part.CreatedAt, + [part.TenantId, part.Part], + cancellationToken); } return i; diff --git a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs index d1e491e9..5278fdde 100644 --- a/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs +++ b/src/Sa.Outbox.PostgreSql/SqlBuilder/SqlOutboxBuilder.cs @@ -8,7 +8,9 @@ namespace Sa.Outbox.PostgreSql.SqlBuilder; /// /// Provides SQL query templates for working with PostgreSQL outbox tables. /// -internal sealed class SqlOutboxBuilder(PgOutboxTableSettings settings, ObjectPool objectPool) +internal sealed class SqlOutboxBuilder( + PgOutboxTableSettings settings, + ObjectPool objectPool) { internal PgOutboxTableSettings Settings => settings; diff --git a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs index 2a2a118d..a5a66f85 100644 --- a/src/Sa.Outbox/Delivery/DeliveryBuilder.cs +++ b/src/Sa.Outbox/Delivery/DeliveryBuilder.cs @@ -8,7 +8,8 @@ namespace Sa.Outbox.Delivery; internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDeliveryBuilder { - public IDeliveryBuilder AddDeliveryScoped<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + public IDeliveryBuilder AddDeliveryScoped< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, Action? configure = null ) @@ -19,7 +20,8 @@ internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDe return this; } - public IDeliveryBuilder AddDelivery<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( + public IDeliveryBuilder AddDelivery< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( string consumerGroupId, Action? configure = null ) @@ -30,7 +32,8 @@ internal sealed partial class DeliveryBuilder(IServiceCollection services) : IDe return this; } - public IDeliveryBuilder AddDeliveryBatching<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() + public IDeliveryBuilder AddDeliveryBatching< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() where TImplementation : class, IDeliveryBatcher { services diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index c9566b2b..b9921c30 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -10,11 +10,14 @@ namespace Sa.Outbox.Delivery; /// internal sealed class DeliveryTenant( IOutboxDeliveryManager deliveryMan, - TimeProvider timeProvider, IDeliveryCourier deliveryCourier, IDeliveryBatcher batcher, - FilterFactory filterFactory) : IDeliveryTenant + FilterFactory filterFactory, + TimeProvider? timeProvider = null) : IDeliveryTenant { + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + public async Task ProcessInTenant( int tenantId, ConsumerGroupSettings settings, @@ -50,11 +53,13 @@ private OutboxMessageFilter CreateFilter(int tenantId, ConsumerGroupSe return filterFactory.CreateFilter( tenantId: tenantId, consumerGroupId: settings.ConsumerGroupId, - now: timeProvider.GetUtcNow(), + now: GetUtcNow(), lookbackInterval: settings.ConsumeSettings.LookbackInterval, batchingWindow: settings.ConsumeSettings.BatchingWindow); } + private DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); + private static IMemoryOwner> RentMemory(int size) { return MemoryPool>.Shared.Rent(size); @@ -96,7 +101,7 @@ private Task ReleaseMessagesAsync( { return deliveryMan.ReturnDelivery( messages, - filter with { NowDate = timeProvider.GetUtcNow() }, + filter with { NowDate = GetUtcNow() }, cancellationToken); } @@ -108,7 +113,7 @@ private IDisposable RenewerLocker( settings.LockRenewal , t => { - var nowDate = timeProvider.GetUtcNow(); + var nowDate = GetUtcNow(); return deliveryMan.ExtendDelivery( settings.LockDuration , filter with diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index 43bd3ef4..34e69834 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -8,11 +8,11 @@ internal static class Setup { public static IServiceCollection AddDeliveryJob< [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer, TMessage>( - this IServiceCollection services, - string consumerGroupId, - bool isSingleton, - Action? сonfigure = null) - where TConsumer : class, IConsumer + this IServiceCollection services, + string consumerGroupId, + bool isSingleton, + Action? сonfigure = null) + where TConsumer : class, IConsumer { ArgumentNullException.ThrowIfNullOrWhiteSpace(consumerGroupId); diff --git a/src/Sa.Outbox/Delivery/OutboxContext.cs b/src/Sa.Outbox/Delivery/OutboxContext.cs index a04ba001..24a32a5a 100644 --- a/src/Sa.Outbox/Delivery/OutboxContext.cs +++ b/src/Sa.Outbox/Delivery/OutboxContext.cs @@ -6,12 +6,14 @@ namespace Sa.Outbox.Delivery; [DebuggerDisplay("#{PayloadId}")] -/// -/// OutboxMessage -/// -internal sealed class OutboxContext(OutboxDeliveryMessage delivery, TimeProvider timeProvider) +internal sealed class OutboxContext( + OutboxDeliveryMessage delivery, + TimeProvider? timeProvider = null) : IOutboxContextOperations { + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + public Guid OutboxId => delivery.OutboxId; public string PayloadId => delivery.Message.PayloadId; public TMessage Payload => delivery.Message.Payload; @@ -133,7 +135,7 @@ private void ErrorWithCode(DeliveryStatusCode errorCode, Exception exception, st null); } - public DateTimeOffset GetUtcNow() => (timeProvider ?? TimeProvider.System).GetUtcNow(); + public DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); private readonly static DeliveryPermanentException DeliveryPermanentException diff --git a/src/Sa.Outbox/Delivery/OutboxContextFactory.cs b/src/Sa.Outbox/Delivery/OutboxContextFactory.cs index 9b495d5c..811056e9 100644 --- a/src/Sa.Outbox/Delivery/OutboxContextFactory.cs +++ b/src/Sa.Outbox/Delivery/OutboxContextFactory.cs @@ -1,6 +1,6 @@ namespace Sa.Outbox.Delivery; -internal sealed class OutboxContextFactory(TimeProvider? timeProvider) : IOutboxContextFactory +internal sealed class OutboxContextFactory(TimeProvider? timeProvider = null) : IOutboxContextFactory { public IOutboxContextOperations Create(OutboxDeliveryMessage deliveryMessage) { diff --git a/src/Sa.Outbox/Delivery/Setup.cs b/src/Sa.Outbox/Delivery/Setup.cs index 40f57447..0c3eb0a7 100644 --- a/src/Sa.Outbox/Delivery/Setup.cs +++ b/src/Sa.Outbox/Delivery/Setup.cs @@ -7,7 +7,8 @@ namespace Sa.Outbox.Delivery; internal static class Setup { - public static IServiceCollection AddOutboxDelivery(this IServiceCollection services, Action configure) + public static IServiceCollection AddOutboxDelivery( + this IServiceCollection services, Action? configure = null) { services.AddMessagesMetadata(); @@ -29,7 +30,7 @@ public static IServiceCollection AddOutboxDelivery(this IServiceCollection servi services.TryAddSingleton(); - configure.Invoke(new DeliveryBuilder(services)); + configure?.Invoke(new DeliveryBuilder(services)); services.TryAddSingleton(); diff --git a/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs b/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs index 8278ec60..364f2443 100644 --- a/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs +++ b/src/Sa.Outbox/Exceptions/DeliveryPermanentException.cs @@ -2,7 +2,9 @@ namespace Sa.Outbox.Exceptions; -public class DeliveryPermanentException(string message, Exception? innerException = null, DeliveryStatusCode statusCode = DeliveryStatusCode.Error) - : DeliveryException(message, innerException, statusCode) +public class DeliveryPermanentException( + string message, + Exception? innerException = null, + DeliveryStatusCode statusCode = DeliveryStatusCode.Error) : DeliveryException(message, innerException, statusCode) { } diff --git a/src/Sa.Outbox/IOutboxBuilder.cs b/src/Sa.Outbox/IOutboxBuilder.cs index 9f426713..53f79122 100644 --- a/src/Sa.Outbox/IOutboxBuilder.cs +++ b/src/Sa.Outbox/IOutboxBuilder.cs @@ -1,5 +1,6 @@ using Sa.Outbox.Delivery; using Sa.Outbox.Metadata; +using Sa.Outbox.Partitional; using Sa.Outbox.Publication; using System.Diagnostics.CodeAnalysis; @@ -34,4 +35,19 @@ public interface IOutboxBuilder IOutboxBuilder WithMetadata(Action configure); + + /// + /// shortcut + /// + IOutboxBuilder AddMetadata( + string partName, + Func? getPayloadId = null) where TMessage : class + { + return WithMetadata((_, m) => m.AddMetadata(partName, getPayloadId)); + } + + IOutboxBuilder AddMetadata() where TMessage : class, IOutboxPublishable + { + return WithMetadata((_, m) => m.AddMetadata()); + } } diff --git a/src/Sa.Outbox/IOutboxPublishable.cs b/src/Sa.Outbox/IOutboxPublishable.cs new file mode 100644 index 00000000..8d4d5c7c --- /dev/null +++ b/src/Sa.Outbox/IOutboxPublishable.cs @@ -0,0 +1,14 @@ +namespace Sa.Outbox; + +public interface IOutboxPublishable +{ + string GetPayloadId(); + + int GetTenantId(); + /// + /// Gets the logical identifier of the partition associated with this type. + /// Used for routing messages to specific tables, shards, or storage groups (e.g., outbox_orders, outbox_eu). + /// + /// "orders", "notifications", "us_west" + static abstract string PartName { get; } +} diff --git a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs index 33ee9c7b..d47a751d 100644 --- a/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs +++ b/src/Sa.Outbox/Metadata/IOutboxMessageMetadataBuilder.cs @@ -2,6 +2,12 @@ public interface IOutboxMessageMetadataBuilder { - IOutboxMessageMetadataBuilder AddMetadata(string partName, Func? getPayloadId = null) - where TMessage : class; + IOutboxMessageMetadataBuilder AddMetadata( + string partName, + Func? getPayloadId = null) where TMessage : class; + + IOutboxMessageMetadataBuilder AddMetadata() where TMessage : class, IOutboxPublishable + { + return AddMetadata(TMessage.PartName, m => m.GetPayloadId()); + } } diff --git a/src/Sa.Outbox/Metadata/MetadataConfiguration.cs b/src/Sa.Outbox/Metadata/MetadataConfiguration.cs index da1186ea..b16a0ef4 100644 --- a/src/Sa.Outbox/Metadata/MetadataConfiguration.cs +++ b/src/Sa.Outbox/Metadata/MetadataConfiguration.cs @@ -6,8 +6,6 @@ internal sealed class MetadataConfiguration : IOutboxMessageMetadataBuilder, IOu private static readonly Func s_Dummy = _ => string.Empty; - private static readonly OutboxMessageMetadata s_Default = new("root", s_Dummy); - public IOutboxMessageMetadataBuilder AddMetadata(string partName, Func? getPayloadId = null) where T : class @@ -40,7 +38,7 @@ public OutboxMessageMetadata GetMetadata(Type messageType) return metadata; } - return s_Default; + return OutboxMessageMetadata.Empty; } diff --git a/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs b/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs index 855193e8..a80bc77c 100644 --- a/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs +++ b/src/Sa.Outbox/Metadata/OutboxMessageMetadata.cs @@ -3,4 +3,10 @@ internal sealed record OutboxMessageMetadata( string PartName, - Func GetPayloadId); + Func GetPayloadId) +{ + public static readonly OutboxMessageMetadata Empty = new("root", m => + { + return (m is IOutboxPublishable msg) ? msg.GetPayloadId() : string.Empty; + }); +} diff --git a/src/Sa.Outbox/Metadata/Setup.cs b/src/Sa.Outbox/Metadata/Setup.cs index 090be7ca..0c54cd08 100644 --- a/src/Sa.Outbox/Metadata/Setup.cs +++ b/src/Sa.Outbox/Metadata/Setup.cs @@ -10,12 +10,16 @@ public static IServiceCollection AddMessagesMetadata( Action? configure = null) { - services.AddSingleton(sp => + if (configure != null) { - var configuration = new MetadataConfiguration(); - configure?.Invoke(sp, configuration); - return configuration; - }); + // multiple configuration + services.AddSingleton(sp => + { + var configuration = new MetadataConfiguration(); + configure.Invoke(sp, configuration); + return configuration; + }); + } services.TryAddSingleton(sp => diff --git a/src/Sa.Outbox/OutboxBuilder.cs b/src/Sa.Outbox/OutboxBuilder.cs index 92f12074..8a53ee09 100644 --- a/src/Sa.Outbox/OutboxBuilder.cs +++ b/src/Sa.Outbox/OutboxBuilder.cs @@ -12,14 +12,14 @@ internal sealed class OutboxBuilder : IOutboxBuilder { private readonly IServiceCollection _services; - private OutboxBuilder(IServiceCollection services) - { - _services = services; - } + private OutboxBuilder(IServiceCollection services) => _services = services; public static OutboxBuilder Create(IServiceCollection services) { - services.AddMessagePublisher(); + services + .AddMessagePublisher() + .AddOutboxDelivery(); + return new OutboxBuilder(services); } @@ -43,12 +43,13 @@ public IOutboxBuilder WithDeliveries(Action build) public IOutboxBuilder WithTenants(Action configure) { - _services.AddTenantProvider(configure); + _services.AddTenantSettings(configure); return this; } - public IOutboxBuilder WithDeliveryBatcher<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() - where TImplementation : class, IDeliveryBatcher + public IOutboxBuilder WithDeliveryBatcher< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TImplementation>() + where TImplementation : class, IDeliveryBatcher { _services .RemoveAll() diff --git a/src/Sa.Outbox/Partitional/ITenantSource.cs b/src/Sa.Outbox/Partitional/ITenantSource.cs index 7f23f04f..5ab829bf 100644 --- a/src/Sa.Outbox/Partitional/ITenantSource.cs +++ b/src/Sa.Outbox/Partitional/ITenantSource.cs @@ -1,5 +1,4 @@ - -namespace Sa.Outbox; +namespace Sa.Outbox.Partitional; /// /// Tenant provider for partitional support. diff --git a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs index 47c1a032..3e76da12 100644 --- a/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs +++ b/src/Sa.Outbox/Partitional/OutboxPartitionalSupport.cs @@ -24,7 +24,8 @@ public async ValueTask GetTenantIds(CancellationToken cancellationToken) return await tenantProvider.GetTenantIds(cancellationToken); } - private async Task> GetPairs(IEnumerable parts, CancellationToken cancellationToken) + private async Task> GetPairs( + IEnumerable parts, CancellationToken cancellationToken) { int[] tenantIds = await GetTenantIds(cancellationToken); if (tenantIds.Length == 0) return []; diff --git a/src/Sa.Outbox/Partitional/Setup.cs b/src/Sa.Outbox/Partitional/Setup.cs index 22c0e151..a4d4fba7 100644 --- a/src/Sa.Outbox/Partitional/Setup.cs +++ b/src/Sa.Outbox/Partitional/Setup.cs @@ -8,15 +8,17 @@ internal static class Setup { public static IServiceCollection AddOutboxPartitional(this IServiceCollection services) { + services.TryAddSingleton(); + // support - messaging to each tenant services.TryAddSingleton(); + services.TryAddSingleton(); return services; } - public static IServiceCollection AddTenantProvider(this IServiceCollection services, Action configure) + public static IServiceCollection AddTenantSettings( + this IServiceCollection services, Action configure) { // support - messaging to each tenant - services.TryAddSingleton(); - services .RemoveAll() .AddSingleton(sp => diff --git a/src/Sa.Outbox/Partitional/TenantSettings.cs b/src/Sa.Outbox/Partitional/TenantSettings.cs index a07d99e6..4398201d 100644 --- a/src/Sa.Outbox/Partitional/TenantSettings.cs +++ b/src/Sa.Outbox/Partitional/TenantSettings.cs @@ -1,4 +1,4 @@ -namespace Sa.Outbox; +namespace Sa.Outbox.Partitional; /// /// Represents the settings for partitioning in the Outbox processing system. @@ -10,7 +10,7 @@ public sealed class TenantSettings /// Gets or sets a value indicating whether the system should automatically detect tenants /// by scanning incoming database messages /// - public bool AutoDetect { get; private set; } = false; + public bool AutoDetect { get; private set; } = true; /// /// Gets or sets a function that retrieves tenant IDs asynchronously. @@ -56,4 +56,10 @@ public TenantSettings WithAutoDetect() AutoDetect = true; return this; } + + public TenantSettings WithNoAutoDetect() + { + AutoDetect = false; + return this; + } } diff --git a/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs b/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs index 5c69f11e..c6f9845f 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxPlugin.cs @@ -1,15 +1,15 @@ namespace Sa.Outbox.PlugServices; -public interface IOutboxPlugin : IAsyncDisposable -{ - string Name { get; } - string Version { get; } - string Provider { get; } // "Postgres", "SqlServer", "Redis", etc. +//public interface IOutboxPlugin : IAsyncDisposable +//{ +// string Name { get; } +// string Version { get; } +// string Provider { get; } // "Postgres", "SqlServer", "Redis", etc. - IOutboxBulkWriter BulkWriter { get; } - IOutboxDeliveryManager DeliveryManager { get; } - IOutboxTenantDetector TenantDetector { get; } +// IOutboxBulkWriter BulkWriter { get; } +// IOutboxDeliveryManager DeliveryManager { get; } +// IOutboxTenantDetector TenantDetector { get; } - ValueTask InitializeAsync(CancellationToken cancellationToken = default); - ValueTask HealthCheckAsync(CancellationToken cancellationToken = default); -} +// ValueTask InitializeAsync(CancellationToken cancellationToken = default); +// ValueTask HealthCheckAsync(CancellationToken cancellationToken = default); +//} diff --git a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs index 702a3852..78f26aa1 100644 --- a/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs +++ b/src/Sa.Outbox/PlugServices/IOutboxTenantDetector.cs @@ -1,4 +1,6 @@ -namespace Sa.Outbox.PlugServices; +using Sa.Outbox.Partitional; + +namespace Sa.Outbox.PlugServices; /// /// Discovers tenant IDs at runtime from system data (e.g., message queues, inbox tables). diff --git a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs index e081aa9d..6fea2c0e 100644 --- a/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/IOutboxMessagePublisher.cs @@ -15,7 +15,7 @@ public interface IOutboxMessagePublisher /// with the number of successfully published messages as the result. ValueTask Publish( IReadOnlyCollection messages, - int tenantId = 0, + int tenantId, CancellationToken cancellationToken = default); @@ -27,27 +27,44 @@ async ValueTask Publish( Func getTenantId, CancellationToken cancellationToken = default) { - - var lookup = messages.ToLookup(getTenantId); ulong totals = 0; - - foreach (var tenantId in lookup.Select(g => g.Key)) + foreach (IGrouping group in messages.ToLookup(getTenantId)) { - var group = lookup[tenantId]; - var tenantMessages = group.ToArray(); + totals += await Publish([.. group], group.Key, cancellationToken); + } + + return totals; + } + - totals += await Publish(tenantMessages, tenantId, cancellationToken); + async ValueTask Publish( + IReadOnlyCollection messages, + CancellationToken cancellationToken = default) + where TMessage : class, IOutboxPublishable + { + ulong totals = 0; + foreach (var group in messages.ToLookup(m => m.GetTenantId())) + { + totals += await Publish([.. group], group.Key, cancellationToken); } return totals; } + /// /// Publishes a single message. /// ValueTask PublishSingle( TMessage message, - int tenantId = 0, + int tenantId, CancellationToken cancellationToken = default) => Publish([message], tenantId, cancellationToken); + + + ValueTask PublishSingle( + TMessage message, + CancellationToken cancellationToken = default) + where TMessage : class, IOutboxPublishable + => Publish([message], message.GetTenantId(), cancellationToken); } diff --git a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs index 007d9e14..451d441e 100644 --- a/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs +++ b/src/Sa.Outbox/Publication/OutboxMessagePublisher.cs @@ -8,8 +8,7 @@ internal sealed class OutboxMessagePublisher( TimeProvider timeProvider, IOutboxBulkWriter bulkWriter, OutboxPublishSettings publishSettings, - IOutboxMessageMetadataProvider metadataProvider -) : IOutboxMessagePublisher + IOutboxMessageMetadataProvider metadataProvider) : IOutboxMessagePublisher { public async ValueTask Publish( IReadOnlyCollection messages, @@ -25,7 +24,8 @@ private async ValueTask Send( int tenantId, CancellationToken cancellationToken) { - var typeInfo = metadataProvider.GetMetadata(); + OutboxMessageMetadata typeInfo = metadataProvider.GetMetadata(); + DateTimeOffset now = timeProvider.GetUtcNow(); int maxBatchSize = publishSettings.MaxBatchSize; @@ -41,6 +41,7 @@ private async ValueTask Send( OutboxMessage[] payloads = DefaultArrayPool.Shared.Rent>(len); Span> payloadsSpan = payloads; + try { int count = 0; diff --git a/src/Sa.Outbox/Sa.Outbox.csproj b/src/Sa.Outbox/Sa.Outbox.csproj index 69bac808..75f4ba30 100644 --- a/src/Sa.Outbox/Sa.Outbox.csproj +++ b/src/Sa.Outbox/Sa.Outbox.csproj @@ -3,7 +3,7 @@ - 0.8.1 + 0.9.0 Simple Outbox infra for publishing and using messages diff --git a/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs index a2f140d6..200c1a23 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/IPartCache.cs @@ -4,7 +4,15 @@ namespace Sa.Partitional.PostgreSql.Cache; internal interface IPartCache { - Task InCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); - Task EnsureCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task InCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); + Task EnsureCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); Task RemoveCache(string tableName, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs index 3264c1c0..cdd46239 100644 --- a/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs +++ b/src/Sa.Partitional.PostgreSql/Cache/PartCache.cs @@ -1,7 +1,7 @@ -using System.Collections.Concurrent; -using Sa.Classes; +using Sa.Classes; using Sa.Extensions; using Sa.Partitional.PostgreSql.Classes; +using System.Collections.Concurrent; namespace Sa.Partitional.PostgreSql.Cache; @@ -15,7 +15,11 @@ IPartRepository repository private readonly ConcurrentDictionary>> _cache = new(); - public async Task InCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public async Task InCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) { if (sqlBuilder[tableName] == null) return false; @@ -46,7 +50,11 @@ private async Task> SelectPartsInDb(string tableName, Canc } } - public async Task EnsureCache(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public async Task EnsureCache( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) { bool result = await InCache(tableName, date, partValues, cancellationToken); if (result) return true; diff --git a/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs b/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs index b2273341..3cdfab0a 100644 --- a/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs +++ b/src/Sa.Partitional.PostgreSql/Cleaning/Setup.cs @@ -8,7 +8,9 @@ internal static class Setup { readonly static Guid JobId = Guid.Parse("7da81411-9db7-4553-8e93-bd1f12d02b38"); - public static IServiceCollection AddPartCleaning(this IServiceCollection services, Action? configure = null) + public static IServiceCollection AddPartCleaning( + this IServiceCollection services, + Action? configure = null) { if (configure != null) diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs index 50c91c11..cdf86766 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/ISettingsBuilder.cs @@ -3,6 +3,8 @@ public interface ISettingsBuilder { + string DefaultSchema { get; } + ISettingsBuilder AddSchema(Action schemaBuilder); ISettingsBuilder AddSchema(string schemaName, Action schemaBuilder); ITableSettingsStorage Build(); diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs index 1b39f36a..06764566 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/SettingsBuilder.cs @@ -6,9 +6,11 @@ internal sealed class SettingsBuilder(string? searchPath) : ISettingsBuilder { private readonly Dictionary _schemas = []; + public string DefaultSchema { get; } = searchPath ?? "public"; + public ISettingsBuilder AddSchema(Action schemaBuilder) { - return AddSchema(searchPath ?? "public", schemaBuilder); + return AddSchema(DefaultSchema, schemaBuilder); } public ISettingsBuilder AddSchema(string schemaName, Action schemaBuilder) diff --git a/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs b/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs index 7f6171ef..bc4c4cc0 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/Builder/Setup.cs @@ -1,20 +1,20 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Npgsql; using Sa.Data.PostgreSql; namespace Sa.Partitional.PostgreSql.Configuration.Builder; internal static class Setup { - public static IServiceCollection AddSettigs(this IServiceCollection services, Action build) + public static IServiceCollection AddSettings( + this IServiceCollection services, + Action build) { services.AddSingleton(build); services.TryAddSingleton(sp => { - string? searchPath = GetSearchPath(sp); - var builder = new SettingsBuilder(searchPath); + SettingsBuilder builder = new(GetDefaultSchema(sp)); var configurators = sp.GetServices>(); @@ -29,19 +29,11 @@ public static IServiceCollection AddSettigs(this IServiceCollection services, Ac return services; } - private static string? GetSearchPath(IServiceProvider sp) + private static string? GetDefaultSchema(IServiceProvider sp) { - var settings = sp.GetService(); - var connectionString = settings?.ConnectionString; - - if (string.IsNullOrWhiteSpace(connectionString)) - { - return null; - } - try { - return new NpgsqlConnectionStringBuilder(connectionString).SearchPath; + return sp.GetService()?.GetSearchPath(); } catch { diff --git a/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs b/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs index a06a2274..d48cda2b 100644 --- a/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs +++ b/src/Sa.Partitional.PostgreSql/Configuration/PartConfiguration.cs @@ -15,7 +15,7 @@ internal sealed class PartConfiguration(IServiceCollection services) : IPartConf public IPartConfiguration AddPartTables(Action configure) { services - .AddSettigs(configure) + .AddSettings(configure) .AddSqlBuilder() ; diff --git a/src/Sa.Partitional.PostgreSql/IPartitionManager.cs b/src/Sa.Partitional.PostgreSql/IPartitionManager.cs index 62d78001..f16d9c96 100644 --- a/src/Sa.Partitional.PostgreSql/IPartitionManager.cs +++ b/src/Sa.Partitional.PostgreSql/IPartitionManager.cs @@ -1,4 +1,6 @@ -namespace Sa.Partitional.PostgreSql; +using Sa.Partitional.PostgreSql.Classes; + +namespace Sa.Partitional.PostgreSql; /// /// interface for managing partitions in the database @@ -31,6 +33,10 @@ public interface IPartitionManager /// An array of values that define the partitions (could be strings or numbers). /// A token to monitor for cancellation requests. /// A value task representing the asynchronous operation, with a boolean result indicating whether the partitions were ensured successfully. - Task EnsureParts(string tableName, DateTimeOffset date, Classes.StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task EnsureParts( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs index 55cf4852..6dfc3f5b 100644 --- a/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs +++ b/src/Sa.Partitional.PostgreSql/Migration/PartMigrationService.cs @@ -5,9 +5,7 @@ namespace Sa.Partitional.PostgreSql.Migration; internal sealed class PartMigrationService( IPartRepository repository , TimeProvider timeProvider - , MigrationScheduleSettings settings -) - : IMigrationService, IDisposable + , MigrationScheduleSettings settings): IMigrationService, IDisposable { private int s_triggered = 0; private readonly CancellationTokenSource _cts = new(); diff --git a/src/Sa.Partitional.PostgreSql/Part.cs b/src/Sa.Partitional.PostgreSql/Part.cs index d228d066..41ab0b5c 100644 --- a/src/Sa.Partitional.PostgreSql/Part.cs +++ b/src/Sa.Partitional.PostgreSql/Part.cs @@ -2,8 +2,7 @@ namespace Sa.Partitional.PostgreSql; -public sealed record Part(string Name, PartByRange PartBy) - : Enumeration(Name.GetHashCode(), Name) +public sealed record Part(string Name, PartByRange PartBy): Enumeration(Name.GetHashCode(), Name) { public const string RootId = "root"; diff --git a/src/Sa.Partitional.PostgreSql/PartitionManager.cs b/src/Sa.Partitional.PostgreSql/PartitionManager.cs index 127573dd..28e04401 100644 --- a/src/Sa.Partitional.PostgreSql/PartitionManager.cs +++ b/src/Sa.Partitional.PostgreSql/PartitionManager.cs @@ -6,7 +6,11 @@ namespace Sa.Partitional.PostgreSql; internal sealed class PartitionManager(IPartCache cache, IMigrationService migrationService) : IPartitionManager { - public Task EnsureParts(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default) + public Task EnsureParts( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default) => cache.EnsureCache(tableName, date, partValues, cancellationToken); public Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default) diff --git a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs index 47d59428..3cf77b55 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/IPartRepository.cs @@ -10,7 +10,12 @@ namespace Sa.Partitional.PostgreSql; /// An array of values that define the partitioning criteria, which can be either string or numeric. /// The method used for partitioning (e.g., by range, list, etc.). /// The date from which this partition is valid. -public sealed record PartByRangeInfo(string Id, string RootTableName, StrOrNum[] PartValues, PgPartBy PartBy, DateTimeOffset FromDate); +public sealed record PartByRangeInfo( + string Id, + string RootTableName, + StrOrNum[] PartValues, + PgPartBy PartBy, + DateTimeOffset FromDate); /// /// Represents a repository interface for managing database partitions. @@ -18,10 +23,23 @@ public sealed record PartByRangeInfo(string Id, string RootTableName, StrOrNum[] /// public interface IPartRepository { - Task CreatePart(string tableName, DateTimeOffset date, StrOrNum[] partValues, CancellationToken cancellationToken = default); + Task CreatePart( + string tableName, + DateTimeOffset date, + StrOrNum[] partValues, + CancellationToken cancellationToken = default); Task Migrate(DateTimeOffset[] dates, CancellationToken cancellationToken = default); - Task Migrate(DateTimeOffset[] dates, Func> resolve, CancellationToken cancellationToken = default); - Task> GetPartsFromDate(string tableName, DateTimeOffset fromDate, CancellationToken cancellationToken = default); - Task> GetPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); + Task Migrate( + DateTimeOffset[] dates, + Func> resolve, + CancellationToken cancellationToken = default); + Task> GetPartsFromDate( + string tableName, + DateTimeOffset fromDate, + CancellationToken cancellationToken = default); + Task> GetPartsToDate( + string tableName, + DateTimeOffset toDate, + CancellationToken cancellationToken = default); Task DropPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default); } diff --git a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs index 88ef8a12..7ab2e15f 100644 --- a/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs +++ b/src/Sa.Partitional.PostgreSql/Partitional/PartRepository.cs @@ -39,9 +39,7 @@ public async Task CreatePart( StrOrNum[] partValues, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(partValues); - - ISqlTableBuilder builder = sqlBuilder[tableName] ?? throw new KeyNotFoundException(nameof(tableName)); + ISqlTableBuilder builder = sqlBuilder[tableName] ?? throw new KeyNotFoundException(tableName); string sql = builder.CreateSql(date, partValues); return await ExecuteDDL(sql, cancellationToken); @@ -85,7 +83,8 @@ public async Task Migrate(DateTimeOffset[] dates, CancellationToken cancell { if (tableSettings.PartByListFieldNames.Length > 0) { - throw new InvalidOperationException($"Migration support is required for table '{table}' because 'PartByListFieldNames' is specified."); + throw new InvalidOperationException( + $"Migration support is required for table '{table}' because 'PartByListFieldNames' is specified."); } } } @@ -108,7 +107,8 @@ public async Task> GetPartsFromDate( return await GetPartsFormDateWithRetry(sql, unixTime, cancellationToken); } - private async Task> GetPartsFormDateWithRetry(string sql, long unixTime, CancellationToken cancellationToken) + private async Task> GetPartsFormDateWithRetry( + string sql, long unixTime, CancellationToken cancellationToken) { return await PgRetryStrategy.ExecuteWithRetry( async t => @@ -149,7 +149,8 @@ public async Task> GetPartsToDate( } } - public async Task DropPartsToDate(string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default) + public async Task DropPartsToDate( + string tableName, DateTimeOffset toDate, CancellationToken cancellationToken = default) { int droppedCount = 0; List list = await GetPartsToDate(tableName, toDate, cancellationToken); diff --git a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj index dda80a3a..a1c52b7c 100644 --- a/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj +++ b/src/Sa.Partitional.PostgreSql/Sa.Partitional.PostgreSql.csproj @@ -3,7 +3,7 @@ - 0.8.0 + 0.9.0 For managing table partitioning in PostgreSQL diff --git a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs index ccd61013..40ec2caa 100644 --- a/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs +++ b/src/Sa.Partitional.PostgreSql/SqlBuilder/SqlBuilder.cs @@ -63,13 +63,11 @@ public string CreatePartSql(string tableName, DateTimeOffset date, StrOrNum[] pa #region privates private ISqlTableBuilder? Find(string tableName) { - ISqlTableBuilder? item = - ( - storage.Schemas.Count == 1 - ? builders.GetValueOrDefault(GetFullName(storage.Schemas.First(), tableName)) - : builders.Values.FirstOrDefault(c => c.FullName == tableName) - ) - ?? builders.GetValueOrDefault(tableName); + ISqlTableBuilder? item = tableName.Contains('.') + ? builders.GetValueOrDefault(tableName) + : builders.GetValueOrDefault(GetFullName(storage.Schemas.First(), tableName)); + + item ??= builders.Values.FirstOrDefault(c => c.FullName == tableName); if (item != null) return item; diff --git a/src/Samples/Configuration.Web/Program.cs b/src/Samples/Configuration.Web/Program.cs index 7d17ca15..fd2c4f5d 100644 --- a/src/Samples/Configuration.Web/Program.cs +++ b/src/Samples/Configuration.Web/Program.cs @@ -12,7 +12,7 @@ string connectionString = builder.Configuration[PG_KEY] ?? throw new ArgumentException(PG_KEY); -using var ds = new PgDataSource(new(connectionString)); +using var ds = IPgDataSource.Create(connectionString); ds.ExecuteScalar(""" CREATE TABLE IF NOT EXISTS settings ( diff --git a/src/Samples/HybridFileStorage.Console/Program.cs b/src/Samples/HybridFileStorage.Console/Program.cs index f94fd464..90a28c35 100644 --- a/src/Samples/HybridFileStorage.Console/Program.cs +++ b/src/Samples/HybridFileStorage.Console/Program.cs @@ -55,7 +55,6 @@ public async Task Run(CancellationToken cancellationToken = default) var isDowload = await storage.DownloadAsync( result.FileId, - string.Empty, async (fs, t) => actual = await fs.ToStrAsync(t), cancellationToken); diff --git a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs index badd43e5..f53a0b15 100644 --- a/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.FileSystemTests/FileSystemStorageTests.cs @@ -43,6 +43,10 @@ public async Task Crud() bool canProcessed = Storage.CanProcess(result.FileId); Assert.True(canProcessed); + var meta = await Storage.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); + + var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); @@ -77,7 +81,12 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy [Fact] public async Task CrudEx() { - var metadata = new UploadFileInput { FileName = "/api/files/download/file/var/www/uploads/image.bin", TenantId = 1 }; + var metadata = new UploadFileInput + { + FileName = "/api/files/download/file/var/www/uploads/image.bin", + TenantId = 1 + }; + using MemoryStream fileContent = FixtureHelper.GetByteStream(); var result = await Storage.UploadAsync(metadata, fileContent, fixture.CancellationToken); @@ -88,6 +97,9 @@ public async Task CrudEx() bool canProcessed = Storage.CanProcess(result.FileId); Assert.True(canProcessed); + var meta = await Storage.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); + var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs index 7f500762..c7663cc8 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/FileIdParserTests.cs @@ -8,14 +8,16 @@ public class FileIdParserTests public void ParseFromFileId_ValidFileId_ReturnsTenantIdAndTimestamp() { string fileId = "pg://files/123/1773210911/foo/some.txt"; - var result = FileIdParser.TryParseFileIdWithFilename( + var result = FileIdParser.TryParse( fileId, + out string scopeName, out int tenantId, out long timestamp, out string filename); Assert.True(result); Assert.Equal(123, tenantId); + Assert.Equal("files", scopeName); Assert.Equal(1773210911, timestamp); Assert.Equal("foo/some.txt", filename); } @@ -24,10 +26,11 @@ public void ParseFromFileId_ValidFileId_ReturnsTenantIdAndTimestamp() public void ParseFromFileId_InvalidFileId() { string fileId = "invalid_file_id"; - var result = FileIdParser.TryParseFileIdWithFilename( + var result = FileIdParser.TryParse( fileId, out _, out _, + out _, out _); Assert.False(result); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index 5396ffa8..1e3b4c95 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -21,23 +21,29 @@ public async Task UploadFileAsync() Console.WriteLine(fixture.ConnectionString); // Arrange - var metadata = new UploadFileInput { FileName = "test.txt", TenantId = 1 }; + var input = new UploadFileInput { FileName = "test.txt", TenantId = 1 }; using MemoryStream fileContent = await CreateStream(fixture.CancellationToken); // Act - var result = await Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); + var result = await Sub.UploadAsync(input, fileContent, fixture.CancellationToken); // Assert Assert.NotNull(result); Assert.NotEmpty(result.FileId); - Assert.StartsWith("pg://files/1/", result.FileId); + Assert.StartsWith("pg://share/1/", result.FileId); object? v = await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM public.files WHERE id = @id", cmd => cmd.Parameters.Add(new("id", result.FileId)), fixture.CancellationToken); var count = (long)v!; Assert.Equal(1, count); + + + Assert.True(Sub.CanProcess(result.FileId)); + + var meta = Sub.GetMetadataAsync(result.FileId, fixture.CancellationToken); + Assert.NotNull(meta); } @@ -88,9 +94,10 @@ public async Task DownloadFileAsync() [Fact] public async Task GetMetadataAsync() { - var metadata = await Sub.GetMetadataAsync("pg://files/7/1773210911/some/data.bin", CancellationToken.None); + var metadata = await Sub.GetMetadataAsync("pg://share/7/1773210911/some/data.bin", CancellationToken.None); Assert.NotNull(metadata); + Assert.Equal("share", metadata.ScopeName); Assert.Equal(7, metadata.TenantId); Assert.Equal("some/data.bin", metadata.FileName); } @@ -115,7 +122,8 @@ public async Task Upload2FileAsync() Assert.Equal(fileId1, fileId2); - long count = await fixture.DataSource.ExecuteScalar("SELECT COUNT(*) FROM public.files WHERE id = @id", + long count = await fixture.DataSource.ExecuteScalar( + "SELECT COUNT(*) FROM public.files WHERE id = @id", cmd => cmd.Parameters.Add(new("id", fileId1)), fixture.CancellationToken); Assert.Equal(1, count); diff --git a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs index f33a711f..f6c08927 100644 --- a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs @@ -77,7 +77,7 @@ public async Task Crud() var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, string.Empty, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); Assert.True(isDeleted); @@ -100,7 +100,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var isSame = await EnsureFileSame(result.FileId, fileContent); Assert.True(isSame); - var isDeleted = await Storage.DeleteAsync(result.FileId, string.Empty, fixture.CancellationToken); + var isDeleted = await Storage.DeleteAsync(result.FileId, fixture.CancellationToken); Assert.True(isDeleted); } @@ -147,7 +147,6 @@ private async Task EnsureFileSame(string fileName, MemoryStream expectedBy using MemoryStream memoryStream = new(); var isDownloaded = await Storage.DownloadAsync(fileName - , scopeName: string.Empty , (stream, ct) => stream.CopyToAsync(memoryStream, ct) , fixture.CancellationToken); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs index e6159588..db2dd4da 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Delivery/DeliveryBatchingWindowTests.cs @@ -52,10 +52,18 @@ public async Task Deliver_Process_MustBe_Work() Console.Write(fixture.ConnectionString); - var cnt = await fixture.Publisher.PublishSingle(new TestMessage { PayloadId = "11", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); + var cnt = await fixture.Publisher.PublishSingle( + new TestMessage { PayloadId = "11", Content = "Message 1", TenantId = 1 }, + tenantId: 1, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(cnt > 0); - cnt = await fixture.Publisher.PublishSingle(new TestMessage { PayloadId = "12", Content = "Message 2", TenantId = 2 }, 2, TestContext.Current.CancellationToken); + cnt = await fixture.Publisher.PublishSingle( + new TestMessage { PayloadId = "12", Content = "Message 2", TenantId = 2 }, + tenantId: 2, + cancellationToken: TestContext.Current.CancellationToken); + Assert.True(cnt > 0); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs index d2c2fb1b..7c8087dd 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/Publisher/OutboxPublisherTests.cs @@ -19,14 +19,18 @@ public async Task Publish_MultipleMessages_ReturnsExpectedResult() { Console.Write(fixture.ConnectionString); - ulong result = await Sub.PublishSingle(new TestMessage { PayloadId = "1", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); + ulong result = await Sub.PublishSingle( + new TestMessage { PayloadId = "1", Content = "Message 1", TenantId = 1 }, 1, TestContext.Current.CancellationToken); - result += await Sub.PublishSingle(new TestMessage { PayloadId = "2", Content = "Message 3", TenantId = 2 }, 2, TestContext.Current.CancellationToken); + result += await Sub.PublishSingle( + new TestMessage { PayloadId = "2", Content = "Message 3", TenantId = 2 }, 2, TestContext.Current.CancellationToken); // Assert Assert.Equal(2, (int)result); - var sql = $"select count(*) from {PgOutboxTableSettings.Defaults.DatabaseTableName}{PgOutboxTableSettings.MessageTable.Suffix}"; + var sql = @$"select count(*) from + {PgOutboxTableSettings.Defaults.DatabaseTableName}{PgOutboxTableSettings.MessageTable.Suffix}"; + int count = await fixture.DataSource.ExecuteReaderFirst(sql, TestContext.Current.CancellationToken); Assert.Equal(2, count); } diff --git a/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs b/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs index 86930118..e56d2fb8 100644 --- a/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs +++ b/src/Tests/Sa.Partitional.PostgreSqlTests/Configuration/ConfigurationPartTests.cs @@ -4,8 +4,6 @@ namespace Sa.Partitional.PostgreSqlTests.Configuration; - - public class ConfigurationPartTests(ConfigurationPartTests.Fixture fixture) : IClassFixture { From 3501af6ee71ee6fdd0478d5ff6b9930673c69b56 Mon Sep 17 00:00:00 2001 From: dundich Date: Mon, 30 Mar 2026 11:39:31 +0300 Subject: [PATCH 02/28] basket in hybrid --- .../FileSystemStorage.cs | 12 ++-- .../FileSystemStorageOptions.cs | 60 ++++++++++++++++++- .../FileSystemStorageSettings.cs | 8 +-- src/Sa.HybridFileStorage.FileSystem/Setup.cs | 3 +- .../ThrowHelper.cs | 19 ------ .../FileIdParser.cs | 10 ++-- .../PostgresFileStorage.cs | 32 +++++----- .../PostgresFileStorageConfiguration.cs | 6 +- .../PostgresFileStorageOptions.cs | 2 +- src/Sa.HybridFileStorage.S3/S3FileStorage.cs | 12 ++-- .../S3FileStorageOptions.cs | 2 +- .../Domain/IFileStorage.cs | 6 +- src/Sa.HybridFileStorage/FileMetadata.cs | 4 +- src/Sa.HybridFileStorage/HybridFileStorage.cs | 24 ++++---- .../HybridFileStorageContainer.cs | 2 +- .../HybridFileStorageExtensions.cs | 14 ++--- .../IHybridFileStorage.cs | 30 +++++----- .../IHybridFileStorageContainer.cs | 2 +- .../InMemoryFileStorage.cs | 50 ++++++++++++++-- .../InMemoryFileStorageOptions.cs | 2 +- src/Sa.HybridFileStorage/Readme.md | 8 ++- .../HybridFileStorage.Console/Program.cs | 2 +- .../PostgresFileStorageTests.cs | 2 +- .../HybridFileStorageTests.cs | 8 +-- 24 files changed, 199 insertions(+), 121 deletions(-) delete mode 100644 src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs index 7baae69b..b84a92df 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorage.cs @@ -17,7 +17,7 @@ internal sealed class FileSystemStorage( Path.GetFullPath(settings.BasePath)); private readonly string _basePathScope = Path.TrimEndingDirectorySeparator( - Path.GetFullPath(Path.Combine(settings.BasePath, settings.ScopeName))); + Path.GetFullPath(Path.Combine(settings.BasePath, settings.Basket))); private readonly string _schemePrefix = $"{settings.StorageType}{SchemeSeparator}"; private readonly string _storageType = settings.StorageType; @@ -25,7 +25,7 @@ internal sealed class FileSystemStorage( private readonly int _bufferSize = settings.BufferSize; private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; - public string ScopeName => settings.ScopeName; + public string Basket => settings.Basket; public string StorageType => _storageType; public bool IsReadOnly => _isReadOnly; @@ -59,7 +59,7 @@ public async Task UploadAsync( string filename = PathSanitizer.SanitizeRelativePath(metadata.FileName); - string relativePath = string.Concat(ScopeName, "/", metadata.TenantId.ToString(), "/", filename); + string relativePath = string.Concat(Basket, "/", metadata.TenantId.ToString(), "/", filename); string filePath = Path.Combine(_basePath, relativePath); EnsurePathWithinBase(filePath); @@ -212,7 +212,7 @@ private void EnsurePathWithinBase(string path) if (!CanProcess(fileId)) return Task.FromResult(null); - // Парсинг формата: "storageType://scope/tenant/filename" + //parse: "storageType://basket/tenant/filename" ReadOnlySpan span = fileId.AsSpan(); int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); if (schemeEnd == -1) @@ -220,7 +220,7 @@ private void EnsurePathWithinBase(string path) var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; - // Парсинг "tenantId/filename" + // "tenantId/filename" int slashIndex = pathPart.IndexOf('/'); if (slashIndex == -1) return Task.FromResult(null); @@ -239,7 +239,7 @@ private void EnsurePathWithinBase(string path) var metadata = new FileMetadata { StorageType = StorageType, - ScopeName = scopeSpan.ToString(), + Basket = scopeSpan.ToString(), FileName = fileNameSpan.ToString(), TenantId = tenantId }; diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs index 16ff528c..5c1143e8 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageOptions.cs @@ -11,5 +11,63 @@ public sealed record FileSystemStorageOptions [StringLength(255)] public string BasePath { get; set; } = string.Empty; public bool IsReadOnly { get; set; } = false; - public string? ScopeName { get; set; } + [Required] + [StringLength(63, MinimumLength = 3)] + public string Basket { get; set; } = FileSystemStorageSettings.DefaultBasket; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(BasePath)) + { + throw new ValidationException("BasePath cannot be empty."); + } + + try + { + // Get the full path to resolve any relative paths + string fullPath = Path.GetFullPath(BasePath); + + // Check if the path contains any invalid characters + if (BasePath.IndexOfAny(Path.GetInvalidPathChars()) >= 0) + { + throw new ValidationException($"BasePath contains invalid characters: {BasePath}"); + } + } + catch (Exception ex) + { + throw new ValidationException($"Invalid BasePath format: {BasePath}. {ex.Message}"); + } + + if (string.IsNullOrWhiteSpace(Basket)) + { + throw new ValidationException("Basket cannot be empty."); + } + + + if (Basket.Length > 63 || Basket.Length < 3) + { + throw new ValidationException($"Basket exceeds maximum length of 63 characters."); + } + + if (!char.IsLetter(Basket[0]) && Basket[0] != '_') + { + throw new ValidationException("Basket must start with a letter or underscore."); + } + + + if (string.IsNullOrWhiteSpace(Basket)) + { + throw new ValidationException("Basket cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(StorageType)) + { + throw new ValidationException("StorageType cannot be empty."); + } + + if (StorageType.Length > 10) + { + throw new ValidationException($"StorageType exceeds maximum length of 63 characters."); + } + } } diff --git a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs index 02db3ffc..e563b53f 100644 --- a/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs +++ b/src/Sa.HybridFileStorage.FileSystem/FileSystemStorageSettings.cs @@ -4,19 +4,17 @@ namespace Sa.HybridFileStorage.FileSystem; public sealed record FileSystemStorageSettings { - [Required] public string StorageType { get; init; } = DefaultStorageType; - [StringLength(255)] + public string Basket { get; init; } = DefaultBasket; + public required string BasePath { get; init; } - [StringLength(100, MinimumLength = 1)] public bool IsReadOnly { get; init; } = false; - public string ScopeName { get; init; } = "share"; - public int BufferSize { get; init; } = 256 * 1024; public const string DefaultStorageType = "fs"; + public const string DefaultBasket = "share"; } diff --git a/src/Sa.HybridFileStorage.FileSystem/Setup.cs b/src/Sa.HybridFileStorage.FileSystem/Setup.cs index e8f1d203..2f903c96 100644 --- a/src/Sa.HybridFileStorage.FileSystem/Setup.cs +++ b/src/Sa.HybridFileStorage.FileSystem/Setup.cs @@ -23,12 +23,13 @@ public static IServiceCollection AddSaFileSystemFileStorage( { FileSystemStorageOptions options = new(); configure.Invoke(sp, options); + options.Validate(); return new FileSystemStorage(new FileSystemStorageSettings { BasePath = options.BasePath, IsReadOnly = options.IsReadOnly, - ScopeName = string.IsNullOrWhiteSpace(options.ScopeName) ? "share" : options.ScopeName, + Basket = options.Basket, StorageType = options.StorageType, }, sp.GetService()); }); diff --git a/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs b/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs deleted file mode 100644 index 152a8c72..00000000 --- a/src/Sa.HybridFileStorage.FileSystem/ThrowHelper.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Sa.HybridFileStorage; -using System.Diagnostics.CodeAnalysis; -using System.Security; - -//static class ThrowHelper -//{ -// [DoesNotReturn] -// public static void ThrowWritableException() => -// throw new HybridFileStorageWritableException(); - -// [DoesNotReturn] -// public static void ThrowInvalidFileIdFormat() => -// throw new FormatException("Invalid file ID format."); - -// [DoesNotReturn] -// public static void ThrowSecurityException(string path, string basePath) => -// throw new SecurityException( -// $"Access denied. Path '{path}' is outside the allowed base directory '{basePath}'."); -//} diff --git a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs index d9304e5b..662cf157 100644 --- a/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs +++ b/src/Sa.HybridFileStorage.Postgres/FileIdParser.cs @@ -8,7 +8,7 @@ internal static class FileIdParser public static bool TryParse( string fileId, - out string scopeName, + out string basket, out int tenantId, out long timestamp, out string fileName) @@ -16,7 +16,7 @@ public static bool TryParse( tenantId = default; timestamp = default; fileName = string.Empty; - scopeName = string.Empty; + basket = string.Empty; if (string.IsNullOrEmpty(fileId)) return false; @@ -28,7 +28,7 @@ public static bool TryParse( int scopeEnd = afterSpan.IndexOf('/'); if (scopeEnd == -1) return false; - scopeName = afterSpan[..scopeEnd].ToString(); + basket = afterSpan[..scopeEnd].ToString(); afterSpan = afterSpan[(scopeEnd + 1)..]; @@ -55,11 +55,11 @@ public static bool TryParse( public static string FormatToFileId( string storageType, - string scopeName, + string basket, int tenantId, DateTimeOffset date, string fileName) - => $"{storageType}://{scopeName}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; + => $"{storageType}://{basket}/{tenantId}/{date.ToUnixTimeSeconds()}/{NormalizeFileName(fileName)}"; public static string NormalizeFileName(string fileName) => fileName.TrimStart('\\', '/').Replace('\\', '/'); diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index e3ba18a9..79198c25 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -15,15 +15,15 @@ internal sealed class PostgresFileStorage( IPartitionManager partManager, RecyclableMemoryStreamManager streamManager, StorageOptions options, - string scopeName, + string basket, TimeProvider? timeProvider = null) : IFileStorage { private const string InsertSql = """ - INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, scope_name, created_at) - VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @scope_name, @created_at) - ON CONFLICT (id, tenant_id, scope_name, created_at) DO UPDATE SET + INSERT INTO {0} (id, name, file_ext, data, size, tenant_id, basket, created_at) + VALUES (@id, @name, @file_ext, @data, @size, @tenant_id, @basket, @created_at) + ON CONFLICT (id, tenant_id, basket, created_at) DO UPDATE SET data = EXCLUDED.data, size = EXCLUDED.size, created_at = EXCLUDED.created_at @@ -32,19 +32,19 @@ ON CONFLICT (id, tenant_id, scope_name, created_at) DO UPDATE SET private const string DeleteSql = """ DELETE FROM {0} - WHERE tenant_id = @tenant_id AND scope_name = @scope_name + WHERE tenant_id = @tenant_id AND basket = @basket AND created_at >= @timestamp AND id = @id """; private const string SelectSql = """ SELECT data FROM {0} - WHERE tenant_id = @tenant_id AND scope_name = @scope_name + WHERE tenant_id = @tenant_id AND basket = @basket AND created_at >= @timestamp AND id = @id """; private readonly string _partName - = string.IsNullOrWhiteSpace(scopeName) ? "share" : Sanitize(scopeName); + = string.IsNullOrWhiteSpace(basket) ? "share" : Sanitize(basket); private readonly string _qualifiedTableName = $"{options.SchemaName}.\"{Sanitize(options.TableName)}\""; @@ -58,7 +58,7 @@ private readonly string _schemePrefix public bool IsReadOnly => options.IsReadOnly; - public string ScopeName => scopeName; + public string Basket => basket; [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -82,12 +82,12 @@ public bool CanProcess(string? fileId) if (schemeEnd == -1) return false; var afterSpan = fileSpan[(schemeEnd + FileIdParser.SchemeSeparator.Length)..]; - int scopeEnd = afterSpan.IndexOf('/'); - if (scopeEnd == -1) return false; + int basketEnd = afterSpan.IndexOf('/'); + if (basketEnd == -1) return false; - var scopeName = afterSpan[..scopeEnd]; + var basket = afterSpan[..basketEnd]; - return scopeName.Equals(_partName, StringComparison.Ordinal); + return basket.Equals(_partName, StringComparison.Ordinal); } public async Task UploadAsync( @@ -136,7 +136,7 @@ await dataSource.ExecuteNonQuery(sql, , new NpgsqlParameter("data", fileStream) , new NpgsqlParameter("size", (int)ms.Length) , new NpgsqlParameter("tenant_id", metadata.TenantId) - , new NpgsqlParameter("scope_name", _partName) + , new NpgsqlParameter("basket", _partName) , new NpgsqlParameter("created_at", createdAt) ], cancellationToken); } @@ -166,7 +166,7 @@ public async Task DeleteAsync(string fileId, CancellationToken cancellatio int rowsAffected = await dataSource.ExecuteNonQuery(sql, [ new NpgsqlParameter("tenant_id", tenantId), - new NpgsqlParameter("scope_name", _partName), + new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ], cancellationToken); @@ -193,7 +193,7 @@ public async Task DownloadAsync( }, [ new NpgsqlParameter("tenant_id", tenantId), - new NpgsqlParameter("scope_name", _partName), + new NpgsqlParameter("basket", _partName), new NpgsqlParameter("timestamp", timestamp), new NpgsqlParameter("id", fileId) ], cancellationToken); @@ -231,7 +231,7 @@ private static string Sanitize(ReadOnlySpan input) var metadata = new FileMetadata { - ScopeName = _partName, + Basket = _partName, StorageType = StorageType, FileName = fileName, TenantId = tenantId diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs index 5fe90bf4..586733e7 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageConfiguration.cs @@ -33,10 +33,10 @@ public PostgresFileStorageConfiguration(IServiceCollection services) "size INT NOT NULL", "file_ext TEXT NOT NULL", "tenant_id INT NOT NULL", - "scope_name TEXT NOT NULL", + "basket TEXT NOT NULL", "data BYTEA NOT NULL" ) - .PartByList("tenant_id", "scope_name") + .PartByList("tenant_id", "basket") .PartByRange(_options.PartOptions.PgPartBy, "created_at"); }); }) @@ -65,7 +65,7 @@ public PostgresFileStorageConfiguration(IServiceCollection services) partManager: pm, streamManager: sm, options: _options.StorageOptions, - scopeName: _options.ScopeName, + basket: _options.PartOptions.Basket, timeProvider: time); return storage; diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs index a4834bbb..34244c3a 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorageOptions.cs @@ -19,6 +19,7 @@ public sealed class PartOptions { public int MigrationScheduleForwardDays { get; set; } = 2; public PgPartBy PgPartBy { get; set; } = PgPartBy.Day; + public string Basket { get; set; } = "share"; } public sealed class PostgresFileStorageOptions @@ -26,5 +27,4 @@ public sealed class PostgresFileStorageOptions public StorageOptions StorageOptions { get; } = new(); public PartOptions PartOptions { get; } = new(); public CleanupOptions CleanupOptions { get; } = new(); - public string ScopeName { get; set; } = "share"; } diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs index 17330289..58ba67e0 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorage.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorage.cs @@ -11,12 +11,12 @@ internal sealed class S3FileStorage( S3FileStorageOptions options, TimeProvider? timeProvider = null) : IFileStorage { - private const string DefaultScopeName = "share"; + private const string DefaultBasket = "share"; private const string SchemeSeparator = "://"; - private readonly string _pathPrefix = string.IsNullOrWhiteSpace(options.ScopeName) - ? DefaultScopeName - : options.ScopeName; + private readonly string _pathPrefix = string.IsNullOrWhiteSpace(options.Basket) + ? DefaultBasket + : options.Basket; private readonly string _schemePrefix = $"{options.StorageType}{SchemeSeparator}"; private readonly string _storageType = options.StorageType; private readonly bool _isReadOnly = options.IsReadOnly; @@ -25,7 +25,7 @@ internal sealed class S3FileStorage( public string StorageType => _storageType; public bool IsReadOnly => _isReadOnly; - public string ScopeName => _pathPrefix; + public string Basket => _pathPrefix; [MethodImpl(MethodImplOptions.AggressiveInlining)] private void EnsureWritable() @@ -142,7 +142,7 @@ private static string GetFilePath(string fileId) return new FileMetadata { - ScopeName = ScopeName, + Basket = Basket, StorageType = StorageType, FileName = fileNameSpan.ToString(), TenantId = tenantId diff --git a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs index 0cf4f5d1..cf856220 100644 --- a/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs +++ b/src/Sa.HybridFileStorage.S3/S3FileStorageOptions.cs @@ -3,7 +3,7 @@ public sealed class S3FileStorageOptions { public string StorageType { get; init; } = "s3"; - public string ScopeName { get; init; } = string.Empty; + public string Basket { get; init; } = "share"; /// /// http://localhost:9000 diff --git a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs index 145ae034..05122d42 100644 --- a/src/Sa.HybridFileStorage/Domain/IFileStorage.cs +++ b/src/Sa.HybridFileStorage/Domain/IFileStorage.cs @@ -6,9 +6,9 @@ public interface IFileStorage { /// - /// scope domain + /// BL - scope domain /// - string ScopeName { get; } + string Basket { get; } /// /// Gets the type of the storage. @@ -42,7 +42,7 @@ Task UploadAsync( /// /// Downloads a file from the storage by its ID. /// - /// pg://files/1/1773210911/test.txt + /// pg://share/1/1773210911/test.txt /// Cancellation token. /// The file stream. Task DownloadAsync( diff --git a/src/Sa.HybridFileStorage/FileMetadata.cs b/src/Sa.HybridFileStorage/FileMetadata.cs index b0290bf7..65e3f4d5 100644 --- a/src/Sa.HybridFileStorage/FileMetadata.cs +++ b/src/Sa.HybridFileStorage/FileMetadata.cs @@ -2,8 +2,8 @@ public sealed class FileMetadata { - public required string StorageType { get; init; } - public required string ScopeName { get; init; } + public required string Basket { get; init; } public required string FileName { get; init; } public int TenantId { get; init; } + public required string StorageType { get; init; } } diff --git a/src/Sa.HybridFileStorage/HybridFileStorage.cs b/src/Sa.HybridFileStorage/HybridFileStorage.cs index f42e885a..f89ff90b 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorage.cs @@ -9,35 +9,35 @@ internal sealed class HybridFileStorage( InterceptorContainer interceptors) : IHybridFileStorage { - public IReadOnlyCollection Storages => container.Storages; + public IEnumerable Storages => container.Storages; - private void EnsureWritable(string scopeName) + private void EnsureWritable(string basket) { - if (!container.Storages.Any(c => c.ScopeName == scopeName)) + if (!container.Storages.Any(c => c.Basket == basket)) { throw new HybridFileStorageNoAvailableException(); } - if (Storages.All(f => f.ScopeName == scopeName && f.IsReadOnly)) + if (Storages.All(f => f.Basket == basket && f.IsReadOnly)) { throw new HybridFileStorageWritableException(); } } public async Task UploadAsync( + string basket, UploadFileInput input, - string scopeName, Stream fileStream, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(scopeName, nameof(scopeName)); + ArgumentNullException.ThrowIfNull(basket, nameof(basket)); - EnsureWritable(scopeName); + EnsureWritable(basket); return await ExecuteStorageOperationAsync( - container.Storages.Where(c => !c.IsReadOnly && c.ScopeName == scopeName), + container.Storages.Where(c => !c.IsReadOnly && c.Basket == basket), async (storage, ct) => await interceptors.ExecuteBeforeUploadAsync(storage, input, fileStream, ct), async (storage, ct) => await storage.UploadAsync(input, fileStream, ct), interceptors.ExecuteAfterUploadAsync, @@ -54,7 +54,7 @@ public async Task DownloadAsync( ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId), + CanProcess(fileId), async (storage, ct) => await interceptors.ExecuteBeforeDownloadAsync(storage, fileId, loadStream, ct), async (storage, ct) => await storage.DownloadAsync(fileId, loadStream, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDownloadAsync(storage, fileId, result, ct), @@ -70,7 +70,7 @@ public async Task DeleteAsync( ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); return await ExecuteStorageOperationAsync( - CanStorages(fileId).Where(c => !c.IsReadOnly), + CanProcess(fileId).Where(c => !c.IsReadOnly), async (storage, ct) => await interceptors.ExecuteBeforeDeleteAsync(storage, fileId, ct), async (storage, ct) => await storage.DeleteAsync(fileId, ct), async (storage, result, ct) => await interceptors.ExecuteAfterDeleteAsync(storage, fileId, result, ct), @@ -120,7 +120,7 @@ private static async Task ExecuteStorageOperationAsync( { ArgumentNullException.ThrowIfNull(fileId, nameof(fileId)); - foreach (var fs in CanStorages(fileId)) + foreach (var fs in container.Storages) { var meta = await fs.GetMetadataAsync(fileId, cancellationToken); if (meta != null) return meta; @@ -130,6 +130,6 @@ private static async Task ExecuteStorageOperationAsync( } - internal IEnumerable CanStorages(string fileId) + internal IEnumerable CanProcess(string fileId) => container.Storages.Where(c => c.CanProcess(fileId)); } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs index bd1f9a40..2a75ebe9 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageContainer.cs @@ -15,5 +15,5 @@ public IHybridFileStorageContainerConfiguration AddStorage(IFileStorage storage) return this; } - public IReadOnlyCollection Storages => _storages; + public IEnumerable Storages => _storages; } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index 5aadcb24..9e171efc 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -9,8 +9,8 @@ public static class HybridFileStorageExtensions public static async Task CopyFromFileAsync( this IHybridFileStorage storage, string filePath, + string basket, UploadFileInput input, - string scopeName, CancellationToken ct = default) { // копируем файл в хранилище @@ -24,16 +24,16 @@ public static async Task CopyFromFileAsync( }); return await storage.UploadAsync( + basket: basket, input: input, - scopeName: scopeName, fileStream: fs, cancellationToken: ct); } - public static async Task CopyToScopeAsync( + public static async Task CopyToBasketAsync( this IHybridFileStorage storage, string fileId, - string targetScopeName, + string basket, int? targetTenantId = default, CancellationToken ct = default) { @@ -46,7 +46,7 @@ public static async Task CopyToScopeAsync( int tenantId = targetTenantId ?? metadata.TenantId; if (tenantId == metadata.TenantId - && string.Equals(metadata.ScopeName, targetScopeName, StringComparison.Ordinal)) + && string.Equals(metadata.Basket, basket, StringComparison.Ordinal)) { ThrowFileAlreadyInTargetScope(); } @@ -63,8 +63,8 @@ public static async Task CopyToScopeAsync( async (sourceStream, downloadCt) => { result = await storage.UploadAsync( + basket, uploadInput, - targetScopeName, sourceStream, downloadCt); }, @@ -127,7 +127,7 @@ public static async Task> CopyToScopeBatchAsync( var ct = cts?.Token ?? cancellationToken; - return await storage.CopyToScopeAsync(fileId, targetScopeName, targetTenantId, ct); + return await storage.CopyToBasketAsync(fileId, targetScopeName, targetTenantId, ct); } catch (Exception ex) { diff --git a/src/Sa.HybridFileStorage/IHybridFileStorage.cs b/src/Sa.HybridFileStorage/IHybridFileStorage.cs index 91179adf..54fb21da 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorage.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorage.cs @@ -12,15 +12,20 @@ public interface IHybridFileStorage /// /// storages /// - IReadOnlyCollection Storages { get; } + IEnumerable Storages { get; } /// - /// Deletes the file associated with the specified file ID asynchronously. + /// Uploads a file asynchronously using the provided input and file stream. /// - /// The unique identifier for the file to be deleted. + /// Metadata about the file being uploaded. + /// A stream containing the file data to be uploaded. /// A cancellation token to cancel the operation if needed. - /// True if the file was successfully deleted; otherwise, false. - Task DeleteAsync(string fileId, CancellationToken cancellationToken = default); + /// A containing the result of the upload operation. + Task UploadAsync( + string basket, + UploadFileInput input, + Stream fileStream, + CancellationToken cancellationToken = default); /// /// Downloads the file associated with the specified file ID asynchronously. @@ -34,19 +39,14 @@ Task DownloadAsync( Func loadStream, CancellationToken cancellationToken = default); + /// - /// Uploads a file asynchronously using the provided input and file stream. + /// Deletes the file associated with the specified file ID asynchronously. /// - /// Metadata about the file being uploaded. - /// A stream containing the file data to be uploaded. + /// The unique identifier for the file to be deleted. /// A cancellation token to cancel the operation if needed. - /// A containing the result of the upload operation. - Task UploadAsync( - UploadFileInput input, - string scopeName, - Stream fileStream, - CancellationToken cancellationToken = default); - + /// True if the file was successfully deleted; otherwise, false. + Task DeleteAsync(string fileId, CancellationToken cancellationToken = default); Task GetMetadataAsync( string fileId, diff --git a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs index 0f3108fd..ddf5bcd5 100644 --- a/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs +++ b/src/Sa.HybridFileStorage/IHybridFileStorageContainer.cs @@ -4,5 +4,5 @@ namespace Sa.HybridFileStorage; public interface IHybridFileStorageContainer : IHybridFileStorageContainerConfiguration { - IReadOnlyCollection Storages { get; } + IEnumerable Storages { get; } } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs index 324174a5..0e116836 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorage.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorage.cs @@ -1,5 +1,6 @@ using Sa.HybridFileStorage.Domain; using System.Collections.Concurrent; +using System.Globalization; namespace Sa.HybridFileStorage; @@ -8,7 +9,7 @@ public sealed class InMemoryFileStorage( InMemoryFileStorageOptions? options = null, TimeProvider? timeProvider = null) : IFileStorage { - private readonly InMemoryFileStorageOptions _options = options ?? new (string.Empty); + private readonly InMemoryFileStorageOptions _options = options ?? new(string.Empty); private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; @@ -17,12 +18,14 @@ public sealed class InMemoryFileStorage( private readonly ConcurrentDictionary _storage = []; - public string ScopeName => _options.ScopeName; + public string Basket => _options.Basket; public string StorageType => DefaultStorageType; public bool IsReadOnly => _options.IsReadOnly; + public const string SchemeSeparator = "://"; + private void EnsureWritable() { if (IsReadOnly) @@ -42,8 +45,9 @@ public async Task UploadAsync( await fileStream.CopyToAsync(memoryStream, cancellationToken); byte[] fileData = memoryStream.ToArray(); - string path = Path.Combine(metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); - string fileId = $"{StorageType}://{path}"; + string path = Path.Combine(Basket, metadata.TenantId.ToString(), metadata.FileName).Replace('\\', '/'); + //"storageType://basket/tenant/filename" + string fileId = $"{StorageType}{SchemeSeparator}{path}"; _storage[fileId] = fileData; @@ -72,8 +76,42 @@ public Task DeleteAsync(string fileId, CancellationToken cancellationToken public bool CanProcess(string fileId) => fileId.StartsWith(StorageType); - public Task GetMetadataAsync(string fileId, CancellationToken cancellationToken = default) + public async Task GetMetadataAsync(string fileId, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + if (!_storage.ContainsKey(fileId)) return null; + + //parse: "storageType://basket/tenant/filename" + ReadOnlySpan span = fileId.AsSpan(); + int schemeEnd = span.IndexOf(SchemeSeparator.AsSpan()); + if (schemeEnd == -1) + return null; + + var pathPart = span[(schemeEnd + SchemeSeparator.Length)..]; + + // "tenantId/filename" + int slashIndex = pathPart.IndexOf('/'); + if (slashIndex == -1) + return null; + + var scopeSpan = pathPart[..slashIndex]; + + var nextSpan = pathPart[(slashIndex + 1)..]; + slashIndex = nextSpan.IndexOf('/'); + + var tenantSpan = nextSpan[..slashIndex]; + var fileNameSpan = nextSpan[(slashIndex + 1)..]; + + if (!int.TryParse(tenantSpan, NumberStyles.None, CultureInfo.InvariantCulture, out int tenantId)) + return null; + + var metadata = new FileMetadata + { + StorageType = StorageType, + Basket = scopeSpan.ToString(), + FileName = fileNameSpan.ToString(), + TenantId = tenantId + }; + + return metadata; } } diff --git a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs index 12f5cebb..db45c76c 100644 --- a/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs +++ b/src/Sa.HybridFileStorage/InMemoryFileStorageOptions.cs @@ -1,3 +1,3 @@ namespace Sa.HybridFileStorage; -public sealed record InMemoryFileStorageOptions(string ScopeName, bool IsReadOnly = false); +public sealed record InMemoryFileStorageOptions(string Basket = "share", bool IsReadOnly = false); diff --git a/src/Sa.HybridFileStorage/Readme.md b/src/Sa.HybridFileStorage/Readme.md index 4ccca1fc..a607c6d7 100644 --- a/src/Sa.HybridFileStorage/Readme.md +++ b/src/Sa.HybridFileStorage/Readme.md @@ -17,7 +17,7 @@ This interface defines a contract for hybrid file storage systems capable of han ## Key Features - ✅ **Unified API** — Single interface for all storage providers -- ✅ **Scope-based isolation** — Multi-tenant support via scopes +- ✅ **Basket-Tenant-based isolation** — Multi-tenant support - ✅ **Read-only mode** — Protect storage from accidental modifications - ✅ **Streaming support** — Memory-efficient file transfers - ✅ **Native AOT ready** — Full compatibility with .NET 10 Native AOT @@ -33,7 +33,7 @@ The `HybridFileStorageExtensions` class provides high-level methods for bulk fil All files are identified using a unified URI-like format: ``` -{storageType}://{scope}/{tenantId}/{fileName} +{storageType}://{basket}/{tenantId}/{fileName} ``` **Examples:** @@ -53,12 +53,14 @@ dotnet add package Sa.HybridFileStorage ```csharp // di -builder.Services.AddSaHybridStorage((_, b) => b.AddStorage(new InMemoryFileStorage())); +builder.AddStorage(new InMemoryFileStorage()) +builder.Services.AddSaHybridStorage(); // some test using var stream = "Hello, HybridFileStorage!".ToStream(); await storage.UploadAsync( + "basket", new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); diff --git a/src/Samples/HybridFileStorage.Console/Program.cs b/src/Samples/HybridFileStorage.Console/Program.cs index 90a28c35..25ecd680 100644 --- a/src/Samples/HybridFileStorage.Console/Program.cs +++ b/src/Samples/HybridFileStorage.Console/Program.cs @@ -46,8 +46,8 @@ public async Task Run(CancellationToken cancellationToken = default) using var stream = expected.ToStream(); var result = await storage.UploadAsync( - new UploadFileInput { FileName = "file.txt" }, string.Empty, + new UploadFileInput { FileName = "file.txt" }, stream, cancellationToken); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index 1e3b4c95..d2090587 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -97,7 +97,7 @@ public async Task GetMetadataAsync() var metadata = await Sub.GetMetadataAsync("pg://share/7/1773210911/some/data.bin", CancellationToken.None); Assert.NotNull(metadata); - Assert.Equal("share", metadata.ScopeName); + Assert.Equal("share", metadata.Basket); Assert.Equal(7, metadata.TenantId); Assert.Equal("some/data.bin", metadata.FileName); } diff --git a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs index f6c08927..3af6bda2 100644 --- a/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorageTests/HybridFileStorageTests.cs @@ -66,7 +66,7 @@ public async Task Crud() var input = new UploadFileInput { FileName = "test.bin", TenantId = 2 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, string.Empty, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(string.Empty, input, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.NotEmpty(result.FileId); @@ -92,7 +92,7 @@ public async Task UploadInMemStorageBySomeInterceptor() var input = new UploadFileInput { FileName = "some.bin", TenantId = 1 }; using MemoryStream fileContent = FixtureHelper.GetByteStream(); - var result = await Storage.UploadAsync(input, string.Empty, fileContent, fixture.CancellationToken); + var result = await Storage.UploadAsync(string.Empty, input, fileContent, fixture.CancellationToken); Assert.NotNull(result); Assert.Equal(InMemoryFileStorage.DefaultStorageType, result.StorageType); @@ -115,7 +115,7 @@ public async Task WhenStorageIsEmptyThrowsInvalidOperationException() using MemoryStream fileContent = FixtureHelper.GetByteStream(); await Assert.ThrowsAsync(() => sp.GetRequiredService().UploadAsync( - new UploadFileInput { FileName = "", TenantId = 1 }, string.Empty, fileContent, fixture.CancellationToken)); + string.Empty, new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); } @@ -133,8 +133,8 @@ public async Task WhenStorageIsReadOnlyThrowsInvalidOperationException() await Assert.ThrowsAsync(() => sp.GetRequiredService().UploadAsync( - new UploadFileInput { FileName = "", TenantId = 1 }, string.Empty, + new UploadFileInput { FileName = "", TenantId = 1 }, fileContent, fixture.CancellationToken)); } From db095858aff5a630f7b6f0d7cf927fb0d22b2085 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 31 Mar 2026 14:30:18 +0300 Subject: [PATCH 03/28] fix mmpeg --- .../Domain/StorageResult.cs | 4 +- src/Sa.Media.FFmpeg/FFMpegOptions.cs | 2 +- src/Sa.Media.FFmpeg/IFFRawExecutor.cs | 2 +- src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj | 3 +- .../Services/FFMpegExecutor.cs | 21 +- .../Services/FFMpegExecutorFactory.cs | 4 +- src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs | 3 +- .../Services/ProcessExecutionResult.cs | 21 + .../Services/ProcessExecutor.cs | 438 ++++++++++++++++++ src/Sa.Media.FFmpeg/Setup.cs | 1 - src/Sa/Classes/IProcessExecutor.cs | 2 +- .../FFMpegProcessorTests.cs | 4 +- 12 files changed, 489 insertions(+), 16 deletions(-) create mode 100644 src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs create mode 100644 src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs diff --git a/src/Sa.HybridFileStorage/Domain/StorageResult.cs b/src/Sa.HybridFileStorage/Domain/StorageResult.cs index d6d49dfb..cd6ceddf 100644 --- a/src/Sa.HybridFileStorage/Domain/StorageResult.cs +++ b/src/Sa.HybridFileStorage/Domain/StorageResult.cs @@ -2,7 +2,7 @@ /// /// Represents the result of a file upload operation. -/// FileId follows URI format: {storage_type}://{path_to_resource}[?parameters] +/// FileId follows URI format: {storage_type}://{basket}/{tenant}/{path_to_resource}[?parameters] /// Examples: /// /// // PostgreSQL storage @@ -15,7 +15,7 @@ /// new StorageResult("file:///var/www/uploads/image.png", "/api/files/download/file/var/www/uploads/image.png", "file", DateTimeOffset.Now) /// /// -/// Unique file identifier in URI format: {storage_type}://{path}[?params] +/// Unique file identifier in URI format: {storage_type}://{basket}/{tenant}/{path}[?params] /// Publicly accessible URL for downloading the file /// Type of storage backend used ("pg", "s3", "file", "azure") /// Timestamp when the file was uploaded diff --git a/src/Sa.Media.FFmpeg/FFMpegOptions.cs b/src/Sa.Media.FFmpeg/FFMpegOptions.cs index a0f9365e..5ce1f4c6 100644 --- a/src/Sa.Media.FFmpeg/FFMpegOptions.cs +++ b/src/Sa.Media.FFmpeg/FFMpegOptions.cs @@ -14,7 +14,7 @@ public sealed record FFMpegOptions public TimeSpan? Timeout => TimeoutSeconds > 0 ? TimeSpan.FromSeconds(TimeoutSeconds.Value) - : default; + : null; // Валидация после десериализации public void Validate() diff --git a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs index 20efee16..115e72da 100644 --- a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs @@ -1,4 +1,4 @@ -using Sa.Classes; +using Sa.Media.FFmpeg.Services; using System.Diagnostics; namespace Sa.Media.FFmpeg; diff --git a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj index f6ff3b28..343e86cc 100644 --- a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj +++ b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.9.1 FFmpeg wrapper true win-x64;win-arm64;linux-x64;linux-arm64;osx-x64 @@ -104,7 +104,6 @@ - diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs index c17701e4..a041695f 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs @@ -1,4 +1,6 @@ -namespace Sa.Media.FFmpeg.Services; +using System.Runtime.CompilerServices; + +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutor(IFFRawExecutor executor) : IFFMpegExecutor { @@ -31,6 +33,8 @@ public async Task ConvertToPcmS16Le( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; var channelCount = outputChannelCount.HasValue ? $"-ac {outputChannelCount}" : string.Empty; var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + @@ -67,6 +71,8 @@ public async Task ConvertToMp3( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f mp3 {Libmp3lameArg()} -ar 16000 -b:a 128k {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); @@ -81,21 +87,34 @@ public async Task ConvertToOgg( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f ogg {LibopuArg(isLibopus)} {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); return result.StandardError; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CheckFiles(string inputFileName, string outputFileName) + { + if (string.Equals(Path.GetFullPath(inputFileName), Path.GetFullPath(outputFileName), StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Input and output files must be different", nameof(outputFileName)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string Libmp3lameArg() => Constants.IsOsLinux ? "-c:a libmp3lame" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string OverArg(bool isOverwrite) => isOverwrite ? "-y" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string LibopuArg(bool isLibopus) { if (!Constants.IsOsLinux) return string.Empty; return isLibopus ? "-c:a libopus" : "-c:a libvorbis"; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string QuotePath(string path) => $"\"{path}\""; } diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs index 5e8b91e1..023221af 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs @@ -1,6 +1,4 @@ -using Sa.Classes; - -namespace Sa.Media.FFmpeg.Services; +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutorFactory( IFFMpegLocator? mpegLocator = null, diff --git a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs index 013bc352..352a9ecf 100644 --- a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs @@ -1,5 +1,4 @@ -using Sa.Classes; -using System.Diagnostics; +using System.Diagnostics; using System.Text; namespace Sa.Media.FFmpeg.Services; diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs new file mode 100644 index 00000000..05a5adcf --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs @@ -0,0 +1,21 @@ +namespace Sa.Media.FFmpeg.Services; + +/// +/// Represents the result of a process execution, including exit code and output streams. +/// +public record ProcessExecutionResult( + /// + /// The exit code returned by the executed process. + /// A value of 0 typically indicates success. + /// + int ExitCode, + + /// + /// The standard output (stdout) captured from the process. + /// + string StandardOutput, + + /// + /// The standard error (stderr) captured from the process. + /// + string StandardError); diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs new file mode 100644 index 00000000..7f820dbe --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs @@ -0,0 +1,438 @@ +using System.Diagnostics; +using System.Text; + +namespace Sa.Media.FFmpeg.Services; + +internal interface IProcessExecutor +{ + /// + /// Executes a process with real-time output handling + /// + Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + /// + /// Executes a process and returns complete output + /// + async Task ExecuteWithResultAsync( + ProcessStartInfo startInfo + , bool captureErrorOutput = true + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + var output = new StringBuilder(); + var error = new StringBuilder(); + + var exitcode = await ExecuteAsync( + startInfo + , s => output.AppendLine(s) + , e => error.AppendLine(e) + , timeout + , cancellationToken + ).ConfigureAwait(false); + + var result = new ProcessExecutionResult( + exitcode, + StandardOutput: output.ToString(), + StandardError: error.ToString()); + + if (result.ExitCode == 0 || captureErrorOutput) return result; + + throw new ProcessExecutionResultException(result); + } + + /// + /// Executes stdout as a stream. + /// Stderr is captured and checked on completion + /// + Task ExecuteStdOutAsync( + ProcessStartInfo startInfo + , Stream inputStream + , Func onOutput + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + static IProcessExecutor Default { get; } = new ProcessExecutor(); +} + + + +internal sealed class ProcessExecutor : IProcessExecutor +{ + public async Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + + cancellationToken.ThrowIfCancellationRequested(); + + startInfo.RedirectStandardOutput = outputDataReceived != null; + startInfo.RedirectStandardError = errorDataReceived != null; + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException($"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + + int exitCode = await ExecuteProcessWithHandlersAsync(process, outputDataReceived, errorDataReceived, timeout, cancellationToken) + .ConfigureAwait(false); + + return exitCode; + } + + private static async Task ExecuteProcessWithHandlersAsync( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + int exitCode; + + try + { + await Run(process, outputDataReceived, errorDataReceived, timeout, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new ProcessTimeoutException("Process execution timed out"); + } + catch (Exception ex) + { + throw new ProcessExecutionException(process.ExitCode, "Process execution failed", ex); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + return exitCode; + } + + private static async Task Run( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + var outputCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var errorCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (outputDataReceived != null) + { + SetupOutputDataReceived(process, outputDataReceived, outputCompletion); + } + + if (errorDataReceived != null) + { + SetupErrorDataReceived(process, errorDataReceived, errorCompletion); + } + + if (process.StartInfo.RedirectStandardOutput) + { + process.BeginOutputReadLine(); + } + + if (process.StartInfo.RedirectStandardError) + { + process.BeginErrorReadLine(); + } + + using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + var waitTask = process.WaitForExitAsync(linkedCts.Token); + + // Wait for all streams to finish reading + List outputTasks = [waitTask]; + + if (outputDataReceived != null) + { + outputTasks.Add(outputCompletion.Task); + } + + if (errorDataReceived != null) + { + outputTasks.Add(errorCompletion.Task); + } + + await Task.WhenAll(outputTasks).ConfigureAwait(false); + } + + private static void SetupErrorDataReceived( + Process process, + Action errorDataReceived, + TaskCompletionSource errorCompletion) + { + process.ErrorDataReceived += (_, e) => + { + if (e.Data != null) errorDataReceived(e.Data); + else errorCompletion.TrySetResult(true); + }; + } + + private static void SetupOutputDataReceived( + Process process, + Action outputDataReceived, + TaskCompletionSource outputCompletion) + { + process.OutputDataReceived += (_, e) => + { + if (e.Data != null) outputDataReceived(e.Data); + else outputCompletion.TrySetResult(true); + }; + } + + /// + /// Запускает процесс, и передаёт поток stdout в callback. + /// Поток stderr собирается автоматически. + /// При завершении — проверяется код возврата. + /// + /// Настройки процесса. + /// Поток для stdin + /// Callback, получающий stdout. Должен быть асинхронным. + /// Задача, завершающаяся после обработки потока и проверки результата. + public async Task ExecuteStdOutAsync( + ProcessStartInfo startInfo, + Stream inputStream, + Func onOutput, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Настройка + startInfo.RedirectStandardInput = true; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + + + var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException( + $"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + StringBuilder stderrBuilder = new(); + int exitCode; + try + { + using var timeoutCts = timeout.HasValue + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + + List backgroundTasks = [ + WriteToStdInAsync(process, inputStream, linkedCts.Token), // write stdin + ReadStandardErrorToBuilderAsync(process, stderrBuilder, linkedCts.Token), // read stderr + ]; + + Stream stdoutStream = process.StandardOutput.BaseStream; + try + { + await onOutput(stdoutStream, linkedCts.Token).ConfigureAwait(false); + } + finally + { + await stdoutStream.DisposeAsync(); + } + await Task.WhenAll(backgroundTasks).ConfigureAwait(false); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + if (exitCode != 0) + { + throw new ProcessExecutionException(exitCode, $"Process failed (exit={exitCode}): {stderrBuilder}"); + } + } + + + private static async Task ReadStandardErrorToBuilderAsync( + Process process, + StringBuilder errorBuilder, + CancellationToken cancellationToken = default) + { + try + { + string error = await process.StandardError.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(error)) + { + errorBuilder.Append(error); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (errorBuilder.Length == 0) + errorBuilder.Append("stderr: read interrupted (cancellation)."); + } + catch (Exception ex) + { + errorBuilder.Append($"stderr: read failed: {ex.Message}"); + } + } + + /// + /// Асинхронно записывает входной поток в stdin процесса. + /// Автоматически закрывает stdin после завершения. + /// + private static async Task WriteToStdInAsync( + Process process, + Stream inputStream, + CancellationToken cancellationToken = default) + { + try + { + // input to stdin + await using (inputStream.ConfigureAwait(false)) + { + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken) + .ConfigureAwait(false); + } + + // Завершаем запись + await process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false); + process.StandardInput.Close(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + process.StandardInput.Close(); + } + catch (IOException) + { + process.StandardInput.Close(); + } + catch (Exception ex) + { + process.StandardInput.Close(); + throw new IOException("Failed to write input stream to process stdin.", ex); + } + } + + private static int SafeDisposeProcess(Process process) + { + int exitCode = -1; + + try + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + + process.StandardInput?.Close(); + process.StandardOutput?.Close(); + + // graceful shutdown + if (process.WaitForExit(500)) + { + exitCode = process.ExitCode; + return exitCode; + } + + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + throw; + } + catch (NotSupportedException) + { + process.Kill(); + } + catch (UnauthorizedAccessException) + { + return exitCode; + } + + if (process.WaitForExit(2000)) + { + exitCode = process.ExitCode; + } + else + { + Console.WriteLine("Process did not terminate after Kill()"); + } + } + catch (InvalidOperationException) + { + Console.WriteLine("Process is invalid or already disposed."); + } + catch (Exception ex) + { + Console.WriteLine($"Unexpected error during process termination: {ex.Message}"); + } + finally + { + try + { + process.Dispose(); + } + catch + { + // skeep + } + } + + return exitCode; + } +} + + +// Custom exceptions +public sealed class ProcessExecutionException(int exitCode, string message, Exception? inner = null) + : Exception(message, inner) +{ + public int Exitcode => exitCode; +} + +public sealed class ProcessExecutionResultException(ProcessExecutionResult result) + : Exception($"Process failed (exit={result.ExitCode}): {result.StandardError}") +{ + public ProcessExecutionResult Result { get; } = result; +} + +public sealed class ProcessStartException(string message) : IOException(message) +{ +} + +public sealed class ProcessTimeoutException(string message) : TimeoutException(message) +{ +} diff --git a/src/Sa.Media.FFmpeg/Setup.cs b/src/Sa.Media.FFmpeg/Setup.cs index a19ad15e..6d78ef8b 100644 --- a/src/Sa.Media.FFmpeg/Setup.cs +++ b/src/Sa.Media.FFmpeg/Setup.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using Sa.Classes; using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpeg; diff --git a/src/Sa/Classes/IProcessExecutor.cs b/src/Sa/Classes/IProcessExecutor.cs index 97d369d2..8f6a3c6b 100644 --- a/src/Sa/Classes/IProcessExecutor.cs +++ b/src/Sa/Classes/IProcessExecutor.cs @@ -172,7 +172,7 @@ private static async Task Run( process.BeginErrorReadLine(); } - using var timeoutCts = timeout.HasValue + using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero ? new CancellationTokenSource(timeout.Value) : null; diff --git a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs index 734341ac..617370b0 100644 --- a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs +++ b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs @@ -1,5 +1,5 @@ -using Sa.Classes; -using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpegTests; From 4c894c90fdcc86a22da88580a95cd3a41124c597 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 31 Mar 2026 14:30:18 +0300 Subject: [PATCH 04/28] fix mmpeg --- .../Domain/StorageResult.cs | 4 +- src/Sa.Media.FFmpeg/FFMpegOptions.cs | 2 +- src/Sa.Media.FFmpeg/IFFRawExecutor.cs | 2 +- src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj | 3 +- .../Services/FFMpegExecutor.cs | 21 +- .../Services/FFMpegExecutorFactory.cs | 4 +- src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs | 3 +- .../Services/ProcessExecutionResult.cs | 21 + .../Services/ProcessExecutor.cs | 438 ++++++++++++++++++ src/Sa.Media.FFmpeg/Setup.cs | 1 - src/Sa/Classes/IProcessExecutor.cs | 2 +- .../FFMpegProcessorTests.cs | 5 +- .../Sa.Media.FFmpegTests.csproj | 3 + src/Tests/Sa.Media.FFmpegTests/data/12345.wav | Bin 0 -> 25388 bytes 14 files changed, 493 insertions(+), 16 deletions(-) create mode 100644 src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs create mode 100644 src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs create mode 100644 src/Tests/Sa.Media.FFmpegTests/data/12345.wav diff --git a/src/Sa.HybridFileStorage/Domain/StorageResult.cs b/src/Sa.HybridFileStorage/Domain/StorageResult.cs index d6d49dfb..cd6ceddf 100644 --- a/src/Sa.HybridFileStorage/Domain/StorageResult.cs +++ b/src/Sa.HybridFileStorage/Domain/StorageResult.cs @@ -2,7 +2,7 @@ /// /// Represents the result of a file upload operation. -/// FileId follows URI format: {storage_type}://{path_to_resource}[?parameters] +/// FileId follows URI format: {storage_type}://{basket}/{tenant}/{path_to_resource}[?parameters] /// Examples: /// /// // PostgreSQL storage @@ -15,7 +15,7 @@ /// new StorageResult("file:///var/www/uploads/image.png", "/api/files/download/file/var/www/uploads/image.png", "file", DateTimeOffset.Now) /// /// -/// Unique file identifier in URI format: {storage_type}://{path}[?params] +/// Unique file identifier in URI format: {storage_type}://{basket}/{tenant}/{path}[?params] /// Publicly accessible URL for downloading the file /// Type of storage backend used ("pg", "s3", "file", "azure") /// Timestamp when the file was uploaded diff --git a/src/Sa.Media.FFmpeg/FFMpegOptions.cs b/src/Sa.Media.FFmpeg/FFMpegOptions.cs index a0f9365e..5ce1f4c6 100644 --- a/src/Sa.Media.FFmpeg/FFMpegOptions.cs +++ b/src/Sa.Media.FFmpeg/FFMpegOptions.cs @@ -14,7 +14,7 @@ public sealed record FFMpegOptions public TimeSpan? Timeout => TimeoutSeconds > 0 ? TimeSpan.FromSeconds(TimeoutSeconds.Value) - : default; + : null; // Валидация после десериализации public void Validate() diff --git a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs index 20efee16..115e72da 100644 --- a/src/Sa.Media.FFmpeg/IFFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/IFFRawExecutor.cs @@ -1,4 +1,4 @@ -using Sa.Classes; +using Sa.Media.FFmpeg.Services; using System.Diagnostics; namespace Sa.Media.FFmpeg; diff --git a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj index f6ff3b28..343e86cc 100644 --- a/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj +++ b/src/Sa.Media.FFmpeg/Sa.Media.FFmpeg.csproj @@ -3,7 +3,7 @@ - 0.9.0 + 0.9.1 FFmpeg wrapper true win-x64;win-arm64;linux-x64;linux-arm64;osx-x64 @@ -104,7 +104,6 @@ - diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs index c17701e4..a041695f 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutor.cs @@ -1,4 +1,6 @@ -namespace Sa.Media.FFmpeg.Services; +using System.Runtime.CompilerServices; + +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutor(IFFRawExecutor executor) : IFFMpegExecutor { @@ -31,6 +33,8 @@ public async Task ConvertToPcmS16Le( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var sampleRate = outputSampleRate.HasValue ? $"-ar {outputSampleRate}" : string.Empty; var channelCount = outputChannelCount.HasValue ? $"-ac {outputChannelCount}" : string.Empty; var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + @@ -67,6 +71,8 @@ public async Task ConvertToMp3( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f mp3 {Libmp3lameArg()} -ar 16000 -b:a 128k {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); @@ -81,21 +87,34 @@ public async Task ConvertToOgg( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { + CheckFiles(inputFileName, outputFileName); + var cmd = $"{OverArg(isOverwrite)} {Constants.CleanBannerFlags} -i {QuotePath(inputFileName)} " + $"-f ogg {LibopuArg(isLibopus)} {QuotePath(outputFileName)}"; var result = await executor.ExecuteAsync(cmd, timeout: timeout, cancellationToken: cancellationToken); return result.StandardError; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CheckFiles(string inputFileName, string outputFileName) + { + if (string.Equals(Path.GetFullPath(inputFileName), Path.GetFullPath(outputFileName), StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Input and output files must be different", nameof(outputFileName)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string Libmp3lameArg() => Constants.IsOsLinux ? "-c:a libmp3lame" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string OverArg(bool isOverwrite) => isOverwrite ? "-y" : string.Empty; + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string LibopuArg(bool isLibopus) { if (!Constants.IsOsLinux) return string.Empty; return isLibopus ? "-c:a libopus" : "-c:a libvorbis"; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string QuotePath(string path) => $"\"{path}\""; } diff --git a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs index 5e8b91e1..023221af 100644 --- a/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs +++ b/src/Sa.Media.FFmpeg/Services/FFMpegExecutorFactory.cs @@ -1,6 +1,4 @@ -using Sa.Classes; - -namespace Sa.Media.FFmpeg.Services; +namespace Sa.Media.FFmpeg.Services; internal sealed class FFMpegExecutorFactory( IFFMpegLocator? mpegLocator = null, diff --git a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs index 013bc352..352a9ecf 100644 --- a/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/FFRawExecutor.cs @@ -1,5 +1,4 @@ -using Sa.Classes; -using System.Diagnostics; +using System.Diagnostics; using System.Text; namespace Sa.Media.FFmpeg.Services; diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs new file mode 100644 index 00000000..05a5adcf --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutionResult.cs @@ -0,0 +1,21 @@ +namespace Sa.Media.FFmpeg.Services; + +/// +/// Represents the result of a process execution, including exit code and output streams. +/// +public record ProcessExecutionResult( + /// + /// The exit code returned by the executed process. + /// A value of 0 typically indicates success. + /// + int ExitCode, + + /// + /// The standard output (stdout) captured from the process. + /// + string StandardOutput, + + /// + /// The standard error (stderr) captured from the process. + /// + string StandardError); diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs new file mode 100644 index 00000000..7f820dbe --- /dev/null +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs @@ -0,0 +1,438 @@ +using System.Diagnostics; +using System.Text; + +namespace Sa.Media.FFmpeg.Services; + +internal interface IProcessExecutor +{ + /// + /// Executes a process with real-time output handling + /// + Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + /// + /// Executes a process and returns complete output + /// + async Task ExecuteWithResultAsync( + ProcessStartInfo startInfo + , bool captureErrorOutput = true + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + var output = new StringBuilder(); + var error = new StringBuilder(); + + var exitcode = await ExecuteAsync( + startInfo + , s => output.AppendLine(s) + , e => error.AppendLine(e) + , timeout + , cancellationToken + ).ConfigureAwait(false); + + var result = new ProcessExecutionResult( + exitcode, + StandardOutput: output.ToString(), + StandardError: error.ToString()); + + if (result.ExitCode == 0 || captureErrorOutput) return result; + + throw new ProcessExecutionResultException(result); + } + + /// + /// Executes stdout as a stream. + /// Stderr is captured and checked on completion + /// + Task ExecuteStdOutAsync( + ProcessStartInfo startInfo + , Stream inputStream + , Func onOutput + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default); + + + static IProcessExecutor Default { get; } = new ProcessExecutor(); +} + + + +internal sealed class ProcessExecutor : IProcessExecutor +{ + public async Task ExecuteAsync( + ProcessStartInfo startInfo + , Action? outputDataReceived = null + , Action? errorDataReceived = null + , TimeSpan? timeout = null + , CancellationToken cancellationToken = default) + { + + cancellationToken.ThrowIfCancellationRequested(); + + startInfo.RedirectStandardOutput = outputDataReceived != null; + startInfo.RedirectStandardError = errorDataReceived != null; + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException($"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + + int exitCode = await ExecuteProcessWithHandlersAsync(process, outputDataReceived, errorDataReceived, timeout, cancellationToken) + .ConfigureAwait(false); + + return exitCode; + } + + private static async Task ExecuteProcessWithHandlersAsync( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + int exitCode; + + try + { + await Run(process, outputDataReceived, errorDataReceived, timeout, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new ProcessTimeoutException("Process execution timed out"); + } + catch (Exception ex) + { + throw new ProcessExecutionException(process.ExitCode, "Process execution failed", ex); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + return exitCode; + } + + private static async Task Run( + Process process, + Action? outputDataReceived, + Action? errorDataReceived, + TimeSpan? timeout, + CancellationToken cancellationToken) + { + var outputCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var errorCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (outputDataReceived != null) + { + SetupOutputDataReceived(process, outputDataReceived, outputCompletion); + } + + if (errorDataReceived != null) + { + SetupErrorDataReceived(process, errorDataReceived, errorCompletion); + } + + if (process.StartInfo.RedirectStandardOutput) + { + process.BeginOutputReadLine(); + } + + if (process.StartInfo.RedirectStandardError) + { + process.BeginErrorReadLine(); + } + + using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + var waitTask = process.WaitForExitAsync(linkedCts.Token); + + // Wait for all streams to finish reading + List outputTasks = [waitTask]; + + if (outputDataReceived != null) + { + outputTasks.Add(outputCompletion.Task); + } + + if (errorDataReceived != null) + { + outputTasks.Add(errorCompletion.Task); + } + + await Task.WhenAll(outputTasks).ConfigureAwait(false); + } + + private static void SetupErrorDataReceived( + Process process, + Action errorDataReceived, + TaskCompletionSource errorCompletion) + { + process.ErrorDataReceived += (_, e) => + { + if (e.Data != null) errorDataReceived(e.Data); + else errorCompletion.TrySetResult(true); + }; + } + + private static void SetupOutputDataReceived( + Process process, + Action outputDataReceived, + TaskCompletionSource outputCompletion) + { + process.OutputDataReceived += (_, e) => + { + if (e.Data != null) outputDataReceived(e.Data); + else outputCompletion.TrySetResult(true); + }; + } + + /// + /// Запускает процесс, и передаёт поток stdout в callback. + /// Поток stderr собирается автоматически. + /// При завершении — проверяется код возврата. + /// + /// Настройки процесса. + /// Поток для stdin + /// Callback, получающий stdout. Должен быть асинхронным. + /// Задача, завершающаяся после обработки потока и проверки результата. + public async Task ExecuteStdOutAsync( + ProcessStartInfo startInfo, + Stream inputStream, + Func onOutput, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Настройка + startInfo.RedirectStandardInput = true; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + + + var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + + if (!process.Start()) + { + process.Dispose(); + throw new ProcessStartException( + $"Failed to start process: '{startInfo.FileName}' with arguments '{startInfo.Arguments}'"); + } + + StringBuilder stderrBuilder = new(); + int exitCode; + try + { + using var timeoutCts = timeout.HasValue + ? new CancellationTokenSource(timeout.Value) + : null; + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts?.Token ?? CancellationToken.None); + + + List backgroundTasks = [ + WriteToStdInAsync(process, inputStream, linkedCts.Token), // write stdin + ReadStandardErrorToBuilderAsync(process, stderrBuilder, linkedCts.Token), // read stderr + ]; + + Stream stdoutStream = process.StandardOutput.BaseStream; + try + { + await onOutput(stdoutStream, linkedCts.Token).ConfigureAwait(false); + } + finally + { + await stdoutStream.DisposeAsync(); + } + await Task.WhenAll(backgroundTasks).ConfigureAwait(false); + } + finally + { + exitCode = SafeDisposeProcess(process); + } + + if (exitCode != 0) + { + throw new ProcessExecutionException(exitCode, $"Process failed (exit={exitCode}): {stderrBuilder}"); + } + } + + + private static async Task ReadStandardErrorToBuilderAsync( + Process process, + StringBuilder errorBuilder, + CancellationToken cancellationToken = default) + { + try + { + string error = await process.StandardError.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(error)) + { + errorBuilder.Append(error); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (errorBuilder.Length == 0) + errorBuilder.Append("stderr: read interrupted (cancellation)."); + } + catch (Exception ex) + { + errorBuilder.Append($"stderr: read failed: {ex.Message}"); + } + } + + /// + /// Асинхронно записывает входной поток в stdin процесса. + /// Автоматически закрывает stdin после завершения. + /// + private static async Task WriteToStdInAsync( + Process process, + Stream inputStream, + CancellationToken cancellationToken = default) + { + try + { + // input to stdin + await using (inputStream.ConfigureAwait(false)) + { + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken) + .ConfigureAwait(false); + } + + // Завершаем запись + await process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false); + process.StandardInput.Close(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + process.StandardInput.Close(); + } + catch (IOException) + { + process.StandardInput.Close(); + } + catch (Exception ex) + { + process.StandardInput.Close(); + throw new IOException("Failed to write input stream to process stdin.", ex); + } + } + + private static int SafeDisposeProcess(Process process) + { + int exitCode = -1; + + try + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + + process.StandardInput?.Close(); + process.StandardOutput?.Close(); + + // graceful shutdown + if (process.WaitForExit(500)) + { + exitCode = process.ExitCode; + return exitCode; + } + + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + if (process.HasExited) + { + exitCode = process.ExitCode; + return exitCode; + } + throw; + } + catch (NotSupportedException) + { + process.Kill(); + } + catch (UnauthorizedAccessException) + { + return exitCode; + } + + if (process.WaitForExit(2000)) + { + exitCode = process.ExitCode; + } + else + { + Console.WriteLine("Process did not terminate after Kill()"); + } + } + catch (InvalidOperationException) + { + Console.WriteLine("Process is invalid or already disposed."); + } + catch (Exception ex) + { + Console.WriteLine($"Unexpected error during process termination: {ex.Message}"); + } + finally + { + try + { + process.Dispose(); + } + catch + { + // skeep + } + } + + return exitCode; + } +} + + +// Custom exceptions +public sealed class ProcessExecutionException(int exitCode, string message, Exception? inner = null) + : Exception(message, inner) +{ + public int Exitcode => exitCode; +} + +public sealed class ProcessExecutionResultException(ProcessExecutionResult result) + : Exception($"Process failed (exit={result.ExitCode}): {result.StandardError}") +{ + public ProcessExecutionResult Result { get; } = result; +} + +public sealed class ProcessStartException(string message) : IOException(message) +{ +} + +public sealed class ProcessTimeoutException(string message) : TimeoutException(message) +{ +} diff --git a/src/Sa.Media.FFmpeg/Setup.cs b/src/Sa.Media.FFmpeg/Setup.cs index a19ad15e..6d78ef8b 100644 --- a/src/Sa.Media.FFmpeg/Setup.cs +++ b/src/Sa.Media.FFmpeg/Setup.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using Sa.Classes; using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpeg; diff --git a/src/Sa/Classes/IProcessExecutor.cs b/src/Sa/Classes/IProcessExecutor.cs index 97d369d2..8f6a3c6b 100644 --- a/src/Sa/Classes/IProcessExecutor.cs +++ b/src/Sa/Classes/IProcessExecutor.cs @@ -172,7 +172,7 @@ private static async Task Run( process.BeginErrorReadLine(); } - using var timeoutCts = timeout.HasValue + using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero ? new CancellationTokenSource(timeout.Value) : null; diff --git a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs index 734341ac..6cb97691 100644 --- a/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs +++ b/src/Tests/Sa.Media.FFmpegTests/FFMpegProcessorTests.cs @@ -1,5 +1,5 @@ -using Sa.Classes; -using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg; +using Sa.Media.FFmpeg.Services; namespace Sa.Media.FFmpegTests; @@ -48,6 +48,7 @@ await Processor.ConvertToPcmS16Le( [Theory] [InlineData("./data/input.mp3")] [InlineData("./data/gsm.wav")] + [InlineData("./data/12345.wav")] public async Task ConvertToPcm16Wav_CallsFFmpegAsStream(string testFilePath) { var ext = Path.GetExtension(testFilePath).TrimStart('.'); diff --git a/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj b/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj index 1a908a16..cfc1d536 100644 --- a/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj +++ b/src/Tests/Sa.Media.FFmpegTests/Sa.Media.FFmpegTests.csproj @@ -11,6 +11,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Tests/Sa.Media.FFmpegTests/data/12345.wav b/src/Tests/Sa.Media.FFmpegTests/data/12345.wav new file mode 100644 index 0000000000000000000000000000000000000000..5045628ede2e90ff569c609c91ca95129939cb3c GIT binary patch literal 25388 zcmZU*1$-1o*grnAavHg~Cq!}g;!@n*-AZvN?vzrb6e&{NwLmGwiUw_QC=`c;cyey* zJO5{smw)^I-rLXK?d{IYBQv}EePpIb`!;RTMk3U=MV~LnOq@~9f)K*6HBUllaZ`jS zWI>~b&lrxN3?UR!35F7%j|lwJjuk?~PL*EK&Xnz1xw6v#{jZjmEi3VJe5O@)td;%uY#A42wLa_iGgXbB&y=-tWz;kpG|0e| z^7rqtCc%HD*ZBDFhy4Gw{wwQ$X=tVYck8H>)@N?DfB&xr%lOwylG{- z&+`4Z9qstP@|MZHi*c|Bk=UIw_;@Z`)-Q{vA*M-WghJnl8#X z`L|w~WvpX(&urYo%vTk8nw?dex~&A@8>ql&S+y-qgD2=>`C(ZsiMhT z*0cXF#j+Cr9aCkk{L9t9BkA+GG7kUCd0D?|{r4}o+V!$e8BdyKnEyOIG#!@pgw_jX zJyh0$c3h^Rf1jS3RR2o%`9A%8ca_!q+{(W_{NJ%zCYPpL@qfk}jr7QXSlG-kF0_wU zM!WMgz7*K}L@515pgjudkP*?Pr7+@%NO-tcc(_i84lz(uyDQ2*+J7Fdb3`WY(l$gW zCqNro>j)?Y!p4+R)a(^@;&B#X!Hv!~{nI zaDkxxP$;L3t{^yPL~h{E2y__a1)jo6b)sn?q?CUe=n_%@8q|T5I@m;LRg+138iWGJ zn)VUYH^3E5E(VUYXBH2;3L5moS?%7?`d{l4J^TuAq(gB~OY2`JXhpj-SZI^Lu?iB| zp_WFA16d+aJe1UQDnrde(4-E8s$hvBf`peA%*>7G#0H`?!s1trtZIk!6o`D!S8)v3xi z@6{p3;>c@*Ly(h^-__`d+a5(p;0Q zO^bTvF3M|Ri?EGz&Zpxex1`$Bhs2+W{Kqnjsjkc>ec3rp-`*==!_jA?wh5nk}rtZR`zq6L-iG?c)8aW+gx}{^%kkkt4a_YON z)PP^v`{)rlfa2Ipqg~%hzf0%E&*fD@H*TEoji-(0ox6oA(|)bklKW+W#ag?NvhP;9 z8#hKyNqLbrKkZr~7jY?YHd}zQ(R1`3FJ<~OwVB_k3gk!SzF3>P>f7)A!mT*wI9A*4 z6(~863WgR*MK!%Q$v=TN<9nubNiR%)mN+!(SKzd%cHWpYo6 zln(mucqckHShp9u^2w)%xzh{u#aCVZr7EW0(Waz3>6I&VOmih%jkpukSvQpoQ-({G z;DJ+1S)*FjOfrGIRQ?vv`zcS1y;jkjybqtovqFPOH=7UhTWYpAyHar>PaJ z(b(U7ETD4G7eRf3>I8KT3=0@y+G_Yo*Oxg&HODK-P}QPTkTwe@UhkjcEA&Qr>v@0i zntea`R`_$d+d?z>x!Ml9>EXHohDxSw=J5e91BwDR1!S6cnrazW=ufe~(sS{6QbjRI z&4oo=ia*LXz#HkU=>6VX)tBP$&pqTvh+X8t>H>6(y1?wwjWIkmI!x!y#sJQ|#++$d zX4Dxf>ynuQe2#2X4oFXhzI;`#rT?(6gD=I`()YWsz2CwqT#7JOERi-T-;ja04V}Pd z>e?AL8h4l~ndh6AnbXaSO{0t!!+M>KNvGT4ugOt(nf4W5@GH3f{x!Y`pUWHO`_9+P zKZ4uK$BI{^WlC>S7hC9e%q(4=KG7I!`ok1$rp@O~*c4{`pdY2X#{@u2%gIxvvV2(V zA!P71xgY)M{s4b%|2}^^E|sq@EEQi!9h66ELv$Vo&{LR0>|Na}{YFEIafGp>ku|P1 z+|b|DEn^dy@2U5wJXxot$xbm}&b1Mqpu zLzidk>#FLD^-~RJ3}+3i3=xK5`pLTfEW>P}EWnql9G1sOEyVW15`ad>x%J#JPR}3b z7Yp;mozg=&O6^ZJqkCAPBAHOu#~#qt(67;7(O=ci(}(CM>Q=K0nWl6xUXQAg56V9I zdugxuQfMSR;eX?=^Ng@kh!y`9k4n4bi%On~P+i=a>O|LPqSzvKzwVCsdwR>>K4oYGhxE%g)+3*7~k7x*uP141*=BxOlCGOtu3ozVAqH?@);&s1WM zvnE|%T_c^J{e}%?Co|LO6VzP13RNb_YNXPq{>Fwr1Z76*&(#LiL;xj-gLiE1Mz z{1*RBEub4Sdl{8!%_g%(_7GEvnL+!gk<`!FgQ}vR$t%^UR#z^|J>=)o8|j23N*&~n z@;D_=DOUTE35dbZ@GMG42hpi?3|)@SqrRmOwHupoD|C?Lshia2ib1WUOjVjGEtF-- z6{UqbL;aR~PZpuKXg{t;MNreIVblVsmqfLpuH!M7!4r_3h^mg1Q}-#46tAqv+mw5X ztejB$k-20JdV<374ZH)N!zb`M9D#>p8^U-y>VkeJ4z&*XLS3TLY7cdRI$u4fE>b6x zAiyvc0EM;0U!vW(53Y{?z*o=&{3oh|SD+4PKH5#*kQ6e5v~XeydZdXO&aGWCekNPREgRwgT7D=$^AG9C2* zMAHW^K#k~)cq$!D@4+UvIvPih#rMew>W#966w0gQ3Ccf$N$xIsgi1<^^g-=No|9eF zAzX!7#Y|%A=`)yrbdQ-><~JrE8R`0@I=Q$7OUK za(roB;235<>FB{7JI;~pgTPTr7oG2Rh)hSAuHvuEOmiK}vgp{vKa;AY>Qav-_Ko{9 ztebf^{gag8d*%3OD{pOQO|rMOU$?cfjVqZ_bhr=|Zm>mok8|7cd}BLHbo9`~jMO8k z4bvl1q`1M6j|0M)k@6>RPiL<6dut`@JL_0SKkKlPVI^S&b@L|`5c^2ad49OgY;GKO zCjlpqN`IMgA-%ZV?U@}?mt$S>{Y#ZzyN)8sz%-{ZLM^4Sc6;6Y{ zC8`~8EOb*sUaG!=Tq&_aMEZz?suAVQHq=MF>&mj_SVKx8Y?5_^O)mbBpOn8VYeP0y zz&eKc(oxHR4WZBDW79(`46VdfbfkTo&@6h5 z`JWnQH_CI`kawe;XEcRYi7iZPQlVa@l1gndDkY7L3=X%2Ab@RQf)>MnZ>(uS^5+F#f7*8BqGC z2^4Kr~6aIp`m8-wqV0D-5D4AFkmUkt4XV#Xi2RRLGvwchD>bkEit)q+M zC#QUu5u33!l`8i*HYfCeQKGQ2&)?DA$63qK+19>fSJ8_6EuZ4D`eirHZCae}SuPHw z4w?!>m&CXeJjrX*mZVlG_b_f!q$7B&p%}lFf8iBxM|WQ*XUixlUnu8R`P4Fd@&HCiaWR(a#hr1ofyc4wT!u)_$j$bO10$Jq}g$|BF+T2G#z1*h*QY+CAllx z7Z-;XJk3$EI(+Q#F)Qn2uD^J%Ya{<0Ei(2BIUKb+VMhv^{vjI2F?oqyi;n|FacjWq_j>GpAw9z zI3pxOO`VtQOsF5dC1k5PO7}adE);pDIb6kqLC^O;e*b>UyN>UVe*7Z$b#YDiufj+C zrztNqIA&eqj8rP)M*5M|>~cl%hNx2()_j=VM6&rip2?24#Xa*ke7f|p&-?lBCcJ<5 zu}`k2c)2@ROs7;+YS`8oA#p-#wT#Lc5ozU;gA>L@{b^B5BiQHaA%26$YiEnc+PL)3qD@TSy;q5mk9!nGSv({7qd2LT!Kd{H$H9uxaYm$edhaLKIVVwUigP&EoUHo_0=rTqoybPk~}NDd--k|zo*tJ zH!E&=gf4i8VI6yn$MJK?m4fQIH?yl` zP5xNz+QmWd4`-?+0C=sW-ZD7kaNGFvaO0|i_jho(svE&6@E6RJ3QrkrQAraR<2h< z^_Zma-9dpy9qokC{=&V&KBhRnfXnTj6aT3&yI; zic*W?O3qqe*;~09d7p8);1s$_|Dr!=P7Hn&k{7luVrFDWW9^K+fj!i5(fQUL0ZRja3d{)nBEV~EXY8ksWlvDaXur}@sw#}+e7<{LpJ#(-rAPAkyhHsK z{-tnG+Nk_YHsG%Tp&!+~)E_YnF#c(D8qXW+8W$Rt>D%buFn#C?I3BH5LzK(X6tSx? ziNC@P;Tmx(xIq3GKSLNHu9v*>WN>f}!$x{Jvx)shH%Ct!MjA#Lk__AQA9WI&3#hsR zbqP>bhI(E8S(+j45O{tzU!HHm@8$al$zVy=mpds3)M8Q-cc8A(&DgfO0R1BUZv9Na z=52II>{WUzRS%yfJypB>N@^qR6um-$@Ir72RmF{BMd=r*t^8V6lo})xoxyXd)ASkU zIlEu~oAIFWq`n)wfcE2Is#{(`cF4brHN_YyT#1)g%AA}EUY$j9qWT@6$hBw(?nasE zwbT}R1N&N+t(&B~!@g!inR9p?T22zxB)N|?RyrWQ5xx;F3WLP}@rkfe94qBZZ)GD{ zhVy8hZj*ky;gzw1X@~KY;e>vhu0J!23cy!LebpuZBYhz)5#I{C_$Hx~^1F3gi`XW!WKFLEq4@5L;oDjv_6_1_s{P5aEP z0xp@4nD!euT?Q*r|DX|Ml6p;Ptt?jRD{WvOC=Zo5VIaTXU*LP-H*v&&g#S&vt&Bqh z=!dLF|JulzCYZkjD|@Z+l71L_g04foMSaL6wK@3ZIx5wb74kjlmH3@-kT2%$`0x7O z`_l!hU=~B=F{&TuG9SSRbQ_pu^d9Os_%kb! zQEIyCRQ{A-N;Rbp;?Kexz9RR^-+&9^O~Nx_h-8yf&@_4@J5}G&Kn&-N*NlgZPYhq^ z_klDcs4I9Y+#gHSCCW(Uu(DWQF4Ykq^Yi(s+!22=*MpnIR}qIuW_2w(Ko8WN(u;+sN-|Y9ZqQF-GwBUfO`J;xsT0Av+pE-2I?AiXXmOE{!%g$^z5;(|@KmLW6QnF< z8aA_E7~IBQriG^OO-)TPCd$}C&oTF*1%xuyO-i<$BL^yv-4_x1#yM^JyDR)QVoq2I)=F)`FaXf)$+PQA$?+ZC&a}vp`3}Zlj+J>Q7EaV zXpt9_Z2Sb@LCNGYaI#gL%T4s=xW01kv>hy-QryMX&66$UQh$eROdeP1yNX2#H-aZH zfyyND6n~7n#J86lP_Om#jNyi!)EPPs_nN!E+v=L*df@unwZ?vv(_y~8u8}T}JKcX* zd2AXM{#RIvSyilVpW~(TvbzfSuZp;_Lb$Jf$)@Z9S#t`n_#)`(5!qG0Xk4wqjFj{2 z@2>aOx6Tf}y24TQ7o#!ao2Umtx#~N|!jb^zTA!6$>%Zp<2A^0HJC)n$!-2fJo|V+4 z$e*edH`Lc#k=#_5Gkp0uUZMkjTzpFJ&)stG+vo_jWxnkgZck80sj$K?$*@0FW)UeE$=+H#8 z(D8NAG3#^B6uuG!{~Dli%3k-yoOf?}eynK?lpeCFk!jU;HO{WRC+4zz$$HB6qo*BT zPOZdt3CWMC8qrDD%YV--wJz6>qu*B*F(MzT1O|S zZ9T;WW3vp|+wulFS0hK%qgr1#DXDorW-lJ?1t@NB|+t@oBD}ePI68PnU9Q7ftgGH7(Al)M!lo?3954Pvxat4fhq>Jr}3u2Of=D z8ZpW=9=8+*xLIqRV!va9_!q>AR*6SlEsERaUe7s^=XI{9^W#R>uGZjerHWDW*h2| zi(coyD_G*XLYju$D8IYaKeZiBzR|04I3&|S;|-)3W_J86i=Q`_nBY?f}H zzMp=Pj@F%GO>9f1A$^(3rjAoBwmzW z$gdQ;S_kdHK8j+_G8Nci?0R+z8w8mSxgWRYg{u(dEQ*eJ=8+@3L&?Yn(wS<~Rh=5)uTgW^zigYIpNM({r zs*%p%Ydix{)LLj9+JJsRXV3w(3L>IiQGJw-QovVfgy^wCtR$OUCx^*8vVe>yBgu5K z72++ZYGYIkp1_CXCV4>qB+no|`ieXPPOp-4 z(wgV?l}lh}pY=Um7C$si->CtO4>xfOJY} zj9QC>2f#){IXlp@l3XaSN@MXHaPJ1bo$%`_<(CG^VZdJ) za2f{M(DFurEYNV5%BYp{Kxr?W(T=nX0Wfh2*_xGIzezQ7D66O)UdJ%X!f9DYX0sEjxpSd;=5cC;(c~!8eB1wJZlM4*@~zCfLouA0Y{F)e9pn z2l5JjQFlc`<>&$mrUQF@ZU&_L!S&Vz_hx@sbE>Sz*1 z+L3R_B~ll4#Z_=RCF374OZSEM(RwtLY$KAIftI10aR0OeDVh?G+7Rx`AIMU;zurS{ zw}nxjM=WHQx(8Zm13F)g3egj|6FZTw$z-*fx*mM?-NEnQ7_G)7_;;!Zvp^+;x`6lN zemD!M$bj!c59uL3(~Kx;sya@-A~lkaNgqYKJPv&Sz0g~#F?*9)tP3-=(G6uAupj7O zFonhwpWI2BC&mbI{FnYYzFFQJkIQTGXNdXaF8j*#h2>;8j#?GjHNqU46!1wm6kQk1 ze!aV`>ksEl*IjpxYnJPgE5S+IYuZyhnS!V;Fc8bLh^w*F;%g=R7E>qOY&oniK)vPj z+(5s}d)1!+nE#+MNI5D^6T;ki)>?(#bN|SBRnX7v5G%1?ht`YxkQ9^NJAF>MzhcGE zQ1c)(N1Wwf>)PsKJgvcsnj+m+qGhv`o`{^@?n=iqa=yTVUm4}KE2()--g#(m#2(KnZ$BacQKnX9_q z=01V8peB}Zix^ZsptfnJE{0x-{!}W-w?$s?3qfKnaiREByewW96){R`FS(^9@)V^9 zMBxUb2DlDYL@lE8X_01`>vVV8Np+>Hcm{rkIzu){6xjjKWwW|V8LaG4zEs94OO+fY zMqQ;wlT~CoDnVl)Go(5-g<4A8qyC~!P*{mu7K~Mo~Ve7A%DRWzM^_ZSpXJq z4JA!!pd3}gAc7>Squ{CdEi&R=I1i7aeuDQ}4rQfIQG=*ZY9XfZAT*EECcmixYJsv| zVU&FNitLcFxSTh8ye@W&>W2-O53wzIsfYphMA7A<8W756<7OR{$c#jzKx<4Vw%VK z98a3>k>j3Yv3Ievf%t>`N)`0mOt*twp;aQkjtq~8j7SPDHq|!#fJEh;_}pIyqQ+-@ zo&7WTbNIPV$HZf}rpqW`sji+8fu=Zv)Pwr{s?1MgfIcbJT!CmDYZxfGTbvpimieHLeo zei6DoXqEmHekwoZZhFUiuDD(91Ku58(Ob=%4R*;^YeC6bN4gXHBg6nSNVm;ADfD_o zNbI-q-Jjugi+DS2@B&6#nZ6^BJNneG|gbTsW*k1zSZy?_{G`LZSr*UeC4U6(yD9fl*K~J= zXNG69JH?gdcwk>=pXO-liSQ2)??Uvhmbt{THT=7n>#^12nAqi!yF%Xw{G)G=zmhBR zeY{2Pldh?*eeU~i$*s6MI0KynoLB6#TwQ&G{Vq8i8`<6gr-K`Z&xnqQ85uJ_+7Qt@ zgb668$M^?1lmFgZ-W%aIxp#Q>x;c-}J=7&Tzjd~C4saX&G*?ME1*vx}%@cx-hAoT? zi5?SuASyHbvPBNS`Z1JC;rToMCf=Ryrk>sIo9?lm5v~kZN9Pk~uru5F)im z(VibT7x#nm94WNkG%DaP%h0f15ep*@g}(`F7yNbL6vI(^Ke{Gm2zh>kkN1RoH+m*} zdVBh~e|NR=K*ovZTkZzGQw+o5)F7S1oDq;7x-7Iu*s$=F&_68010AL-YKL;6L0CVvMuKqv|&qp6WNgjq+AW&_y`>`PWF?=qCKH}|edsq-f9gjZ1Gx!D zh+CblI+UNlmzbushixW!uTtRYuc}kPis+6`APZpsE%-Nl8^6KV@IE{f_rMnX5^V!KEr$?dRkp;8PXwRW#uKny4ut>@QIRR2$WTdQpJE3IWx>0~_FPsQm;``g=fg zzEU(72Iw&ocuIjh%_R8NaA;MqCeq=&hGl(#178Ex8$j_6`4!O9RcPTI$pcP<0PU5B zS~bA-s1F?10DGW1w2=pK{jYQ`UlS30KAQn2Qrca0AEHx25Tz%NTT5vft?`( z($ym?p!6!}Z_*AhbsoupR_}w{Ezn7FoUB4b&A_A8U$LZi0UccPkkDhT~{5k=`m?(w&i(phFlEa&SMu3OC4cr6t=&I!QL`YH}2N94`=0>lo#LGS2ju zyS?ca*&y6E%ocy5?}-oC&T=)?M%R^E;oW3E3YQaDqkM|wF$<+AJc6pC_P{OF`}BRO zIY9;_LU-eN zVG}kpF1edpNB4z%S`A} z@hxFr;`Y)M=9F3kpHOQOl|AmCr~g8#LPj%@;@|WxC6!tLw)6)27VaV2m=36dvKQUP zH>Lj6LDWv_OHF|2(@CW~J5~8oTx$;JFNs4;Ykij(icrdT_?&W@oxm@lkFku<$K${GPj8dr@m3=kR9}OoGskd zuTTz3U72li6L~8=NBKcA(zQ@`sW-iaHS)W3H(1X1f%359xvOd~CQ6PHKk2usz2%z5 zL}eTKo|%H{D#Pi@^gyYTx{sP6om8ijFvWs9fv4xZdWE(sdNP+9i*HD)sAkMsVK`Gn zs;JkfL25oahI->_Xfm103|CXgU-Wz>8C^tul+9{9aVZs)E~Fv#Oj?cCFpCu%Ucequ z75S1b5P7&`3})y158{JtCAp6Ji5-X@h%a=fs5-(X`Z05mpP*i(M=3AGne$Ei3puHbm9MFZs!fR|C*gUs25io))L1$jpQC3(jA||2lDSDm zQB$!S-Y=WMS2IRozH2a z-qQ8)6?vW*DVUWPLxeEh}FIi`JcE~%}uIA1W za@?MHG3i~xqS%cQ%YrDw3nYpmTxaimPdDEgZX{$sK7+U5cxABk%(u&FFIiu-yO1vE zUQ)r`L0m|sm{CNtm}Uv*l9nYfv4e)J#sbiL;N9W-2$`?N(lqrxc%YKhHBxV1 ztgER#vM8nCYLTt@hI6a<6&`Im9&$W7G~N@xAYn!9gz&Aw6;0FdU3raQ^CkGAxS9NQ zalaCR8WL6=E3f7lcaHsol_`2&G|{@jxt#k`wdrOA)(LAG?TaZF+bE`c#={cv<}_U-8GvmyF*EM zM#QnO?2tc#_L&;$CQyUOC@BZ>ILGrp@a2VF;H``hFAJmio&Jjc-+b?VcHbwif;dXv z0$61>`$8XQ+G;)*;0nkNIB4!`YHr}!L9_$Cf@joBxuxV3&x>C1ffz2m6emfIr1er= z`760P#4?`&atKD3(S6*7%7@tRRocv)qA$_?=>d?3`4Gl*2Py~ujGKW!u_fT1bcocs zly1rgd6t|azn7j%1yZs+SEiI7mG-I(8PPLPb^H=fpmM2R^eXxky_23nr_WMF53)t7Qs2{0CU?o?j za`9rEh%dt|i(E1XG8=EH3)K#4npzHKYIFw+`L$XbaEwBFqn+pz3dS{XE8GS*zzN_B zeTMeKERI_6c7I6rkR@cOBRwY9$PwUh7dZ&qdB7t1@J_J+Qm6x(ZG$=is;dLYMDtimrQSj>pcNNT$cNe= z0cpJe@8(m$ZV$l2nGaWV;Ni*u3T?o9sv)lGfPEss|K|e4^bg>Ur+}1R16Im~x0eh2 zXFh0EfL{@?mo`6#0bgSzAf`CLKsAAK26&jlzyrvEM>890KZZ7*09wleJ|ys;ngGp3 z1J(%#9s^5#cA8I)2js_s9HFH;(7~|)R7zkg07Ur_@Lo1}9G%bt3uPjpjR+_m06JyB zSEtQDWWfg*1e`>`t|3D0iU)j$C4i=Kpne`8NE;wQ2F?dT*~rqe;c#q*vNUX(zmtZm z4B$R()|ekkyGwU>>96LG^n<==;9B!N#sCM}Oeh1$VE|q<52WTN)n+e=z^UewwFA#K z(3uA|ZPptLwN1dQ=2JDok>;_~=8S1Gn_N&%^D%34v>^QpYG`x7G%kFlX9T#W=@o*^ zP%{vCh=pC7U#9V|&8ySsl=12YxvlU|oA-oEXH;pkqO@6NeCaGKZ7vz4I0GlztgdR{ z(aeB0<3OT!rG98Dj7p7v1JH;>kHP0tNK&9ril8SM@O}OXrD_7z_E@BRYN~Qa>95Aa zozw#Eh1#eW{be!WSXBK7gEpOpqnu0nJ9|)W5)9`&Fr@Zc!(r zm8d(;hxo%H9E_Xd>JUk|LpH1L)zhRd_)bA`7|#Z58#s}`vj)M@Hl6>J~0 z55~rH7`0mTU?tqGc~G_yPQshPBUc$LpeJNG+|yei-f@tuCI<3AHIVn}Wr)+fQYCdM znL?^T{2>JS!8+QA9zh)B3S?F9KvO`oyCCXvo(zEVV*$}mKqK%XbP4ss-=N!Q4swwp zXaVVn`alo7K*JzEdkI(v&&UNeT>VEGDi4&Ni2HcS@8W)i*#S?aRJ5FqHTVO+4v7v+ z4DS+FU}+PWVH{46Q?`SDLUzjbu8#SR!;ZDClkOj#2OWL~<67ex%KOP^_DX;~d~eM7 z_;QJC;@jAUk-IGmjSDGJ>g{jseh6=q`wqp8{p0yzl0iN#MT^&b7WX|{v!YYQe_FGg zo5gX=V0poTQZ>JpW^w>UZO`Ws_7-0(&MUdi)gc|(uA$pvzDX=e9gwy+ zIVZkSWMW_k+9$pCJhi1*zbQespIpQJyx>ymqnYXt(rEus=Z%uJMH`D^tfzcKm67ZR z%h%EAi5*h=rOitIEkTT=f^IXb<;uR9jtJ{q$T$viuJZQ~IC&6yq&Amtaid)utq%$> z7Y#01?+RA#Q2PSEiyji+H?>xJXiAN8T+~;VD8mZn1h>k$-g*-9tou7Rdsp&;d{zB} z7?k?_NB3Ualp=F+gA(u)N~`b`^Qy?qxRuF=(`KZEmn)3^Cd6dCrYig`*BF~@ePXTb ztmw6Idm$fdA9!B-ayHi>TbJUQMX4ob_jBKiNLqx?nBDTl?j$L@3f=+X1HB)6QWv;@4<+FjEzvBXmRqb=EOlD@+~n3(WC zqxL3A<-SYWlQcfMICP=;0Ogc=_y)Up+ckTU?~CC!CG16jL7J+b`JFq@H-+bJ`W)yy2+j{?0p?ixBTfJ>)~u zk3uW1z?a}X;mhEH#Z0vj4b@FF8UyYH&k61qvd%Ir=*z&B#!)&KwS}CM`-n4y4}2HC z6U?Tn0NHW3h0dZPyb^0j^&mg(p`5FH0sfM6FmEFhx1=UgZi=OA(DUeZ^k(`Xy`COU zC()TyUy8#^upTdg=jA$>+jvN=r*g_MWr^~gG8=M>UMU$6Yx$(M1Y5r!gsYZd4o{}` zKx>tuegGDsgK-ob!)k9W5>|s9j6z!lBAfE6$^@@6ob8t2lLHbj((HuaCZP9%xO&Nvui8a+< zlzkBC-UKt>0?2OM5@w`q!Rx>zole(ctI;BJnHff9(Zi@eVRp%8l~TG&!yzJ*@9*rd z?;Yy<(YwbV#_Qw`_z!xZ>F<4W)(ejK<<@Pk1aN>2<;KW#xV(@39&J$;iH0= z82-Ss6&Ek~#`t&nw{hd8Dau&&HHsrO=eqmjKJHv`& z_Q#KlKOYa%OTuD;4(R6M##gR2Wly z#50}`P`@$W3L}VuUZ`W#I_gL80QXV^`95UimY4g=OXOc=J=jtm!JZibNIweo zMq|(nKsP^uRdN%(LLL-|X-oj?T!A=pTkvzvgFMb5>I0>w@~d1^ek=VZ9hUA(wLrSr zFsqA&9-oMkVg6S-%yPR<{R45sJJdl6d=1oj+zYZx<^iHV2+`JMrch%Kk7UxID* z7NVXxWEvu9E)JlkLQM*0fkn_>>K64Kl}25|o$(t$J|41=q=7Vt)s--h>__#8`bt$` z4%}qO8F>vDtuf@6EC${$z&y0e=q%a?vW!IS!FOu}AL}LXLC+(j$q<->KMZn9)&Nqw z1O8$Jc9WJJ(jPKP#=^6<2Wkk2E)p;n2lmYSQZIBS*km`rqI(V~un5q$mPMhhDxs}6 z5D3;p8f*=~5^DqYRb!}M9&kSm{APn*AAvQN3Doa^RaFeOO$_*}n?lL@Kd78FbihSnJf1Tt9QY#P{*)xa)Gg8nf8 z-^$9I6N6Wi$z^@%P*Z**#*>_qg zZKRojcP$e|vtonbzqUq&W+`fxsn(7c`ctz>wLBUDMt0d+A!TcQ7@?fDI!4(_7&S5leen zTi-*ooi(n_rG2Q?(=@B;M$6&Sa-B4Jw2UN;Gj0F8K2F){RGKcem20#NpR%h8aL+?D z$_L&B?XI%JlduGAxh&EW;#U{3HYap4u88|$D;iGb!z^JzIRg1~Rp7nQpL|8i!TlY8 zYvPaS9@vD>fs1$GGYuh?5C^hEqvu!#e-{tPE0B7RC*ud?W;dl;MfBPZ3ZXM#$M(}t&MhW;dsz6Su&%gtH z0G{Qml;59SDSj67>I8+W8VU1%TTAv2Sk|< z;?w=zy-(bSoF{E@wtB8RTz%ybtmP4F@rDhFlp|_|zqcGQKh$l(6_lf*z$Xb)#0>d~ z8i}V;F4_aQ>yDZsZuB*EjkJaoHz{nI-?8MPyPDKpH$G^2WK8^ugoO#^VsC{#2)v=o zC8tEmf6m{Ft1b*y;!p=VUe_6-r3!S`FLDchq4$OBg8h42ko7>xVJq$E>8nNB>t6;o zj_ecbjR}Z367ey(lW{7&Q$=D1tYnZWbe2z}PSj)Oj_!_b7(103LRLs)UuI}6ux|eyf#SuL;woi2ZsM#SE&70Y-WQMemZ_b;=PBKLKs7Y+9{zsip zcZR8e7Aq}E+jh_vZJ%WC@7}}j!m~^pg0jQEivBjDd|3bBU}Gqgj^4q{ zU5oUyoJy=Xn335@x{K^{=1;ss4U=VVpYJR0UH5JG->z9sr(?gPo^z8YM|#AV%%LIQ zN1l#c6|N6G6rk64rk-biP4rYXTba2nep^-m_hFi*;H?daCke13MSQ+34^ntq1Ok-L01QSW` zL9dmcCBJZiEA+SEqPUfQ)<4%f+P%zi%wFI9)L#$nH#!54gylp&fL>o3P{}~kRmcRT zp<-9wlciKUW;J_|UB|9rC*v>F$I2nVx<|Q5+%T>i{{{aK7vkIMzT&jn-a7YlpYR$Z z2^m8ytheb8~FC}96EjYD<9V~C`?>+bI{=U0CbAy^pZq7M( zpS9QCd+oi~Uh7+{?Vgs!P4BeCx`^`5eXFSD46l4|*}RhB@zR{}p=D--e=Ob4%m|$f zw?tmf`7pXSd?ZxqRQfM^Gl{ktowy^>keHpUN;V~iVJ4f^y5Q6&YzYr`SA~9(Ut3gL z{#CEPmR?-=ZhmKEnsdH?z^^w0o!^BniH^xh(#l&RAG;g;mEJ^ec(Shh+2qZs`Kc;z zL#iV6Kw@r3bL-faJtrre+R`=FtjaAcYA-1+UszUH*qC3D^C;0%hx{pI99R;njJzCu zCptLV6h7)!n-k=6xGd=-O7fTAG`j)@u@v6Kyf7aC1pf zSy@S+!ojgWMjvt~f=B+FsdtWrc1J#q+!|RGc|Tm_PB(Gnr6yIKYDhioU5bT$jrVlw z{$y2RKghe~?PcHYXi0X|`$Hp-{ey^AlV0=0`u1kyV5s__O9yAt2%bK7Pky-UDZ9*|3zeXY-iE1 zlIr5k@s+U=x%0w#p-HSrZO%7tVYmtea(Q@GsJH78BlfU!9jN2ZwC{bwnRK()C;d55 zVIL;9cTel8>Bwn2)beagL05@+G<=!*T^7o28iTxfB4 z8@{nMWKO7X{!1o?UB2V@Psh`X(v9FD2=4Z#rQBqk7@Ucq!LRvtZF_B^(rk%r%fFy- zb~KH-@LXvq1pA;Y@H>x=D8)kyaX*rHZ?uDt$RXOq@xi;&dAO^$!%sqWL@4{1?hNQJXtU|wjy_FG!!{MG{fzO4`7_T z*6ok?U>GLTa(u@9ofpiy^t;{yZx{HF$I-h-dQ;LTx!an!p=(^{E$z3rP3n3+HOX8Z zKALl5Y-oXrZ6Y4+aNhXbKSVza&!iVW=j{F$^x55L;~n(p|Jad+6Sp%UUFQ8J_3PBx zUN@@4V0MmIy}sT^vh~!FW9GTeA&Kd!bDXB|w?KbOLF-%_-xzx{zb&^qIww5I{lI*Q zXL+H&&%X;T(~**%%oTc0-@B=U;&ax&XB4I{Y^~{SVx$GoG0BXzzJw>r;ygXcf35$gLc_q z-ar%Ihc*{R;~RqBFNE$@P#KE{W+3{lbeB_nf0#G-fm@O1Wq(F9Z$bkM@qKwldllM5 zxam^O(w6N zd^OS=g}N4|T6&vu*#TbWSK6gA_PO{|q`OJij%MDpx|=jl>uZrVsa$^YDhaDCguC*% z$xET^bulOv@}~SOKDwN8wy7?UQ;yoAoP7nHfWq{-3!bKpt2Ctx=vDb=&gA=i#!{`# z;a=I?(iy!_SR9)hQ26o!MlOMlSVk)bP_7R=>%_#O zp>7ZSy2o4H#KD~6L@m73NNSXasEbqD0r)%&LaK-w4%7F_19@b|@c64gxUHfsN=bBr?{P)m;3&Gi+$*6xtJIiTpH)q?6@Y-Ao)um<;*>Z+M z{ZdN2Vm^1?f~#uErg_>xS@{YT_3@Z9nJhN0Q)_NwyfiBYFvDLoFM*tR*=%>NGxs}> z5(86jUNdu?Mc}Qjai*F^=jUb|XYor+opYi2z!_y`Gsj;id(S?k|0+%r?_oQ*8fsh3 z5?WNjig6A!syMfQ%*d{z+(2-1903oP^d9&pbeu=lkBPa-w5 ziGnS4cMwZFhe7ZY96yy(Lei` zPesiCNM_&p3Q7Z+>mT#9m7P67-<7i-6|nZ~K>q5X*T~8dW=(3QJ-Nh44a<1fm9?}8 zng=t8Or6am^pNswHnNT=((4G+v??EG-D^Tx6tD9>^ecJGBML$*z|TO4?q`Nv%-T{% zJBsK-2TM>fTq{>5p5)^MEA$C!k{v6}ig^}Z?>_v>LRZR+l3gjD z*K6rDIcx&EZJ&gi;?{!pB|0TvSLarN=t8RC)92#f4x3MMMzWa zb^1n`OSNNauh)L6y;Bik+Fi8A2}7YhK^7M6p|Xm|;-eh8vj316jrOT7S*VmxSN*1_ zGTD*Dop1oM>ZoSLn@K;=e`^opA+n@i(mo_Rl8vjfwj5DW{#M!LWC;>KdP9_Sr$6Zx z;?d^R)jpv=)e)2s7rGM-ai+HFPGw}fu~)YA>6w&+GMOqf_1p5Y`UK@|#&DZQH(R?j zk)V`XkgZi^)H4BX(Ftrz*(-f33YOo1?`+<8OF`f3nYBRKPtntr{Ry~G4~U|Dp1saq zefPqT{L7Znm9=@Xhx-_DQy8 zjePc5(C(nutX(ePAQ&yZ5sZLh8tu5r1{~C)vB^<#Wkj_oL0E>4%0)9*wiL(QJ6fMCWm`UI-4D4I YAAvN9H;q}gboT1>=eql$JlXO7KaQ(w@Bjb+ literal 0 HcmV?d00001 From dad6b5667d2df32e69e4019749c2b701b5f035c0 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 31 Mar 2026 16:59:27 +0300 Subject: [PATCH 05/28] fix hybrid pg --- .../PostgresFileStorage.cs | 11 ++++------- .../PostgresFileStorageTests.cs | 16 ++++++++++++++++ .../Sa.HybridFileStorage.PostgresTests.csproj | 8 +++++++- .../data/12345.wav | Bin 0 -> 25388 bytes 4 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index 79198c25..92b07661 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -112,20 +112,17 @@ await partManager.EnsureParts( string fileExtension = FileIdParser.GetFileExtension(metadata.FileName); + Stream ms = fileStream; - if (fileStream is not MemoryStream ms) + if (!fileStream.CanSeek) // fileStream is not MemoryStream ms) { ms = streamManager.GetStream(); await fileStream.CopyToAsync(ms, cancellationToken); + ms.Position = 0; } try { - if (ms.CanSeek) - { - ms.Position = 0; - } - var sql = string.Format(InsertSql, _qualifiedTableName); await dataSource.ExecuteNonQuery(sql, @@ -133,7 +130,7 @@ await dataSource.ExecuteNonQuery(sql, new NpgsqlParameter("id", fileId) , new NpgsqlParameter("name", metadata.FileName) , new NpgsqlParameter("file_ext", fileExtension) - , new NpgsqlParameter("data", fileStream) + , new NpgsqlParameter("data", ms) , new NpgsqlParameter("size", (int)ms.Length) , new NpgsqlParameter("tenant_id", metadata.TenantId) , new NpgsqlParameter("basket", _partName) diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index d2090587..dc8a885e 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -130,6 +130,22 @@ public async Task Upload2FileAsync() } + [Theory] + [InlineData("./data/12345.wav")] + public async Task UploadWavFileAsync(string filePath) + { + Console.WriteLine(fixture.ConnectionString); + + // Arrange + var metadata = new UploadFileInput { FileName = filePath, TenantId = 1 }; + using var fileContent = File.OpenRead(filePath); + + var r = await Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); + + Assert.NotEmpty(r.FileId); + } + + private static async Task CreateStream(CancellationToken cancellationToken) { var fileContent = new MemoryStream(); diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj b/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj index 265c4575..3b4c9d52 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/Sa.HybridFileStorage.PostgresTests.csproj @@ -1,4 +1,4 @@ - + @@ -11,4 +11,10 @@ + + + PreserveNewest + + + diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav b/src/Tests/Sa.HybridFileStorage.PostgresTests/data/12345.wav new file mode 100644 index 0000000000000000000000000000000000000000..5045628ede2e90ff569c609c91ca95129939cb3c GIT binary patch literal 25388 zcmZU*1$-1o*grnAavHg~Cq!}g;!@n*-AZvN?vzrb6e&{NwLmGwiUw_QC=`c;cyey* zJO5{smw)^I-rLXK?d{IYBQv}EePpIb`!;RTMk3U=MV~LnOq@~9f)K*6HBUllaZ`jS zWI>~b&lrxN3?UR!35F7%j|lwJjuk?~PL*EK&Xnz1xw6v#{jZjmEi3VJe5O@)td;%uY#A42wLa_iGgXbB&y=-tWz;kpG|0e| z^7rqtCc%HD*ZBDFhy4Gw{wwQ$X=tVYck8H>)@N?DfB&xr%lOwylG{- z&+`4Z9qstP@|MZHi*c|Bk=UIw_;@Z`)-Q{vA*M-WghJnl8#X z`L|w~WvpX(&urYo%vTk8nw?dex~&A@8>ql&S+y-qgD2=>`C(ZsiMhT z*0cXF#j+Cr9aCkk{L9t9BkA+GG7kUCd0D?|{r4}o+V!$e8BdyKnEyOIG#!@pgw_jX zJyh0$c3h^Rf1jS3RR2o%`9A%8ca_!q+{(W_{NJ%zCYPpL@qfk}jr7QXSlG-kF0_wU zM!WMgz7*K}L@515pgjudkP*?Pr7+@%NO-tcc(_i84lz(uyDQ2*+J7Fdb3`WY(l$gW zCqNro>j)?Y!p4+R)a(^@;&B#X!Hv!~{nI zaDkxxP$;L3t{^yPL~h{E2y__a1)jo6b)sn?q?CUe=n_%@8q|T5I@m;LRg+138iWGJ zn)VUYH^3E5E(VUYXBH2;3L5moS?%7?`d{l4J^TuAq(gB~OY2`JXhpj-SZI^Lu?iB| zp_WFA16d+aJe1UQDnrde(4-E8s$hvBf`peA%*>7G#0H`?!s1trtZIk!6o`D!S8)v3xi z@6{p3;>c@*Ly(h^-__`d+a5(p;0Q zO^bTvF3M|Ri?EGz&Zpxex1`$Bhs2+W{Kqnjsjkc>ec3rp-`*==!_jA?wh5nk}rtZR`zq6L-iG?c)8aW+gx}{^%kkkt4a_YON z)PP^v`{)rlfa2Ipqg~%hzf0%E&*fD@H*TEoji-(0ox6oA(|)bklKW+W#ag?NvhP;9 z8#hKyNqLbrKkZr~7jY?YHd}zQ(R1`3FJ<~OwVB_k3gk!SzF3>P>f7)A!mT*wI9A*4 z6(~863WgR*MK!%Q$v=TN<9nubNiR%)mN+!(SKzd%cHWpYo6 zln(mucqckHShp9u^2w)%xzh{u#aCVZr7EW0(Waz3>6I&VOmih%jkpukSvQpoQ-({G z;DJ+1S)*FjOfrGIRQ?vv`zcS1y;jkjybqtovqFPOH=7UhTWYpAyHar>PaJ z(b(U7ETD4G7eRf3>I8KT3=0@y+G_Yo*Oxg&HODK-P}QPTkTwe@UhkjcEA&Qr>v@0i zntea`R`_$d+d?z>x!Ml9>EXHohDxSw=J5e91BwDR1!S6cnrazW=ufe~(sS{6QbjRI z&4oo=ia*LXz#HkU=>6VX)tBP$&pqTvh+X8t>H>6(y1?wwjWIkmI!x!y#sJQ|#++$d zX4Dxf>ynuQe2#2X4oFXhzI;`#rT?(6gD=I`()YWsz2CwqT#7JOERi-T-;ja04V}Pd z>e?AL8h4l~ndh6AnbXaSO{0t!!+M>KNvGT4ugOt(nf4W5@GH3f{x!Y`pUWHO`_9+P zKZ4uK$BI{^WlC>S7hC9e%q(4=KG7I!`ok1$rp@O~*c4{`pdY2X#{@u2%gIxvvV2(V zA!P71xgY)M{s4b%|2}^^E|sq@EEQi!9h66ELv$Vo&{LR0>|Na}{YFEIafGp>ku|P1 z+|b|DEn^dy@2U5wJXxot$xbm}&b1Mqpu zLzidk>#FLD^-~RJ3}+3i3=xK5`pLTfEW>P}EWnql9G1sOEyVW15`ad>x%J#JPR}3b z7Yp;mozg=&O6^ZJqkCAPBAHOu#~#qt(67;7(O=ci(}(CM>Q=K0nWl6xUXQAg56V9I zdugxuQfMSR;eX?=^Ng@kh!y`9k4n4bi%On~P+i=a>O|LPqSzvKzwVCsdwR>>K4oYGhxE%g)+3*7~k7x*uP141*=BxOlCGOtu3ozVAqH?@);&s1WM zvnE|%T_c^J{e}%?Co|LO6VzP13RNb_YNXPq{>Fwr1Z76*&(#LiL;xj-gLiE1Mz z{1*RBEub4Sdl{8!%_g%(_7GEvnL+!gk<`!FgQ}vR$t%^UR#z^|J>=)o8|j23N*&~n z@;D_=DOUTE35dbZ@GMG42hpi?3|)@SqrRmOwHupoD|C?Lshia2ib1WUOjVjGEtF-- z6{UqbL;aR~PZpuKXg{t;MNreIVblVsmqfLpuH!M7!4r_3h^mg1Q}-#46tAqv+mw5X ztejB$k-20JdV<374ZH)N!zb`M9D#>p8^U-y>VkeJ4z&*XLS3TLY7cdRI$u4fE>b6x zAiyvc0EM;0U!vW(53Y{?z*o=&{3oh|SD+4PKH5#*kQ6e5v~XeydZdXO&aGWCekNPREgRwgT7D=$^AG9C2* zMAHW^K#k~)cq$!D@4+UvIvPih#rMew>W#966w0gQ3Ccf$N$xIsgi1<^^g-=No|9eF zAzX!7#Y|%A=`)yrbdQ-><~JrE8R`0@I=Q$7OUK za(roB;235<>FB{7JI;~pgTPTr7oG2Rh)hSAuHvuEOmiK}vgp{vKa;AY>Qav-_Ko{9 ztebf^{gag8d*%3OD{pOQO|rMOU$?cfjVqZ_bhr=|Zm>mok8|7cd}BLHbo9`~jMO8k z4bvl1q`1M6j|0M)k@6>RPiL<6dut`@JL_0SKkKlPVI^S&b@L|`5c^2ad49OgY;GKO zCjlpqN`IMgA-%ZV?U@}?mt$S>{Y#ZzyN)8sz%-{ZLM^4Sc6;6Y{ zC8`~8EOb*sUaG!=Tq&_aMEZz?suAVQHq=MF>&mj_SVKx8Y?5_^O)mbBpOn8VYeP0y zz&eKc(oxHR4WZBDW79(`46VdfbfkTo&@6h5 z`JWnQH_CI`kawe;XEcRYi7iZPQlVa@l1gndDkY7L3=X%2Ab@RQf)>MnZ>(uS^5+F#f7*8BqGC z2^4Kr~6aIp`m8-wqV0D-5D4AFkmUkt4XV#Xi2RRLGvwchD>bkEit)q+M zC#QUu5u33!l`8i*HYfCeQKGQ2&)?DA$63qK+19>fSJ8_6EuZ4D`eirHZCae}SuPHw z4w?!>m&CXeJjrX*mZVlG_b_f!q$7B&p%}lFf8iBxM|WQ*XUixlUnu8R`P4Fd@&HCiaWR(a#hr1ofyc4wT!u)_$j$bO10$Jq}g$|BF+T2G#z1*h*QY+CAllx z7Z-;XJk3$EI(+Q#F)Qn2uD^J%Ya{<0Ei(2BIUKb+VMhv^{vjI2F?oqyi;n|FacjWq_j>GpAw9z zI3pxOO`VtQOsF5dC1k5PO7}adE);pDIb6kqLC^O;e*b>UyN>UVe*7Z$b#YDiufj+C zrztNqIA&eqj8rP)M*5M|>~cl%hNx2()_j=VM6&rip2?24#Xa*ke7f|p&-?lBCcJ<5 zu}`k2c)2@ROs7;+YS`8oA#p-#wT#Lc5ozU;gA>L@{b^B5BiQHaA%26$YiEnc+PL)3qD@TSy;q5mk9!nGSv({7qd2LT!Kd{H$H9uxaYm$edhaLKIVVwUigP&EoUHo_0=rTqoybPk~}NDd--k|zo*tJ zH!E&=gf4i8VI6yn$MJK?m4fQIH?yl` zP5xNz+QmWd4`-?+0C=sW-ZD7kaNGFvaO0|i_jho(svE&6@E6RJ3QrkrQAraR<2h< z^_Zma-9dpy9qokC{=&V&KBhRnfXnTj6aT3&yI; zic*W?O3qqe*;~09d7p8);1s$_|Dr!=P7Hn&k{7luVrFDWW9^K+fj!i5(fQUL0ZRja3d{)nBEV~EXY8ksWlvDaXur}@sw#}+e7<{LpJ#(-rAPAkyhHsK z{-tnG+Nk_YHsG%Tp&!+~)E_YnF#c(D8qXW+8W$Rt>D%buFn#C?I3BH5LzK(X6tSx? ziNC@P;Tmx(xIq3GKSLNHu9v*>WN>f}!$x{Jvx)shH%Ct!MjA#Lk__AQA9WI&3#hsR zbqP>bhI(E8S(+j45O{tzU!HHm@8$al$zVy=mpds3)M8Q-cc8A(&DgfO0R1BUZv9Na z=52II>{WUzRS%yfJypB>N@^qR6um-$@Ir72RmF{BMd=r*t^8V6lo})xoxyXd)ASkU zIlEu~oAIFWq`n)wfcE2Is#{(`cF4brHN_YyT#1)g%AA}EUY$j9qWT@6$hBw(?nasE zwbT}R1N&N+t(&B~!@g!inR9p?T22zxB)N|?RyrWQ5xx;F3WLP}@rkfe94qBZZ)GD{ zhVy8hZj*ky;gzw1X@~KY;e>vhu0J!23cy!LebpuZBYhz)5#I{C_$Hx~^1F3gi`XW!WKFLEq4@5L;oDjv_6_1_s{P5aEP z0xp@4nD!euT?Q*r|DX|Ml6p;Ptt?jRD{WvOC=Zo5VIaTXU*LP-H*v&&g#S&vt&Bqh z=!dLF|JulzCYZkjD|@Z+l71L_g04foMSaL6wK@3ZIx5wb74kjlmH3@-kT2%$`0x7O z`_l!hU=~B=F{&TuG9SSRbQ_pu^d9Os_%kb! zQEIyCRQ{A-N;Rbp;?Kexz9RR^-+&9^O~Nx_h-8yf&@_4@J5}G&Kn&-N*NlgZPYhq^ z_klDcs4I9Y+#gHSCCW(Uu(DWQF4Ykq^Yi(s+!22=*MpnIR}qIuW_2w(Ko8WN(u;+sN-|Y9ZqQF-GwBUfO`J;xsT0Av+pE-2I?AiXXmOE{!%g$^z5;(|@KmLW6QnF< z8aA_E7~IBQriG^OO-)TPCd$}C&oTF*1%xuyO-i<$BL^yv-4_x1#yM^JyDR)QVoq2I)=F)`FaXf)$+PQA$?+ZC&a}vp`3}Zlj+J>Q7EaV zXpt9_Z2Sb@LCNGYaI#gL%T4s=xW01kv>hy-QryMX&66$UQh$eROdeP1yNX2#H-aZH zfyyND6n~7n#J86lP_Om#jNyi!)EPPs_nN!E+v=L*df@unwZ?vv(_y~8u8}T}JKcX* zd2AXM{#RIvSyilVpW~(TvbzfSuZp;_Lb$Jf$)@Z9S#t`n_#)`(5!qG0Xk4wqjFj{2 z@2>aOx6Tf}y24TQ7o#!ao2Umtx#~N|!jb^zTA!6$>%Zp<2A^0HJC)n$!-2fJo|V+4 z$e*edH`Lc#k=#_5Gkp0uUZMkjTzpFJ&)stG+vo_jWxnkgZck80sj$K?$*@0FW)UeE$=+H#8 z(D8NAG3#^B6uuG!{~Dli%3k-yoOf?}eynK?lpeCFk!jU;HO{WRC+4zz$$HB6qo*BT zPOZdt3CWMC8qrDD%YV--wJz6>qu*B*F(MzT1O|S zZ9T;WW3vp|+wulFS0hK%qgr1#DXDorW-lJ?1t@NB|+t@oBD}ePI68PnU9Q7ftgGH7(Al)M!lo?3954Pvxat4fhq>Jr}3u2Of=D z8ZpW=9=8+*xLIqRV!va9_!q>AR*6SlEsERaUe7s^=XI{9^W#R>uGZjerHWDW*h2| zi(coyD_G*XLYju$D8IYaKeZiBzR|04I3&|S;|-)3W_J86i=Q`_nBY?f}H zzMp=Pj@F%GO>9f1A$^(3rjAoBwmzW z$gdQ;S_kdHK8j+_G8Nci?0R+z8w8mSxgWRYg{u(dEQ*eJ=8+@3L&?Yn(wS<~Rh=5)uTgW^zigYIpNM({r zs*%p%Ydix{)LLj9+JJsRXV3w(3L>IiQGJw-QovVfgy^wCtR$OUCx^*8vVe>yBgu5K z72++ZYGYIkp1_CXCV4>qB+no|`ieXPPOp-4 z(wgV?l}lh}pY=Um7C$si->CtO4>xfOJY} zj9QC>2f#){IXlp@l3XaSN@MXHaPJ1bo$%`_<(CG^VZdJ) za2f{M(DFurEYNV5%BYp{Kxr?W(T=nX0Wfh2*_xGIzezQ7D66O)UdJ%X!f9DYX0sEjxpSd;=5cC;(c~!8eB1wJZlM4*@~zCfLouA0Y{F)e9pn z2l5JjQFlc`<>&$mrUQF@ZU&_L!S&Vz_hx@sbE>Sz*1 z+L3R_B~ll4#Z_=RCF374OZSEM(RwtLY$KAIftI10aR0OeDVh?G+7Rx`AIMU;zurS{ zw}nxjM=WHQx(8Zm13F)g3egj|6FZTw$z-*fx*mM?-NEnQ7_G)7_;;!Zvp^+;x`6lN zemD!M$bj!c59uL3(~Kx;sya@-A~lkaNgqYKJPv&Sz0g~#F?*9)tP3-=(G6uAupj7O zFonhwpWI2BC&mbI{FnYYzFFQJkIQTGXNdXaF8j*#h2>;8j#?GjHNqU46!1wm6kQk1 ze!aV`>ksEl*IjpxYnJPgE5S+IYuZyhnS!V;Fc8bLh^w*F;%g=R7E>qOY&oniK)vPj z+(5s}d)1!+nE#+MNI5D^6T;ki)>?(#bN|SBRnX7v5G%1?ht`YxkQ9^NJAF>MzhcGE zQ1c)(N1Wwf>)PsKJgvcsnj+m+qGhv`o`{^@?n=iqa=yTVUm4}KE2()--g#(m#2(KnZ$BacQKnX9_q z=01V8peB}Zix^ZsptfnJE{0x-{!}W-w?$s?3qfKnaiREByewW96){R`FS(^9@)V^9 zMBxUb2DlDYL@lE8X_01`>vVV8Np+>Hcm{rkIzu){6xjjKWwW|V8LaG4zEs94OO+fY zMqQ;wlT~CoDnVl)Go(5-g<4A8qyC~!P*{mu7K~Mo~Ve7A%DRWzM^_ZSpXJq z4JA!!pd3}gAc7>Squ{CdEi&R=I1i7aeuDQ}4rQfIQG=*ZY9XfZAT*EECcmixYJsv| zVU&FNitLcFxSTh8ye@W&>W2-O53wzIsfYphMA7A<8W756<7OR{$c#jzKx<4Vw%VK z98a3>k>j3Yv3Ievf%t>`N)`0mOt*twp;aQkjtq~8j7SPDHq|!#fJEh;_}pIyqQ+-@ zo&7WTbNIPV$HZf}rpqW`sji+8fu=Zv)Pwr{s?1MgfIcbJT!CmDYZxfGTbvpimieHLeo zei6DoXqEmHekwoZZhFUiuDD(91Ku58(Ob=%4R*;^YeC6bN4gXHBg6nSNVm;ADfD_o zNbI-q-Jjugi+DS2@B&6#nZ6^BJNneG|gbTsW*k1zSZy?_{G`LZSr*UeC4U6(yD9fl*K~J= zXNG69JH?gdcwk>=pXO-liSQ2)??Uvhmbt{THT=7n>#^12nAqi!yF%Xw{G)G=zmhBR zeY{2Pldh?*eeU~i$*s6MI0KynoLB6#TwQ&G{Vq8i8`<6gr-K`Z&xnqQ85uJ_+7Qt@ zgb668$M^?1lmFgZ-W%aIxp#Q>x;c-}J=7&Tzjd~C4saX&G*?ME1*vx}%@cx-hAoT? zi5?SuASyHbvPBNS`Z1JC;rToMCf=Ryrk>sIo9?lm5v~kZN9Pk~uru5F)im z(VibT7x#nm94WNkG%DaP%h0f15ep*@g}(`F7yNbL6vI(^Ke{Gm2zh>kkN1RoH+m*} zdVBh~e|NR=K*ovZTkZzGQw+o5)F7S1oDq;7x-7Iu*s$=F&_68010AL-YKL;6L0CVvMuKqv|&qp6WNgjq+AW&_y`>`PWF?=qCKH}|edsq-f9gjZ1Gx!D zh+CblI+UNlmzbushixW!uTtRYuc}kPis+6`APZpsE%-Nl8^6KV@IE{f_rMnX5^V!KEr$?dRkp;8PXwRW#uKny4ut>@QIRR2$WTdQpJE3IWx>0~_FPsQm;``g=fg zzEU(72Iw&ocuIjh%_R8NaA;MqCeq=&hGl(#178Ex8$j_6`4!O9RcPTI$pcP<0PU5B zS~bA-s1F?10DGW1w2=pK{jYQ`UlS30KAQn2Qrca0AEHx25Tz%NTT5vft?`( z($ym?p!6!}Z_*AhbsoupR_}w{Ezn7FoUB4b&A_A8U$LZi0UccPkkDhT~{5k=`m?(w&i(phFlEa&SMu3OC4cr6t=&I!QL`YH}2N94`=0>lo#LGS2ju zyS?ca*&y6E%ocy5?}-oC&T=)?M%R^E;oW3E3YQaDqkM|wF$<+AJc6pC_P{OF`}BRO zIY9;_LU-eN zVG}kpF1edpNB4z%S`A} z@hxFr;`Y)M=9F3kpHOQOl|AmCr~g8#LPj%@;@|WxC6!tLw)6)27VaV2m=36dvKQUP zH>Lj6LDWv_OHF|2(@CW~J5~8oTx$;JFNs4;Ykij(icrdT_?&W@oxm@lkFku<$K${GPj8dr@m3=kR9}OoGskd zuTTz3U72li6L~8=NBKcA(zQ@`sW-iaHS)W3H(1X1f%359xvOd~CQ6PHKk2usz2%z5 zL}eTKo|%H{D#Pi@^gyYTx{sP6om8ijFvWs9fv4xZdWE(sdNP+9i*HD)sAkMsVK`Gn zs;JkfL25oahI->_Xfm103|CXgU-Wz>8C^tul+9{9aVZs)E~Fv#Oj?cCFpCu%Ucequ z75S1b5P7&`3})y158{JtCAp6Ji5-X@h%a=fs5-(X`Z05mpP*i(M=3AGne$Ei3puHbm9MFZs!fR|C*gUs25io))L1$jpQC3(jA||2lDSDm zQB$!S-Y=WMS2IRozH2a z-qQ8)6?vW*DVUWPLxeEh}FIi`JcE~%}uIA1W za@?MHG3i~xqS%cQ%YrDw3nYpmTxaimPdDEgZX{$sK7+U5cxABk%(u&FFIiu-yO1vE zUQ)r`L0m|sm{CNtm}Uv*l9nYfv4e)J#sbiL;N9W-2$`?N(lqrxc%YKhHBxV1 ztgER#vM8nCYLTt@hI6a<6&`Im9&$W7G~N@xAYn!9gz&Aw6;0FdU3raQ^CkGAxS9NQ zalaCR8WL6=E3f7lcaHsol_`2&G|{@jxt#k`wdrOA)(LAG?TaZF+bE`c#={cv<}_U-8GvmyF*EM zM#QnO?2tc#_L&;$CQyUOC@BZ>ILGrp@a2VF;H``hFAJmio&Jjc-+b?VcHbwif;dXv z0$61>`$8XQ+G;)*;0nkNIB4!`YHr}!L9_$Cf@joBxuxV3&x>C1ffz2m6emfIr1er= z`760P#4?`&atKD3(S6*7%7@tRRocv)qA$_?=>d?3`4Gl*2Py~ujGKW!u_fT1bcocs zly1rgd6t|azn7j%1yZs+SEiI7mG-I(8PPLPb^H=fpmM2R^eXxky_23nr_WMF53)t7Qs2{0CU?o?j za`9rEh%dt|i(E1XG8=EH3)K#4npzHKYIFw+`L$XbaEwBFqn+pz3dS{XE8GS*zzN_B zeTMeKERI_6c7I6rkR@cOBRwY9$PwUh7dZ&qdB7t1@J_J+Qm6x(ZG$=is;dLYMDtimrQSj>pcNNT$cNe= z0cpJe@8(m$ZV$l2nGaWV;Ni*u3T?o9sv)lGfPEss|K|e4^bg>Ur+}1R16Im~x0eh2 zXFh0EfL{@?mo`6#0bgSzAf`CLKsAAK26&jlzyrvEM>890KZZ7*09wleJ|ys;ngGp3 z1J(%#9s^5#cA8I)2js_s9HFH;(7~|)R7zkg07Ur_@Lo1}9G%bt3uPjpjR+_m06JyB zSEtQDWWfg*1e`>`t|3D0iU)j$C4i=Kpne`8NE;wQ2F?dT*~rqe;c#q*vNUX(zmtZm z4B$R()|ekkyGwU>>96LG^n<==;9B!N#sCM}Oeh1$VE|q<52WTN)n+e=z^UewwFA#K z(3uA|ZPptLwN1dQ=2JDok>;_~=8S1Gn_N&%^D%34v>^QpYG`x7G%kFlX9T#W=@o*^ zP%{vCh=pC7U#9V|&8ySsl=12YxvlU|oA-oEXH;pkqO@6NeCaGKZ7vz4I0GlztgdR{ z(aeB0<3OT!rG98Dj7p7v1JH;>kHP0tNK&9ril8SM@O}OXrD_7z_E@BRYN~Qa>95Aa zozw#Eh1#eW{be!WSXBK7gEpOpqnu0nJ9|)W5)9`&Fr@Zc!(r zm8d(;hxo%H9E_Xd>JUk|LpH1L)zhRd_)bA`7|#Z58#s}`vj)M@Hl6>J~0 z55~rH7`0mTU?tqGc~G_yPQshPBUc$LpeJNG+|yei-f@tuCI<3AHIVn}Wr)+fQYCdM znL?^T{2>JS!8+QA9zh)B3S?F9KvO`oyCCXvo(zEVV*$}mKqK%XbP4ss-=N!Q4swwp zXaVVn`alo7K*JzEdkI(v&&UNeT>VEGDi4&Ni2HcS@8W)i*#S?aRJ5FqHTVO+4v7v+ z4DS+FU}+PWVH{46Q?`SDLUzjbu8#SR!;ZDClkOj#2OWL~<67ex%KOP^_DX;~d~eM7 z_;QJC;@jAUk-IGmjSDGJ>g{jseh6=q`wqp8{p0yzl0iN#MT^&b7WX|{v!YYQe_FGg zo5gX=V0poTQZ>JpW^w>UZO`Ws_7-0(&MUdi)gc|(uA$pvzDX=e9gwy+ zIVZkSWMW_k+9$pCJhi1*zbQespIpQJyx>ymqnYXt(rEus=Z%uJMH`D^tfzcKm67ZR z%h%EAi5*h=rOitIEkTT=f^IXb<;uR9jtJ{q$T$viuJZQ~IC&6yq&Amtaid)utq%$> z7Y#01?+RA#Q2PSEiyji+H?>xJXiAN8T+~;VD8mZn1h>k$-g*-9tou7Rdsp&;d{zB} z7?k?_NB3Ualp=F+gA(u)N~`b`^Qy?qxRuF=(`KZEmn)3^Cd6dCrYig`*BF~@ePXTb ztmw6Idm$fdA9!B-ayHi>TbJUQMX4ob_jBKiNLqx?nBDTl?j$L@3f=+X1HB)6QWv;@4<+FjEzvBXmRqb=EOlD@+~n3(WC zqxL3A<-SYWlQcfMICP=;0Ogc=_y)Up+ckTU?~CC!CG16jL7J+b`JFq@H-+bJ`W)yy2+j{?0p?ixBTfJ>)~u zk3uW1z?a}X;mhEH#Z0vj4b@FF8UyYH&k61qvd%Ir=*z&B#!)&KwS}CM`-n4y4}2HC z6U?Tn0NHW3h0dZPyb^0j^&mg(p`5FH0sfM6FmEFhx1=UgZi=OA(DUeZ^k(`Xy`COU zC()TyUy8#^upTdg=jA$>+jvN=r*g_MWr^~gG8=M>UMU$6Yx$(M1Y5r!gsYZd4o{}` zKx>tuegGDsgK-ob!)k9W5>|s9j6z!lBAfE6$^@@6ob8t2lLHbj((HuaCZP9%xO&Nvui8a+< zlzkBC-UKt>0?2OM5@w`q!Rx>zole(ctI;BJnHff9(Zi@eVRp%8l~TG&!yzJ*@9*rd z?;Yy<(YwbV#_Qw`_z!xZ>F<4W)(ejK<<@Pk1aN>2<;KW#xV(@39&J$;iH0= z82-Ss6&Ek~#`t&nw{hd8Dau&&HHsrO=eqmjKJHv`& z_Q#KlKOYa%OTuD;4(R6M##gR2Wly z#50}`P`@$W3L}VuUZ`W#I_gL80QXV^`95UimY4g=OXOc=J=jtm!JZibNIweo zMq|(nKsP^uRdN%(LLL-|X-oj?T!A=pTkvzvgFMb5>I0>w@~d1^ek=VZ9hUA(wLrSr zFsqA&9-oMkVg6S-%yPR<{R45sJJdl6d=1oj+zYZx<^iHV2+`JMrch%Kk7UxID* z7NVXxWEvu9E)JlkLQM*0fkn_>>K64Kl}25|o$(t$J|41=q=7Vt)s--h>__#8`bt$` z4%}qO8F>vDtuf@6EC${$z&y0e=q%a?vW!IS!FOu}AL}LXLC+(j$q<->KMZn9)&Nqw z1O8$Jc9WJJ(jPKP#=^6<2Wkk2E)p;n2lmYSQZIBS*km`rqI(V~un5q$mPMhhDxs}6 z5D3;p8f*=~5^DqYRb!}M9&kSm{APn*AAvQN3Doa^RaFeOO$_*}n?lL@Kd78FbihSnJf1Tt9QY#P{*)xa)Gg8nf8 z-^$9I6N6Wi$z^@%P*Z**#*>_qg zZKRojcP$e|vtonbzqUq&W+`fxsn(7c`ctz>wLBUDMt0d+A!TcQ7@?fDI!4(_7&S5leen zTi-*ooi(n_rG2Q?(=@B;M$6&Sa-B4Jw2UN;Gj0F8K2F){RGKcem20#NpR%h8aL+?D z$_L&B?XI%JlduGAxh&EW;#U{3HYap4u88|$D;iGb!z^JzIRg1~Rp7nQpL|8i!TlY8 zYvPaS9@vD>fs1$GGYuh?5C^hEqvu!#e-{tPE0B7RC*ud?W;dl;MfBPZ3ZXM#$M(}t&MhW;dsz6Su&%gtH z0G{Qml;59SDSj67>I8+W8VU1%TTAv2Sk|< z;?w=zy-(bSoF{E@wtB8RTz%ybtmP4F@rDhFlp|_|zqcGQKh$l(6_lf*z$Xb)#0>d~ z8i}V;F4_aQ>yDZsZuB*EjkJaoHz{nI-?8MPyPDKpH$G^2WK8^ugoO#^VsC{#2)v=o zC8tEmf6m{Ft1b*y;!p=VUe_6-r3!S`FLDchq4$OBg8h42ko7>xVJq$E>8nNB>t6;o zj_ecbjR}Z367ey(lW{7&Q$=D1tYnZWbe2z}PSj)Oj_!_b7(103LRLs)UuI}6ux|eyf#SuL;woi2ZsM#SE&70Y-WQMemZ_b;=PBKLKs7Y+9{zsip zcZR8e7Aq}E+jh_vZJ%WC@7}}j!m~^pg0jQEivBjDd|3bBU}Gqgj^4q{ zU5oUyoJy=Xn335@x{K^{=1;ss4U=VVpYJR0UH5JG->z9sr(?gPo^z8YM|#AV%%LIQ zN1l#c6|N6G6rk64rk-biP4rYXTba2nep^-m_hFi*;H?daCke13MSQ+34^ntq1Ok-L01QSW` zL9dmcCBJZiEA+SEqPUfQ)<4%f+P%zi%wFI9)L#$nH#!54gylp&fL>o3P{}~kRmcRT zp<-9wlciKUW;J_|UB|9rC*v>F$I2nVx<|Q5+%T>i{{{aK7vkIMzT&jn-a7YlpYR$Z z2^m8ytheb8~FC}96EjYD<9V~C`?>+bI{=U0CbAy^pZq7M( zpS9QCd+oi~Uh7+{?Vgs!P4BeCx`^`5eXFSD46l4|*}RhB@zR{}p=D--e=Ob4%m|$f zw?tmf`7pXSd?ZxqRQfM^Gl{ktowy^>keHpUN;V~iVJ4f^y5Q6&YzYr`SA~9(Ut3gL z{#CEPmR?-=ZhmKEnsdH?z^^w0o!^BniH^xh(#l&RAG;g;mEJ^ec(Shh+2qZs`Kc;z zL#iV6Kw@r3bL-faJtrre+R`=FtjaAcYA-1+UszUH*qC3D^C;0%hx{pI99R;njJzCu zCptLV6h7)!n-k=6xGd=-O7fTAG`j)@u@v6Kyf7aC1pf zSy@S+!ojgWMjvt~f=B+FsdtWrc1J#q+!|RGc|Tm_PB(Gnr6yIKYDhioU5bT$jrVlw z{$y2RKghe~?PcHYXi0X|`$Hp-{ey^AlV0=0`u1kyV5s__O9yAt2%bK7Pky-UDZ9*|3zeXY-iE1 zlIr5k@s+U=x%0w#p-HSrZO%7tVYmtea(Q@GsJH78BlfU!9jN2ZwC{bwnRK()C;d55 zVIL;9cTel8>Bwn2)beagL05@+G<=!*T^7o28iTxfB4 z8@{nMWKO7X{!1o?UB2V@Psh`X(v9FD2=4Z#rQBqk7@Ucq!LRvtZF_B^(rk%r%fFy- zb~KH-@LXvq1pA;Y@H>x=D8)kyaX*rHZ?uDt$RXOq@xi;&dAO^$!%sqWL@4{1?hNQJXtU|wjy_FG!!{MG{fzO4`7_T z*6ok?U>GLTa(u@9ofpiy^t;{yZx{HF$I-h-dQ;LTx!an!p=(^{E$z3rP3n3+HOX8Z zKALl5Y-oXrZ6Y4+aNhXbKSVza&!iVW=j{F$^x55L;~n(p|Jad+6Sp%UUFQ8J_3PBx zUN@@4V0MmIy}sT^vh~!FW9GTeA&Kd!bDXB|w?KbOLF-%_-xzx{zb&^qIww5I{lI*Q zXL+H&&%X;T(~**%%oTc0-@B=U;&ax&XB4I{Y^~{SVx$GoG0BXzzJw>r;ygXcf35$gLc_q z-ar%Ihc*{R;~RqBFNE$@P#KE{W+3{lbeB_nf0#G-fm@O1Wq(F9Z$bkM@qKwldllM5 zxam^O(w6N zd^OS=g}N4|T6&vu*#TbWSK6gA_PO{|q`OJij%MDpx|=jl>uZrVsa$^YDhaDCguC*% z$xET^bulOv@}~SOKDwN8wy7?UQ;yoAoP7nHfWq{-3!bKpt2Ctx=vDb=&gA=i#!{`# z;a=I?(iy!_SR9)hQ26o!MlOMlSVk)bP_7R=>%_#O zp>7ZSy2o4H#KD~6L@m73NNSXasEbqD0r)%&LaK-w4%7F_19@b|@c64gxUHfsN=bBr?{P)m;3&Gi+$*6xtJIiTpH)q?6@Y-Ao)um<;*>Z+M z{ZdN2Vm^1?f~#uErg_>xS@{YT_3@Z9nJhN0Q)_NwyfiBYFvDLoFM*tR*=%>NGxs}> z5(86jUNdu?Mc}Qjai*F^=jUb|XYor+opYi2z!_y`Gsj;id(S?k|0+%r?_oQ*8fsh3 z5?WNjig6A!syMfQ%*d{z+(2-1903oP^d9&pbeu=lkBPa-w5 ziGnS4cMwZFhe7ZY96yy(Lei` zPesiCNM_&p3Q7Z+>mT#9m7P67-<7i-6|nZ~K>q5X*T~8dW=(3QJ-Nh44a<1fm9?}8 zng=t8Or6am^pNswHnNT=((4G+v??EG-D^Tx6tD9>^ecJGBML$*z|TO4?q`Nv%-T{% zJBsK-2TM>fTq{>5p5)^MEA$C!k{v6}ig^}Z?>_v>LRZR+l3gjD z*K6rDIcx&EZJ&gi;?{!pB|0TvSLarN=t8RC)92#f4x3MMMzWa zb^1n`OSNNauh)L6y;Bik+Fi8A2}7YhK^7M6p|Xm|;-eh8vj316jrOT7S*VmxSN*1_ zGTD*Dop1oM>ZoSLn@K;=e`^opA+n@i(mo_Rl8vjfwj5DW{#M!LWC;>KdP9_Sr$6Zx z;?d^R)jpv=)e)2s7rGM-ai+HFPGw}fu~)YA>6w&+GMOqf_1p5Y`UK@|#&DZQH(R?j zk)V`XkgZi^)H4BX(Ftrz*(-f33YOo1?`+<8OF`f3nYBRKPtntr{Ry~G4~U|Dp1saq zefPqT{L7Znm9=@Xhx-_DQy8 zjePc5(C(nutX(ePAQ&yZ5sZLh8tu5r1{~C)vB^<#Wkj_oL0E>4%0)9*wiL(QJ6fMCWm`UI-4D4I YAAvN9H;q}gboT1>=eql$JlXO7KaQ(w@Bjb+ literal 0 HcmV?d00001 From ebbdbe636ae072dd8aabc098ba8df0e3fecf727a Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 31 Mar 2026 17:06:21 +0300 Subject: [PATCH 06/28] fix mmpeg --- src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs index 7f820dbe..56804e32 100644 --- a/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs +++ b/src/Sa.Media.FFmpeg/Services/ProcessExecutor.cs @@ -87,7 +87,8 @@ ProcessStartInfo startInfo } - int exitCode = await ExecuteProcessWithHandlersAsync(process, outputDataReceived, errorDataReceived, timeout, cancellationToken) + int exitCode = await ExecuteProcessWithHandlersAsync( + process, outputDataReceived, errorDataReceived, timeout, cancellationToken) .ConfigureAwait(false); return exitCode; @@ -152,7 +153,7 @@ private static async Task Run( process.BeginErrorReadLine(); } - using var timeoutCts = timeout.HasValue && timeout.Value == TimeSpan.Zero + using var timeoutCts = timeout.HasValue && timeout.Value != TimeSpan.Zero ? new CancellationTokenSource(timeout.Value) : null; From 8379359d96593e0d60bc06d985da2c6df30d04bd Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 31 Mar 2026 20:27:34 +0300 Subject: [PATCH 07/28] hybrid --- .../Domain/UploadFileInput.cs | 2 ++ .../HybridFileStorageExtensions.cs | 31 ++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs index 3e8b6002..3672b5ed 100644 --- a/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs +++ b/src/Sa.HybridFileStorage/Domain/UploadFileInput.cs @@ -4,4 +4,6 @@ public sealed record UploadFileInput { public int TenantId { get; init; } = 0; public string FileName { get; init; } = string.Empty; + + public static UploadFileInput Empty { get; } = new(); } diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index 9e171efc..e4acbd9c 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -5,7 +5,6 @@ namespace Sa.HybridFileStorage; public static class HybridFileStorageExtensions { - public static async Task CopyFromFileAsync( this IHybridFileStorage storage, string filePath, @@ -34,7 +33,7 @@ public static async Task CopyToBasketAsync( this IHybridFileStorage storage, string fileId, string basket, - int? targetTenantId = default, + Func? configure = null, CancellationToken ct = default) { var metadata = await storage.GetMetadataAsync(fileId, ct); @@ -43,19 +42,19 @@ public static async Task CopyToBasketAsync( { ThrowSourceFileNotFound(fileId); } - int tenantId = targetTenantId ?? metadata.TenantId; - if (tenantId == metadata.TenantId - && string.Equals(metadata.Basket, basket, StringComparison.Ordinal)) + UploadFileInput uploadInput = configure?.Invoke(metadata) ?? new UploadFileInput { - ThrowFileAlreadyInTargetScope(); - } + TenantId = metadata.TenantId, + FileName = metadata.FileName + }; - var uploadInput = new UploadFileInput + if (uploadInput.TenantId == metadata.TenantId + && string.Equals(metadata.Basket, basket, StringComparison.Ordinal) + && string.Equals(metadata.FileName, uploadInput.FileName, StringComparison.Ordinal)) { - FileName = metadata.FileName, - TenantId = tenantId, - }; + ThrowFileAlreadyInTargetScope(); + } StorageResult result = default!; bool downloaded = await storage.DownloadAsync( @@ -91,8 +90,8 @@ static void ThrowSourceFileNotFound(string fileId) => public static async Task> CopyToScopeBatchAsync( this IHybridFileStorage storage, IEnumerable fileIds, - string targetScopeName, - int? targetTenantId = default, + string basket, + Func? configure = null, BatchOptions? options = default, CancellationToken cancellationToken = default) { @@ -127,7 +126,11 @@ public static async Task> CopyToScopeBatchAsync( var ct = cts?.Token ?? cancellationToken; - return await storage.CopyToBasketAsync(fileId, targetScopeName, targetTenantId, ct); + return await storage.CopyToBasketAsync( + fileId, + basket, + configure, + ct: ct); } catch (Exception ex) { From 18521dbc23100a5a469edfae9764cd37b5e28ff8 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 1 Apr 2026 18:44:38 +0300 Subject: [PATCH 08/28] hybrid serialize --- src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs | 3 ++- .../Configuration/IPgOutboxConfiguration.cs | 3 +++ .../Configuration/PgOutboxConfiguration.cs | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs index e4acbd9c..e4d782be 100644 --- a/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs +++ b/src/Sa.HybridFileStorage/HybridFileStorageExtensions.cs @@ -10,6 +10,7 @@ public static async Task CopyFromFileAsync( string filePath, string basket, UploadFileInput input, + int bufferSize = 81920, CancellationToken ct = default) { // копируем файл в хранилище @@ -18,7 +19,7 @@ public static async Task CopyFromFileAsync( Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.Read, - BufferSize = 4096, + BufferSize = bufferSize, Options = FileOptions.Asynchronous | FileOptions.SequentialScan, }); diff --git a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs index f7f6796c..9be33a89 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/IPgOutboxConfiguration.cs @@ -1,5 +1,6 @@ using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.Serialization; +using System.Diagnostics.CodeAnalysis; namespace Sa.Outbox.PostgreSql.Configuration; @@ -9,6 +10,8 @@ IPgOutboxConfiguration WithMessageSerializer( Func messageSerializerFactory); IPgOutboxConfiguration WithMessageSerializer(TService instance) where TService : class, IOutboxMessageSerializer; + IPgOutboxConfiguration WithMessageSerializer<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TService>() + where TService : class, IOutboxMessageSerializer; IPgOutboxConfiguration WithOutboxSettings(Action? configure = null); IPgOutboxConfiguration WithDataSource(Action? configure = null); } diff --git a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs index 26831f11..3c21476f 100644 --- a/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs +++ b/src/Sa.Outbox.PostgreSql/Configuration/PgOutboxConfiguration.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.Data.PostgreSql; using Sa.Outbox.PostgreSql.Serialization; +using System.Diagnostics.CodeAnalysis; namespace Sa.Outbox.PostgreSql.Configuration; @@ -41,6 +42,14 @@ public IPgOutboxConfiguration WithMessageSerializer(TService instance) return this; } + public IPgOutboxConfiguration WithMessageSerializer<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TService>() + where TService : class, IOutboxMessageSerializer + { + services.RemoveAll(); + services.TryAddSingleton(); + return this; + } + internal IPgOutboxConfiguration WithDefaultSerializer() { services.TryAddSingleton(OutboxMessageSerializer.Instance); From e082dc5d0ab2290a7c204451d57cf99b9b214cc7 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 8 Apr 2026 16:39:49 +0300 Subject: [PATCH 09/28] ~ ResetLazy --- src/Sa/Classes/ResetLazy.cs | 96 +++++++------------ src/Sa/Classes/Retry.cs | 18 ++-- src/Tests/SaTests/Classes/ResetLazyTests.cs | 101 ++++++++++++++++++++ 3 files changed, 149 insertions(+), 66 deletions(-) create mode 100644 src/Tests/SaTests/Classes/ResetLazyTests.cs diff --git a/src/Sa/Classes/ResetLazy.cs b/src/Sa/Classes/ResetLazy.cs index 8c7c017f..62ea25d7 100644 --- a/src/Sa/Classes/ResetLazy.cs +++ b/src/Sa/Classes/ResetLazy.cs @@ -1,98 +1,73 @@ using System.Diagnostics; +using System.Runtime.CompilerServices; namespace Sa.Classes; -internal interface IResetLazy -{ - object? Value { get; } - void Reset(); - void Load(); -} - /// /// Provides support for lazy initialization with reset /// /// The type of object that is being lazily initialized. [DebuggerStepThrough] -internal sealed class ResetLazy(Func valueFactory, LazyThreadSafetyMode mode = LazyThreadSafetyMode.ExecutionAndPublication, Action? valueReset = null) : IResetLazy +internal sealed class ResetLazy( + Func valueFactory, + LazyThreadSafetyMode mode = LazyThreadSafetyMode.ExecutionAndPublication, + Action? valueReset = null) { - record Box(T Value); + private record Box(T Value); - private readonly Func _valueFactory = valueFactory ?? throw new ArgumentNullException(nameof(valueFactory)); + private readonly Func _valueFactory = valueFactory + ?? throw new ArgumentNullException(nameof(valueFactory)); private readonly Lock _syncLock = new(); - private Box? _box; + private volatile Box? _box; public T Value { - [DebuggerStepThrough] - get - { - Box? b1 = _box; - if (b1 != null) - return b1.Value; - - if (mode == LazyThreadSafetyMode.ExecutionAndPublication) - { - return LockExecutionAndPublication(); - } - else if (mode == LazyThreadSafetyMode.PublicationOnly) - { - return LockPublicationOnly(); - } - else - { - return CreateAndStoreValuw(); - } - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (_box ?? Initialize()).Value; } - private T CreateAndStoreValuw() - { - Box? b = new(CreateValue()); - _box = b; - return b.Value; - } + public bool IsValueCreated => _box != null; - private T LockPublicationOnly() + private Box Initialize() { - T newValue = CreateValue(); - - lock (_syncLock) + return mode switch { - Box? b2 = _box; - if (b2 != null) - return b2.Value; + LazyThreadSafetyMode.None => CreateAndStore(), + LazyThreadSafetyMode.PublicationOnly => CreatePublicationOnly(), + LazyThreadSafetyMode.ExecutionAndPublication => CreateExecutionAndPublication(), + _ => throw new ArgumentOutOfRangeException(nameof(mode)) + }; + } - _box = new Box(newValue); + private Box CreateAndStore() => _box = new(CreateValue()); - return _box.Value; - } + private Box CreatePublicationOnly() + { + var newValue = new Box(_valueFactory()); + // Если за это время кто-то уже записал значение, CompareExchange вернет старое, + // а наше новое просто уйдет в GC. + var existing = Interlocked.CompareExchange(ref _box, newValue, null); + return (existing ?? newValue); } - private T LockExecutionAndPublication() + private Box CreateExecutionAndPublication() { lock (_syncLock) { - Box? b2 = _box; - if (b2 != null) - return b2.Value; - - _box = new Box(CreateValue()); - - return _box.Value; + return _box ?? CreateAndStore(); } } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private T CreateValue() => _valueFactory(); public void Load() => _ = Value; - public bool IsValueCreated => _box != null; - object? IResetLazy.Value => Value; public void Reset() { @@ -111,10 +86,11 @@ public void Reset() private void ResetBox() { - if (IsValueCreated) + Box? oldBox = Interlocked.Exchange(ref _box, null); + + if (oldBox != null && valueReset != null) { - valueReset?.Invoke(_box!.Value); - _box = null; + valueReset(oldBox.Value); } } } diff --git a/src/Sa/Classes/Retry.cs b/src/Sa/Classes/Retry.cs index 657e448e..191efddd 100644 --- a/src/Sa/Classes/Retry.cs +++ b/src/Sa/Classes/Retry.cs @@ -243,7 +243,8 @@ public static IEnumerable GenerateConstant(TimeSpan delay, int retryCo } [DebuggerStepThrough] - public static IEnumerable GenerateLinear(TimeSpan initialDelay, int retryCount, double factor = 1.0, bool fastFirst = true) + public static IEnumerable GenerateLinear( + TimeSpan initialDelay, int retryCount, double factor = 1.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); if (factor < 0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 0"); @@ -252,7 +253,8 @@ public static IEnumerable GenerateLinear(TimeSpan initialDelay, int re } [DebuggerStepThrough] - public static IEnumerable GenerateExponential(TimeSpan initialDelay, int retryCount, double factor = 2.0, bool fastFirst = true) + public static IEnumerable GenerateExponential( + TimeSpan initialDelay, int retryCount, double factor = 2.0, bool fastFirst = true) { ValidateParameters(initialDelay, retryCount, nameof(initialDelay)); if (factor < 1.0) throw new ArgumentOutOfRangeException(nameof(factor), factor, "should be >= 1.0"); @@ -261,7 +263,8 @@ public static IEnumerable GenerateExponential(TimeSpan initialDelay, i } [DebuggerStepThrough] - public static IEnumerable GenerateJitter(TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst = true) + public static IEnumerable GenerateJitter( + TimeSpan medianFirstRetryDelay, int retryCount, bool fastFirst = true) { ValidateParameters(medianFirstRetryDelay, retryCount, nameof(medianFirstRetryDelay)); return retryCount == 0 ? Empty() : Generator.GenJitter(medianFirstRetryDelay, retryCount, fastFirst); @@ -284,7 +287,8 @@ public static IEnumerable GenConstant(TimeSpan delay, int retryCount, } } - public static IEnumerable GenLinear(TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) + public static IEnumerable GenLinear( + TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) { @@ -300,7 +304,8 @@ public static IEnumerable GenLinear(TimeSpan initialDelay, int retryCo } } - public static IEnumerable GenExponential(TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) + public static IEnumerable GenExponential( + TimeSpan initialDelay, int retryCount, double factor, bool fastFirst) { if (fastFirst) { @@ -335,7 +340,8 @@ public static IEnumerable GenJitter(TimeSpan medianFirstRetryDelay, in double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(pFactor * t)); double formulaIntrinsicValue = next - prev; - yield return TimeSpan.FromTicks((long)Math.Min(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); + yield return TimeSpan.FromTicks( + (long)Math.Min(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); prev = next; } } diff --git a/src/Tests/SaTests/Classes/ResetLazyTests.cs b/src/Tests/SaTests/Classes/ResetLazyTests.cs new file mode 100644 index 00000000..cbe77930 --- /dev/null +++ b/src/Tests/SaTests/Classes/ResetLazyTests.cs @@ -0,0 +1,101 @@ +using Sa.Classes; +using System.Collections.Concurrent; + +namespace SaTests.Classes; + +public class ResetLazyTests +{ + [Fact] + public void Value_ShouldBeInitializedLazy() + { + // Arrange + int counter = 0; + var lazy = new ResetLazy(() => Interlocked.Increment(ref counter)); + + // Assert + Assert.Equal(0, counter); + Assert.False(lazy.IsValueCreated); + + // Act + var val = lazy.Value; + + // Assert + Assert.Equal(1, val); + Assert.Equal(1, counter); + Assert.True(lazy.IsValueCreated); + } + + [Fact] + public void Reset_ShouldClearValue_AndCallCleanup() + { + // Arrange + int counter = 0; + int cleanupValue = -1; + var lazy = new ResetLazy>( + () => [++counter], + valueReset: list => cleanupValue = list[0] + ); + + // Act + var firstList = lazy.Value; + lazy.Reset(); + + // Assert + Assert.False(lazy.IsValueCreated); + Assert.Equal(1, cleanupValue); // Cleanup вызван для первого списка + + var secondList = lazy.Value; + Assert.Equal(2, secondList[0]); // Создан новый список + } + + [Fact] + public async Task MultithreadedAccess_ShouldCreateValueOnlyOnce() + { + // Arrange + int factoryCalls = 0; + var lazy = new ResetLazy(() => + { + Interlocked.Increment(ref factoryCalls); + Thread.Sleep(50); // Симуляция работы + return Guid.NewGuid().ToString(); + }); + + // Act + var tasks = Enumerable.Range(0, 50) + .Select(_ => Task.Run(() => lazy.Value)) + .ToList(); + + var results = await Task.WhenAll(tasks); + + // Assert + Assert.Equal(1, factoryCalls); // Фабрика вызвана ровно 1 раз + Assert.All(results, r => Assert.Equal(results[0], r)); // Все потоки получили одну строку + } + + [Fact] + public async Task MultithreadedReset_ShouldBeSafe() + { + // Arrange + var disposedItems = new ConcurrentBag(); + int factoryCalls = 0; + var lazy = new ResetLazy( + () => Interlocked.Increment(ref factoryCalls), + valueReset: val => disposedItems.Add(val) + ); + + // Act + // Интенсивно читаем и сбрасываем из разных потоков + var tasks = Enumerable.Range(0, 100).Select(i => Task.Run(() => + { + if (i % 2 == 0) _ = lazy.Value; + else lazy.Reset(); + })); + + await Task.WhenAll(tasks); + + // Assert + // Проверяем, что сумма созданных объектов совпадает с суммой удаленных + 1 (если текущий остался) + int currentExist = lazy.IsValueCreated ? 1 : 0; + Assert.Equal(factoryCalls, disposedItems.Count + currentExist); + } +} From 6e24dd0fe28e3e7cdeae6614b2fe02b27ab1c160 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 8 Apr 2026 16:44:38 +0300 Subject: [PATCH 10/28] ~ ResetLazy --- src/Sa/Classes/ResetLazy.cs | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/src/Sa/Classes/ResetLazy.cs b/src/Sa/Classes/ResetLazy.cs index 62ea25d7..e0637b2d 100644 --- a/src/Sa/Classes/ResetLazy.cs +++ b/src/Sa/Classes/ResetLazy.cs @@ -46,11 +46,9 @@ private Box Initialize() private Box CreatePublicationOnly() { - var newValue = new Box(_valueFactory()); - // Если за это время кто-то уже записал значение, CompareExchange вернет старое, - // а наше новое просто уйдет в GC. - var existing = Interlocked.CompareExchange(ref _box, newValue, null); - return (existing ?? newValue); + var newBox = new Box(_valueFactory()); + var existing = Interlocked.CompareExchange(ref _box, newBox, null); + return (existing ?? newBox); } private Box CreateExecutionAndPublication() @@ -70,21 +68,6 @@ private Box CreateExecutionAndPublication() public void Reset() - { - if (mode != LazyThreadSafetyMode.None) - { - lock (_syncLock) - { - ResetBox(); - } - } - else - { - ResetBox(); - } - } - - private void ResetBox() { Box? oldBox = Interlocked.Exchange(ref _box, null); From fc4983b3a5f30347461b4a712920f830608990fd Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 8 Apr 2026 16:55:42 +0300 Subject: [PATCH 11/28] resetLazy tests --- src/Sa/Classes/ResetLazy.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Sa/Classes/ResetLazy.cs b/src/Sa/Classes/ResetLazy.cs index e0637b2d..06deb567 100644 --- a/src/Sa/Classes/ResetLazy.cs +++ b/src/Sa/Classes/ResetLazy.cs @@ -14,7 +14,7 @@ internal sealed class ResetLazy( LazyThreadSafetyMode mode = LazyThreadSafetyMode.ExecutionAndPublication, Action? valueReset = null) { - private record Box(T Value); + private sealed record Box(T Value); private readonly Func _valueFactory = valueFactory ?? throw new ArgumentNullException(nameof(valueFactory)); @@ -38,7 +38,7 @@ private Box Initialize() LazyThreadSafetyMode.None => CreateAndStore(), LazyThreadSafetyMode.PublicationOnly => CreatePublicationOnly(), LazyThreadSafetyMode.ExecutionAndPublication => CreateExecutionAndPublication(), - _ => throw new ArgumentOutOfRangeException(nameof(mode)) + _ => throw new InvalidOperationException($"Unsupported thread safety mode: {mode}") }; } From f53c00a1214478e273d73948c5211db2de08c969 Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 8 Apr 2026 17:56:55 +0300 Subject: [PATCH 12/28] classes --- src/Sa/Classes/LockRenewer.cs | 11 +- src/Sa/Classes/WorkQueue.cs | 318 +++++++++++++++----- src/Tests/SaTests/Classes/WorkQueueTests.cs | 28 +- 3 files changed, 270 insertions(+), 87 deletions(-) diff --git a/src/Sa/Classes/LockRenewer.cs b/src/Sa/Classes/LockRenewer.cs index 05089426..c601e0da 100644 --- a/src/Sa/Classes/LockRenewer.cs +++ b/src/Sa/Classes/LockRenewer.cs @@ -4,7 +4,11 @@ namespace Sa.Classes; internal static class LockRenewer { - public static IDisposable KeepLocked(TimeSpan lockExpiration, Func extendLocked, bool blockImmediately = false, CancellationToken cancellationToken = default) + public static IDisposable KeepLocked( + TimeSpan lockExpiration, + Func extendLocked, + bool blockImmediately = false, + CancellationToken cancellationToken = default) { var timer = new PeriodicTimer(lockExpiration); var task = Task.Run(async () => @@ -55,8 +59,11 @@ public static async Task WaitForConditionAsync( { while (sw.Elapsed < timeout) { - if (!cancellationToken.IsCancellationRequested && await predicate(cancellationToken).ConfigureAwait(false)) + if (!cancellationToken.IsCancellationRequested + && await predicate(cancellationToken).ConfigureAwait(false)) + { return true; + } await Task.Delay(interval, cancellationToken); } diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index eb73be00..f1b97b74 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -1,19 +1,29 @@ -using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using System.Threading.Channels; namespace Sa.Classes; +/// +/// Интерфейс для выполнения бизнес-логики. +/// internal interface IWork { Task Execute(TModel model, CancellationToken cancellationToken); } +/// +/// Интерфейс для наблюдения за изменениями состояния задач в очереди. +/// internal interface IWorkObserver { Task HandleChanges(TModel model, WorkInfo work, CancellationToken cancellationToken); } -internal record struct WorkInfo( +/// +/// Представляет состояние и метаданные задачи в очереди. +/// +internal sealed record WorkInfo( long Id, WorkStatus Status, DateTimeOffset EnqueuedTime, @@ -22,7 +32,7 @@ internal record struct WorkInfo( Exception? LastError = null ) { - public readonly bool IsEmpty => Id == 0; + public bool IsEmpty => Id == 0; } internal enum WorkStatus @@ -34,71 +44,107 @@ internal enum WorkStatus Cancelled } +/// +/// Асинхронная очередь задач с ограничением параллелизма. +/// internal interface IWorkQueue : IDisposable, IAsyncDisposable { + /// + /// Показывает, включена ли обработка задач. + /// bool IsEnabled { get; } + + /// + /// Текущее количество активно выполняющихся задач. + /// int ActiveTasks { get; } + + /// + /// Определяет, свободна ли очередь (нет активных и нет в очереди). + /// bool IsIdle(); + + /// + /// Максимальное количество параллельных задач (динамически изменяется). + /// int ConcurrencyLimit { get; set; } + /// + /// Добавляет задачу в очередь асинхронно. + /// ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default); + /// + /// Блокируется до момента, когда очередь станет полностью пустой. + /// Task WaitForIdleAsync(CancellationToken cancellationToken = default); - Task ShutdownAsync(); - event EventHandler? StatusChanged; + /// + /// Завершает обработку и отменяет ожидание новых задач. + /// + Task ShutdownAsync(); } internal sealed class WorkQueue : IWorkQueue { - private readonly Channel _queue; - - private readonly List _activeItems = []; + private readonly Channel _queue; + private readonly HashSet _activeItemIds = []; private readonly Lock _rootSync = new(); - - private int _maxConcurrency = Environment.ProcessorCount; - private long _taskIdCounter = 0; - private bool _isEnabled = true; - private bool _disposed = false; - - private readonly Task _processingTask; + private readonly SemaphoreSlim _concurrencySemaphore; + private readonly ILogger? _logger = null; private readonly TimeProvider _timeProvider; - public readonly IWork _processor; public readonly IWorkObserver? _watcher; - private readonly CancellationTokenSource _shutdownCts = new(); + private readonly Task _processingTask; + private readonly Action? _statusChanged; + + private volatile TaskCompletionSource _idleTcs = new(); + private volatile int _activeCount = 0; + private volatile bool _isEnabled = true; - public event EventHandler? StatusChanged; + private int _maxConcurrency; + private long _taskIdCounter = 0; + private bool _disposed = false; public WorkQueue( IWork processor, IWorkObserver? observer = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + int? maxQueueCapacity = null, + int? initialConcurrencyLimit = null, + Action? statusChanged = null, + ILogger>? logger = null) { _processor = processor; _watcher = observer ?? processor as IWorkObserver; _timeProvider = timeProvider ?? TimeProvider.System; + _maxConcurrency = initialConcurrencyLimit ?? Environment.ProcessorCount; + _statusChanged = statusChanged; + _logger = logger; - _queue = Channel.CreateUnbounded(new UnboundedChannelOptions + _queue = Channel.CreateBounded(new BoundedChannelOptions(maxQueueCapacity ?? 1000) { AllowSynchronousContinuations = true, SingleReader = false, SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait }); + _concurrencySemaphore = new SemaphoreSlim(_maxConcurrency, _maxConcurrency); + _processingTask = LoopAsync(); } public bool IsEnabled => _isEnabled; - public int ActiveTasks => _activeItems.Count; + public int ActiveTasks => _activeCount; public int QueuedTasks => _queue.Reader.Count; public bool IsIdle() { lock (_rootSync) { - return _activeItems.Count == 0 && _queue.Reader.Count == 0; + return _activeItemIds.Count == 0 && _queue.Reader.Count == 0; } } @@ -108,44 +154,75 @@ public int ConcurrencyLimit set { ObjectDisposedException.ThrowIf(_disposed, this); - if (value < 1) throw new ArgumentOutOfRangeException(nameof(value), "Must be at least 1."); - _maxConcurrency = value; + ArgumentOutOfRangeException.ThrowIfLessThan(value, 1); + + var oldValue = Interlocked.Exchange(ref _maxConcurrency, value); + int delta = value - oldValue; + + if (delta > 0) + { + try + { + _concurrencySemaphore.Release(delta); + } + catch (SemaphoreFullException) + { + // Игнорируем, если семафор уже на максимуме + } + // Если уменьшаем — новые задачи просто не смогут захватить слот, + // а текущие завершатся естественным образом + } } } public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); - if (!_isEnabled) throw new InvalidOperationException("Task scheduler has been stopped."); + if (!_isEnabled) + throw new InvalidOperationException("Task scheduler has been stopped."); - var shapshot = new WorkItemSnapshot(model, cancellationToken) + var snapshot = new WorkItem(model, cancellationToken) { Id = Interlocked.Increment(ref _taskIdCounter), Status = WorkStatus.Queued, EnqueuedTime = _timeProvider.GetUtcNow(), }; - await OnStatusChanged(shapshot); - - await _queue.Writer.WriteAsync(shapshot, cancellationToken); + await OnStatusChangedAsync(snapshot); + await _queue.Writer.WriteAsync(snapshot, cancellationToken); } public async Task ShutdownAsync() { if (!_isEnabled || _disposed) return; - _isEnabled = false; + + // _queue.Writer.Complete(); + await _shutdownCts.CancelAsync(); - await _processingTask; + try + { + await _processingTask; + } + catch (Exception ex) + { + _logger?.LogError(ex, "Error during shutdown"); + } } public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); + while (!IsIdle()) { - await Task.Delay(100, cancellationToken); + var currentTcs = _idleTcs; + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + // Ждем сигнал от TCS или отмену. + // Если ActivateItem вызовется, currentTcs будет завершен -> цикл повторится. + var completedTask = await Task.WhenAny(currentTcs.Task, Task.Delay(Timeout.Infinite, cts.Token)); } } @@ -156,11 +233,11 @@ private async Task LoopAsync() { while (!ct.IsCancellationRequested && await _queue.Reader.WaitToReadAsync(ct)) { - while (_isEnabled - && ActiveTasks < _maxConcurrency - && _queue.Reader.TryRead(out var snapshot)) + // Каждый поток пытается читать, пока есть данные + while (_isEnabled && _queue.Reader.TryRead(out var item)) { - _ = Execute(snapshot); // Не ждём, запускаем параллельно + // SemaphoreSlim автоматически ограничит реальную параллельность + _ = ExecuteWithConcurrencyControlAsync(item, ct); } } } @@ -168,77 +245,162 @@ private async Task LoopAsync() { /* ignore */ } + catch (Exception ex) + { + _logger?.LogError(ex, "Unhandled error in queue processing loop"); + } } - private async Task Execute(WorkItemSnapshot snapshot) + // Обёртка: захват слота -> выполнение -> освобождение + private async Task ExecuteWithConcurrencyControlAsync(WorkItem item, CancellationToken loopCt) { - ActivateSnapshot(snapshot); + // Ждём доступного слота (с учётом отмены) + try + { + await _concurrencySemaphore.WaitAsync(loopCt); + } + catch (OperationCanceledException) + { + // Если отменили ожидание слота — возвращаем задачу в очередь или отменяем + await CancelWorkItemAsync(item); + return; + } + + + try + { + // Проверяем, не остановили ли очередь пока ждали слот + if (!_isEnabled) + { + await CancelWorkItemAsync(item); + return; + } + + await ExecuteAsync(item); + } + catch (Exception ex) + { + _logger?.LogError(ex, "Unhandled error executing work item #{WorkItemId}", item.Id); + + if (item.Status is WorkStatus.Queued or WorkStatus.Running) + { + item.Status = WorkStatus.Faulted; + item.LastError = ex; + item.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(item); + } + } + finally + { + _concurrencySemaphore.Release(); // Освобождаем слот + } + } + + private async Task CancelWorkItemAsync(WorkItem item) + { + item.Status = WorkStatus.Cancelled; + item.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(item); + } + + private async Task ExecuteAsync(WorkItem item) + { + ActivateItem(item); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + _shutdownCts.Token, + item.CancellationToken); - using var cts = CancellationTokenSource.CreateLinkedTokenSource(_shutdownCts.Token, snapshot.CancellationToken); var ct = cts.Token; + try { ct.ThrowIfCancellationRequested(); - snapshot.StartedTime = _timeProvider.GetUtcNow(); - snapshot.Status = WorkStatus.Running; - await OnStatusChanged(snapshot); + item.StartedTime = _timeProvider.GetUtcNow(); + item.Status = WorkStatus.Running; + await OnStatusChangedAsync(item); - await _processor.Execute(snapshot.Model, ct); - snapshot.Status = WorkStatus.Completed; + await _processor.Execute(item.Model, ct); + item.Status = WorkStatus.Completed; } catch (OperationCanceledException) when (ct.IsCancellationRequested) { - snapshot.Status = WorkStatus.Cancelled; + item.Status = WorkStatus.Cancelled; } catch (Exception ex) { - snapshot.Status = WorkStatus.Faulted; - snapshot.LastError = ex; + item.Status = WorkStatus.Faulted; + item.LastError = ex; } finally { - snapshot.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChanged(snapshot); - - DeactiveSnapshot(snapshot); + item.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(item); + DeactivateItem(item); } } - private void ActivateSnapshot(WorkItemSnapshot snapshot) + private void ActivateItem(WorkItem item) { lock (_rootSync) { - _activeItems.Add(snapshot); + _activeItemIds.Add(item.Id); + Volatile.Write(ref _activeCount, _activeItemIds.Count); + + var oldTcs = _idleTcs; + _idleTcs = new TaskCompletionSource(); + oldTcs.TrySetResult(); // Пробуждаем ожидающие потоки } } - private void DeactiveSnapshot(WorkItemSnapshot snapshot) + private void DeactivateItem(WorkItem item) { + TaskCompletionSource? tcsToNotify = null; + lock (_rootSync) { - _activeItems.Remove(snapshot); + _activeItemIds.Remove(item.Id); + Volatile.Write(ref _activeCount, _activeItemIds.Count); + + if (_activeItemIds.Count == 0 && _queue.Reader.Count == 0) + { + tcsToNotify = _idleTcs; + } } + + tcsToNotify?.TrySetResult(); } - private async Task OnStatusChanged(WorkItemSnapshot snapshot) + private async Task OnStatusChangedAsync(WorkItem item) { - var info = snapshot.ToInfo(); + var info = item.ToInfo(); + var watcher = _watcher; - if (_watcher != null) + if (watcher != null) { try { - await _watcher.HandleChanges(snapshot.Model, info, _shutdownCts.Token); + await watcher.HandleChanges(item.Model, info, _shutdownCts.Token); + } + catch (Exception ex) + { + _logger?.LogError(ex, "Observer failed for work item #{WorkItemId}", info.Id); } - catch { /* ignore */ } } - try + if (_statusChanged != null) { - StatusChanged?.Invoke(this, info); + try + { + _statusChanged(info); + } + catch (Exception ex) + { + _logger?.LogError(ex, "Event handler failed for work item #{WorkItemId}", info.Id); + } + } - catch { /* ignore */ } } public void Dispose() @@ -247,11 +409,20 @@ public void Dispose() _disposed = true; _isEnabled = false; - _queue.Writer.Complete(); - + _queue.Writer.TryComplete(); _shutdownCts.Cancel(); - _processingTask.Wait(5000); + try + { + _processingTask.Wait(TimeSpan.FromSeconds(5)); + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + _logger?.LogError(ex, "Error during synchronous disposal"); + } + + _concurrencySemaphore.Dispose(); _shutdownCts.Dispose(); } @@ -261,15 +432,22 @@ public async ValueTask DisposeAsync() _disposed = true; _isEnabled = false; - _queue.Writer.Complete(); + _queue.Writer.TryComplete(); await _shutdownCts.CancelAsync(); - await _processingTask; - + try + { + await _processingTask; + } + catch (Exception ex) + { + _logger?.LogError(ex, "Error during synchronous disposal"); + } + _concurrencySemaphore.Dispose(); _shutdownCts.Dispose(); } - private sealed class WorkItemSnapshot(TModel item, CancellationToken token) + private sealed class WorkItem(TModel item, CancellationToken token) { public long Id { get; init; } public TModel Model => item; diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index 40bd5ed7..e653736d 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -99,13 +99,11 @@ public async Task Enqueue_MultipleTasks_AllExecuted() public async Task ConcurrencyLimit_Respected() { // Arrange - var concurrencyLimit = 3; var delay = TimeSpan.FromMilliseconds(120); var processor = new TestWorkWithDelay(delay); - using var queue = new WorkQueue(processor) - { - ConcurrencyLimit = concurrencyLimit - }; + + var concurrencyLimit = 3; + using var queue = new WorkQueue(processor, initialConcurrencyLimit: concurrencyLimit); var models = new List { new(), new(), new(), new(), new(), new(), new() }; @@ -120,7 +118,6 @@ public async Task ConcurrencyLimit_Respected() await queue.WaitForIdleAsync(cancellationToken: TestToken); - Console.WriteLine(queue.ActiveTasks); Assert.Equal(0, queue.ActiveTasks); Assert.Equal(0, queue.QueuedTasks); @@ -207,14 +204,14 @@ public async Task StatusChanged_Event_FiresOnStateChange() WorkInfo? lastInfo = null; var eventTcs = new TaskCompletionSource(); - using var queue = new WorkQueue(processor); - - queue.StatusChanged += (sender, info) => - { - lastInfo = info; - if (info.Status == WorkStatus.Completed) - eventTcs.TrySetResult(info); - }; + using var queue = new WorkQueue( + processor, + statusChanged: (info) => + { + lastInfo = info; + if (info.Status == WorkStatus.Completed) + eventTcs.TrySetResult(info); + }); var model = new TestModel(); @@ -281,6 +278,7 @@ public async Task Enqueue_Disposed_Throws() var queue = new WorkQueue(new TestWork()); await queue.DisposeAsync(); - await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); + await Assert.ThrowsAsync(() + => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); } } From b1d1d826cd8608d91afb57e27bb39b47d1cc3c64 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 9 Apr 2026 11:54:34 +0300 Subject: [PATCH 13/28] outbox multi cunsomer --- .../Services/Plug/OutboxDeliveryManager.cs | 34 +++++++++---------- src/Sa.Outbox/Delivery/DeliveryProcessor.cs | 11 +++--- src/Sa.Outbox/Delivery/DeliveryTenant.cs | 8 ++--- src/Sa/Classes/WorkQueue.cs | 13 +++---- src/Tests/SaTests/Classes/WorkQueueTests.cs | 3 ++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs index 6bbf80d5..b7005e9c 100644 --- a/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs +++ b/src/Sa.Outbox.PostgreSql/Services/Plug/OutboxDeliveryManager.cs @@ -33,17 +33,6 @@ public async Task RentDelivery( return await startCmd.ExecuteFill(writeBuffer, lockDuration, filter, cancellationToken); } - private async Task EnsureParts(OutboxMessageFilter filter, CancellationToken cancellationToken) - { - await partRepository.EnsureMsgParts( - [new OutboxPartInfo(filter.TenantId, filter.Part, filter.NowDate)], - cancellationToken); - - await partRepository.EnsureTaskParts( - [new OutboxPartInfo(filter.TenantId, filter.ConsumerGroupId, filter.NowDate)], - cancellationToken); - } - public async Task ReturnDelivery( ReadOnlyMemory> messages, OutboxMessageFilter filter, @@ -61,6 +50,23 @@ public async Task ReturnDelivery( return await finishCmd.Execute(messages, errors, filter, cancellationToken); } + public Task ExtendDelivery( + TimeSpan lockExpiration, + OutboxMessageFilter filter, + CancellationToken cancellationToken) + => extendCmd.Execute(lockExpiration, filter, cancellationToken); + + private async Task EnsureParts(OutboxMessageFilter filter, CancellationToken cancellationToken) + { + await partRepository.EnsureMsgParts( + [new OutboxPartInfo(filter.TenantId, filter.Part, filter.NowDate)], + cancellationToken); + + await partRepository.EnsureTaskParts( + [new OutboxPartInfo(filter.TenantId, filter.ConsumerGroupId, filter.NowDate)], + cancellationToken); + } + private async ValueTask> GetErrors( ReadOnlyMemory> messages, CancellationToken cancellationToken) @@ -79,10 +85,4 @@ private async ValueTask> GetErrors ExtendDelivery( - TimeSpan lockExpiration, - OutboxMessageFilter filter, - CancellationToken cancellationToken) - => extendCmd.Execute(lockExpiration, filter, cancellationToken); } diff --git a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs index 080a9b60..27790942 100644 --- a/src/Sa.Outbox/Delivery/DeliveryProcessor.cs +++ b/src/Sa.Outbox/Delivery/DeliveryProcessor.cs @@ -41,7 +41,7 @@ public async Task ProcessMessages( continueProcessing = ShouldContinueProcessing( sentCount, iterations, - settings.ConsumeSettings, + consumeSettings, cancellationToken); } while (continueProcessing); @@ -72,12 +72,9 @@ private async Task ProcessForEachTenant( ConsumerGroupSettings settings, CancellationToken cancellationToken) { - if (settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism == 1) - { - return await ProcessTenantsSequential(tenantIds, settings, cancellationToken); - } - - return await ProcessTenantsParallel(tenantIds, settings, cancellationToken); + return (settings.ConsumeSettings.PerTenantMaxDegreeOfParallelism == 1) + ? await ProcessTenantsSequential(tenantIds, settings, cancellationToken) + : await ProcessTenantsParallel(tenantIds, settings, cancellationToken); } private async Task ProcessTenantsSequential( diff --git a/src/Sa.Outbox/Delivery/DeliveryTenant.cs b/src/Sa.Outbox/Delivery/DeliveryTenant.cs index b9921c30..4356996c 100644 --- a/src/Sa.Outbox/Delivery/DeliveryTenant.cs +++ b/src/Sa.Outbox/Delivery/DeliveryTenant.cs @@ -60,11 +60,6 @@ private OutboxMessageFilter CreateFilter(int tenantId, ConsumerGroupSe private DateTimeOffset GetUtcNow() => _timeProvider.GetUtcNow(); - private static IMemoryOwner> RentMemory(int size) - { - return MemoryPool>.Shared.Rent(size); - } - private async Task CalculateBatchSizeAsync( ConsumeSettings consumeSettings, OutboxMessageFilter filter, @@ -124,4 +119,7 @@ private IDisposable RenewerLocker( , t); } , cancellationToken: cancellationToken); + + private static IMemoryOwner> RentMemory(int size) + => MemoryPool>.Shared.Rent(size); } diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index f1b97b74..22d8a119 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -197,8 +197,6 @@ public async Task ShutdownAsync() if (!_isEnabled || _disposed) return; _isEnabled = false; - // _queue.Writer.Complete(); - await _shutdownCts.CancelAsync(); try @@ -222,7 +220,7 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default // Ждем сигнал от TCS или отмену. // Если ActivateItem вызовется, currentTcs будет завершен -> цикл повторится. - var completedTask = await Task.WhenAny(currentTcs.Task, Task.Delay(Timeout.Infinite, cts.Token)); + await Task.WhenAny(currentTcs.Task, Task.Delay(Timeout.Infinite, cts.Token)); } } @@ -346,7 +344,7 @@ private void ActivateItem(WorkItem item) lock (_rootSync) { _activeItemIds.Add(item.Id); - Volatile.Write(ref _activeCount, _activeItemIds.Count); + _activeCount = _activeItemIds.Count; var oldTcs = _idleTcs; _idleTcs = new TaskCompletionSource(); @@ -361,7 +359,7 @@ private void DeactivateItem(WorkItem item) lock (_rootSync) { _activeItemIds.Remove(item.Id); - Volatile.Write(ref _activeCount, _activeItemIds.Count); + _activeCount = _activeItemIds.Count; if (_activeItemIds.Count == 0 && _queue.Reader.Count == 0) { @@ -416,7 +414,10 @@ public void Dispose() { _processingTask.Wait(TimeSpan.FromSeconds(5)); } - catch (OperationCanceledException) { } + catch (OperationCanceledException) + { + // ignore + } catch (Exception ex) { _logger?.LogError(ex, "Error during synchronous disposal"); diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index e653736d..49b265a7 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -140,6 +140,7 @@ public async Task FaultedTask_StatusIsFaulted() // Assert var completed = observer.Changes.Find(x => x.Status == WorkStatus.Faulted); + Assert.NotNull(completed); Assert.IsType(completed.LastError); Assert.Contains("Test exception", completed.LastError?.Message); } @@ -163,6 +164,8 @@ public async Task CancelledTask_StatusIsCancelled() // Assert var cancelled = observer.Changes.Find(x => x.Status == WorkStatus.Cancelled); + + Assert.NotNull(cancelled); Assert.NotEqual(0, cancelled.Id); } From 24475d033ab752a1bc39df01d4314cbe45a71f42 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 9 Apr 2026 19:09:28 +0300 Subject: [PATCH 14/28] ~ sked --- src/.gitignore | 3 ++- src/Sa.Schedule/Engine/JobController.cs | 12 ++++++++++-- src/Sa.Schedule/Engine/JobPipeline.cs | 8 ++++++-- src/Sa.Schedule/Engine/JobScheduler.cs | 9 ++++++--- src/Sa.Schedule/Engine/JobTiming.cs | 9 ++++++--- src/Sa.Schedule/Engine/Scheduler.cs | 12 ++++++++---- src/Sa.Schedule/Settings/ScheduleBuilder.cs | 15 +++++++++------ src/Sa.Schedule/Settings/ScheduleSettings.cs | 8 ++++++-- 8 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/.gitignore b/src/.gitignore index 1ce70e5a..c8e63aed 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -405,4 +405,5 @@ FodyWeavers.xsd Sa.Media.FFmpeg/runtimes/native/ nupkgs/ -.packages/ \ No newline at end of file +.packages/ +*.lscache \ No newline at end of file diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index 693ad5a7..f5c13dfa 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -25,7 +25,9 @@ internal sealed partial class JobController( public async ValueTask WaitToRun(CancellationToken cancellationToken) { - if (context.NumRuns == 0 && settings.Properties.InitialDelay.HasValue && settings.Properties.InitialDelay.Value != TimeSpan.Zero) + if (context.NumRuns == 0 + && settings.Properties.InitialDelay.HasValue + && settings.Properties.InitialDelay.Value != TimeSpan.Zero) { context.Status = JobStatus.WaitingToRun; await Task.Delay(settings.Properties.InitialDelay.Value, cancellationToken); @@ -126,7 +128,13 @@ public void ExecutionFailed(Exception exception) if (context.FailedRetries < settings.ErrorHandling.RetryCount) { context.FailedRetries++; - LogFailedRetryAttempts(context.Logger, context.JobName, context.FailedRetries, errorHandling.RetryCount, exception.GetType().Name, exception.Message); + LogFailedRetryAttempts( + context.Logger, + context.JobName, + context.FailedRetries, + errorHandling.RetryCount, + exception.GetType().Name, + exception.Message); return; } diff --git a/src/Sa.Schedule/Engine/JobPipeline.cs b/src/Sa.Schedule/Engine/JobPipeline.cs index d17009e6..1ff5ac8b 100644 --- a/src/Sa.Schedule/Engine/JobPipeline.cs +++ b/src/Sa.Schedule/Engine/JobPipeline.cs @@ -34,7 +34,10 @@ public JobPipeline( .Interceptors .Reverse() .Aggregate(originalJob, (job, s) - => new JobProxy(job, (IJobInterceptor)_scope.ServiceProvider.GetRequiredKeyedService(s.HandlerType, s.Key), s.Key)); + => new JobProxy( + job, + (IJobInterceptor)_scope.ServiceProvider.GetRequiredKeyedService(s.HandlerType, s.Key), + s.Key)); } else { @@ -46,5 +49,6 @@ public JobPipeline( public void Dispose() => _scope.Dispose(); - public Task Execute(IJobContext context, CancellationToken cancellationToken) => _job.Execute(context, cancellationToken); + public Task Execute(IJobContext context, CancellationToken cancellationToken) + => _job.Execute(context, cancellationToken); } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 2e0c7d6b..63a3461f 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -2,9 +2,11 @@ namespace Sa.Schedule.Engine; -internal sealed class JobScheduler(IJobRunner runner, IJobController controller) : IJobScheduler, IDisposable, IAsyncDisposable +internal sealed class JobScheduler(IJobRunner runner, IJobController controller) + : IJobScheduler, IDisposable, IAsyncDisposable { - private readonly static IChangeToken NoneChangeToken = new CancellationChangeToken(CancellationToken.None); + private readonly static IChangeToken NoneChangeToken + = new CancellationChangeToken(CancellationToken.None); private readonly Lock _locked = new(); @@ -17,7 +19,8 @@ internal sealed class JobScheduler(IJobRunner runner, IJobController controller) public IJobContext Context => controller.Context; - public bool IsActive => _stoppingTask?.Task.Status == TaskStatus.WaitingForActivation; + public bool IsActive => _stoppingTask?.Task.Status + == TaskStatus.WaitingForActivation; public IChangeToken GetActiveChangeToken() { diff --git a/src/Sa.Schedule/Engine/JobTiming.cs b/src/Sa.Schedule/Engine/JobTiming.cs index 05b986b1..a8551132 100644 --- a/src/Sa.Schedule/Engine/JobTiming.cs +++ b/src/Sa.Schedule/Engine/JobTiming.cs @@ -1,12 +1,15 @@ namespace Sa.Schedule.Engine; -internal sealed class JobTiming(Func nextTime, string name) : IJobTiming +internal sealed class JobTiming( + Func nextTime, string name) : IJobTiming { public string TimingName => name; - public DateTimeOffset? GetNextOccurrence(DateTimeOffset dateTime, IJobContext context) => nextTime(dateTime, context); + public DateTimeOffset? GetNextOccurrence( + DateTimeOffset dateTime, IJobContext context) => nextTime(dateTime, context); public static IJobTiming EveryTime(TimeSpan timeSpan, string? name = null) => new JobTiming((dateTime, _) => dateTime.Add(timeSpan), name ?? $"every {timeSpan}"); - public static IJobTiming Default { get; } = EveryTime(TimeSpan.FromSeconds(1), "default every seconds"); + public static IJobTiming Default { get; } + = EveryTime(TimeSpan.FromSeconds(1), "default every seconds"); } diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index 437e177e..6acf8676 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -1,6 +1,7 @@ namespace Sa.Schedule.Engine; -internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) : IScheduler, IDisposable, IAsyncDisposable +internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) + : IScheduler, IDisposable, IAsyncDisposable { private bool _disposed; @@ -13,11 +14,14 @@ internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) /// /// Start all jobs /// - public int Start(CancellationToken cancellationToken) => Schedules.Count(c => c.Start(cancellationToken)); + public int Start(CancellationToken cancellationToken) + => Schedules.Count(c => c.Start(cancellationToken)); - public int Restart() => Schedules.Count(c => c.Restart()); + public int Restart() + => Schedules.Count(c => c.Restart()); - public async Task Stop() => await Task.WhenAll(Schedules.Select(c => c.Stop())); + public async Task Stop() + => await Task.WhenAll(Schedules.Select(c => c.Stop())); public void Dispose() { diff --git a/src/Sa.Schedule/Settings/ScheduleBuilder.cs b/src/Sa.Schedule/Settings/ScheduleBuilder.cs index d7e29720..42a1f220 100644 --- a/src/Sa.Schedule/Settings/ScheduleBuilder.cs +++ b/src/Sa.Schedule/Settings/ScheduleBuilder.cs @@ -32,8 +32,9 @@ public ScheduleBuilder(IServiceCollection services) } - public IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Guid? jobId = null) - where T : class, IJob + public IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + Guid? jobId = null) + where T : class, IJob { Guid id = GetId(jobId); _services.TryAddKeyedScoped(id); @@ -44,8 +45,9 @@ public ScheduleBuilder(IServiceCollection services) return new JobBuilder(jobSettings); } - public IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Action configure, Guid? jobId = null) - where T : class, IJob + public IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + Action configure, Guid? jobId = null) + where T : class, IJob { Guid id = GetId(jobId); _services.TryAddKeyedScoped(id); @@ -88,8 +90,9 @@ public IScheduleBuilder UseHostedService() return this; } - public IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(object? key = null) - where T : class, IJobInterceptor + public IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + object? key = null) + where T : class, IJobInterceptor { _services.AddSingleton(new JobInterceptorSettings(typeof(T), key)); _services.TryAddKeyedScoped(key); diff --git a/src/Sa.Schedule/Settings/ScheduleSettings.cs b/src/Sa.Schedule/Settings/ScheduleSettings.cs index 24b08406..f7240292 100644 --- a/src/Sa.Schedule/Settings/ScheduleSettings.cs +++ b/src/Sa.Schedule/Settings/ScheduleSettings.cs @@ -8,11 +8,15 @@ internal sealed class ScheduleSettings : IScheduleSettings public Func? HandleError { get; private set; } - public IEnumerable GetJobSettings() => _storage.Values.Where(c => c.Properties.Disabled != true); + public IEnumerable GetJobSettings() + => _storage.Values.Where(c => c.Properties.Disabled != true); public void UseHostedService() => IsHostedService = true; - internal static ScheduleSettings Create(IEnumerable jobSettings, bool isHostedService, Func? handleError) + internal static ScheduleSettings Create( + IEnumerable jobSettings, + bool isHostedService, + Func? handleError) { IEnumerable items = jobSettings.GroupBy( c => (c.JobId, c.JobType) From b702009faeac141be4435e9204c88c9ea661a9f4 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 9 Apr 2026 19:31:47 +0300 Subject: [PATCH 15/28] pg hybrid --- src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs | 4 ++++ .../PostgresFileStorageTests.cs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs index 92b07661..afe57568 100644 --- a/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs +++ b/src/Sa.HybridFileStorage.Postgres/PostgresFileStorage.cs @@ -118,6 +118,10 @@ await partManager.EnsureParts( { ms = streamManager.GetStream(); await fileStream.CopyToAsync(ms, cancellationToken); + } + + if (fileStream.CanSeek) // fileStream is not MemoryStream ms) + { ms.Position = 0; } diff --git a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs index dc8a885e..0164ff4c 100644 --- a/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs +++ b/src/Tests/Sa.HybridFileStorage.PostgresTests/PostgresFileStorageTests.cs @@ -115,7 +115,7 @@ public async Task Upload2FileAsync() var t1 = Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); var t2 = Sub.UploadAsync(metadata, fileContent, fixture.CancellationToken); - await Task.WhenAll([t1, t2]); + await Task.WhenAll(t1, t2); var fileId1 = (await t1).FileId; var fileId2 = (await t2).FileId; From a1337b61375da4ab0c9419bd8aeb28a49ae86a58 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 10 Apr 2026 15:42:09 +0300 Subject: [PATCH 16/28] ~ sked --- src/Sa.Schedule/Engine/IJobController.cs | 3 - src/Sa.Schedule/Engine/IJobFactory.cs | 1 - src/Sa.Schedule/Engine/JobContext.cs | 15 ++- src/Sa.Schedule/Engine/JobController.cs | 79 ++++++------ src/Sa.Schedule/Engine/JobErrorHandler.cs | 8 +- src/Sa.Schedule/Engine/JobFactory.cs | 16 ++- src/Sa.Schedule/Engine/JobPipeline.cs | 2 +- src/Sa.Schedule/Engine/JobScheduler.cs | 119 +++++++++++------- src/Sa.Schedule/Engine/ScheduleHost.cs | 5 +- src/Sa.Schedule/Engine/Scheduler.cs | 33 +++-- src/Sa.Schedule/IJobBuilder.cs | 6 +- src/Sa.Schedule/IJobContext.cs | 6 +- src/Sa.Schedule/IJobScheduler.cs | 11 +- src/Sa.Schedule/IScheduler.cs | 4 +- src/Sa.Schedule/JobStatus.cs | 10 -- src/Sa.Schedule/Sa.Schedule.csproj | 37 +++--- src/Sa.Schedule/Settings/JobSettings.cs | 2 +- src/Sa.Schedule/Settings/ScheduleBuilder.cs | 10 +- src/Sa.Schedule/Settings/ScheduleSettings.cs | 33 +++-- src/Sa/Classes/WorkQueue.cs | 76 ++++++++--- src/Samples/Schedule.Console/Program.cs | 10 +- src/Tests/Fixtures/Sa.Fixture/SaFixture.cs | 3 +- .../OutboxParallelMessagingTests.cs | 2 +- .../OutboxTenantParallelismTests.cs | 2 +- .../Sa.Outbox.PostgreSqlTests/OutboxTests.cs | 2 +- .../OutboxTwoGroupsTests.cs | 2 +- .../PartitionAsJobTests.cs | 12 +- .../SchedulePostSetupTests.cs | 9 +- .../Sa.ScheduleTests/ScheduleSetupTests.cs | 5 +- src/Tests/SaTests/Classes/WorkQueueTests.cs | 76 ++++++++--- 30 files changed, 372 insertions(+), 227 deletions(-) delete mode 100644 src/Sa.Schedule/JobStatus.cs diff --git a/src/Sa.Schedule/Engine/IJobController.cs b/src/Sa.Schedule/Engine/IJobController.cs index 92d5f4e0..4934ea0b 100644 --- a/src/Sa.Schedule/Engine/IJobController.cs +++ b/src/Sa.Schedule/Engine/IJobController.cs @@ -13,9 +13,6 @@ internal enum CanJobExecuteResult /// internal interface IJobController { - // scope context - public IJobContext Context { get; } - // scope events ValueTask WaitToRun(CancellationToken cancellationToken); diff --git a/src/Sa.Schedule/Engine/IJobFactory.cs b/src/Sa.Schedule/Engine/IJobFactory.cs index 84d988c2..2ec3c723 100644 --- a/src/Sa.Schedule/Engine/IJobFactory.cs +++ b/src/Sa.Schedule/Engine/IJobFactory.cs @@ -2,6 +2,5 @@ internal interface IJobFactory { - IJobController CreateJobController(IJobSettings settings); IJobScheduler CreateJobSchedule(IJobSettings settings); } diff --git a/src/Sa.Schedule/Engine/JobContext.cs b/src/Sa.Schedule/Engine/JobContext.cs index 0a4fc4ee..e7b8d520 100644 --- a/src/Sa.Schedule/Engine/JobContext.cs +++ b/src/Sa.Schedule/Engine/JobContext.cs @@ -7,9 +7,8 @@ namespace Sa.Schedule.Engine; internal sealed class JobContext(IJobSettings settings) : IJobContext { - public string JobName => settings.Properties.JobName ?? $"{settings.JobId}"; - - public JobStatus Status { get; set; } + public string JobName => settings.Properties.JobName + ?? $"{settings.JobId}"; public IJobSettings Settings => settings; @@ -29,13 +28,14 @@ internal sealed class JobContext(IJobSettings settings) : IJobContext public ulong NumRuns { get; set; } - public IServiceProvider JobServices { get; set; } = NullJobServices.Instance; - - public Queue Stack { get; private set; } = new(); + public Queue Stack { get; private set; } = []; IEnumerable IJobContext.Stack => Stack.Reverse(); - public ILogger Logger => JobServices.GetService>() ?? NullLogger.Instance; + public IServiceProvider ServiceProvider { get; set; } = NullJobServices.Instance; + + public ILogger Logger => ServiceProvider.GetService>() + ?? NullLogger.Instance; public IJobContext Clone() { @@ -47,7 +47,6 @@ public IJobContext Clone() LastError = LastError, CreatedAt = CreatedAt, ExecuteAt = ExecuteAt, - Status = Status, NumRuns = NumRuns, Stack = new Queue(Stack.Select(x => x.Clone())), }; diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index f5c13dfa..944f004a 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -15,21 +15,16 @@ internal sealed partial class JobController( TimeProvider timeProvider) : IJobController { - private readonly JobContext context = new(settings); + private readonly JobContext _context = new(settings); - private JobPipeline? _job; - - public IJobContext Context => context; - - public DateTimeOffset UtcNow => timeProvider.GetUtcNow(); + private volatile JobPipeline? _job; public async ValueTask WaitToRun(CancellationToken cancellationToken) { - if (context.NumRuns == 0 + if (_context.NumRuns == 0 && settings.Properties.InitialDelay.HasValue && settings.Properties.InitialDelay.Value != TimeSpan.Zero) { - context.Status = JobStatus.WaitingToRun; await Task.Delay(settings.Properties.InitialDelay.Value, cancellationToken); } } @@ -38,41 +33,33 @@ public void Running() { _job = new JobPipeline(settings, interceptorSettings, scopeFactory); - context.JobServices = _job.JobServices; - context.Status = JobStatus.Running; + _context.ServiceProvider = _job.ServiceProvider; - if (context.NumRuns == 0) context.CreatedAt = UtcNow; - context.NumRuns++; + if (_context.NumRuns == 0) _context.CreatedAt = timeProvider.GetUtcNow(); + _context.NumRuns++; } public void Stopped(TaskStatus status) { - switch (status) - { - case TaskStatus.Faulted: context.Status = JobStatus.Failed; break; - case TaskStatus.Canceled: context.Status = JobStatus.Cancelled; break; - case TaskStatus.RanToCompletion: context.Status = JobStatus.Completed; break; - } - - context.JobServices = NullJobServices.Instance; + _context.ServiceProvider = NullJobServices.Instance; _job?.Dispose(); } public async ValueTask CanExecute(CancellationToken cancellationToken) { - if (settings.Properties.IsRunOnce == true && context.NumIterations > 0) + if (settings.Properties.IsRunOnce == true && _context.NumIterations > 0) return CanJobExecuteResult.Abort; - if (context.NumIterations == 0 && settings.Properties.Immediate == true) + if (_context.NumIterations == 0 && settings.Properties.Immediate == true) return CanJobExecuteResult.Ok; IJobTiming? timing = settings.Properties.Timing; if (timing != null) { - DateTimeOffset now = UtcNow; + DateTimeOffset now = timeProvider.GetUtcNow(); - DateTimeOffset? next = timing.GetNextOccurrence(now, context); + DateTimeOffset? next = timing.GetNextOccurrence(now, _context); if (!next.HasValue) return CanJobExecuteResult.Abort; @@ -89,8 +76,8 @@ public async ValueTask CanExecute(CancellationToken cancell if (stackSize > 0) { - if (context.Stack.Count == stackSize) context.Stack.Dequeue(); - context.Stack.Enqueue(context.Clone()); + if (_context.Stack.Count == stackSize) _context.Stack.Dequeue(); + _context.Stack.Enqueue(_context.Clone()); } return !cancellationToken.IsCancellationRequested @@ -100,45 +87,50 @@ public async ValueTask CanExecute(CancellationToken cancell public Task Execute(CancellationToken cancellationToken) { - context.NumIterations++; - context.ExecuteAt = UtcNow; - return _job!.Execute(Context, cancellationToken); + _context.NumIterations++; + _context.ExecuteAt = timeProvider.GetUtcNow(); + return _job!.Execute(_context, cancellationToken); } public void ExecutionCompleted() { - context.CompetedIterations++; - context.FailedRetries = 0; + _context.CompetedIterations++; + _context.FailedRetries = 0; } public void ExecutionFailed(Exception exception) { - JobException error = new(context, exception); - context.FailedIterations++; - context.LastError = error; + JobException error = new(_context, exception); + _context.FailedIterations++; + _context.LastError = error; IJobErrorHandling errorHandling = settings.ErrorHandling; - if (errorHandling.HasSuppressError && errorHandling.SuppressError?.Invoke(exception) == true) + if (errorHandling.HasSuppressError + && errorHandling.SuppressError?.Invoke(exception) == true) { - LogJobWasSuppressed(context.Logger, context.JobName, exception.GetType().Name, exception.Message); + LogJobWasSuppressed( + _context.Logger, + _context.JobName, + exception.GetType().Name, + exception.Message); return; } - if (context.FailedRetries < settings.ErrorHandling.RetryCount) + if (_context.FailedRetries < settings.ErrorHandling.RetryCount) { - context.FailedRetries++; + _context.FailedRetries++; LogFailedRetryAttempts( - context.Logger, - context.JobName, - context.FailedRetries, + _context.Logger, + _context.JobName, + _context.FailedRetries, errorHandling.RetryCount, exception.GetType().Name, exception.Message); return; } - context.JobServices.GetService()?.HandleError(Context, error); + _context.ServiceProvider.GetService()?.HandleError(_context, error); } @@ -152,5 +144,6 @@ public void ExecutionFailed(Exception exception) EventId = 402, Level = LogLevel.Warning, Message = "[{JobName}] {FailedRetryAttempts} out of {RetryCount} reps when the job failed due to an error: {Type} “{Error}”")] - static partial void LogFailedRetryAttempts(ILogger logger, string jobName, int failedRetryAttempts, int retryCount, string type, string error); + static partial void LogFailedRetryAttempts( + ILogger logger, string jobName, int failedRetryAttempts, int retryCount, string type, string error); } diff --git a/src/Sa.Schedule/Engine/JobErrorHandler.cs b/src/Sa.Schedule/Engine/JobErrorHandler.cs index f8739d80..84576e5c 100644 --- a/src/Sa.Schedule/Engine/JobErrorHandler.cs +++ b/src/Sa.Schedule/Engine/JobErrorHandler.cs @@ -29,11 +29,15 @@ private void DoHandleError(IJobContext context, Exception exception) break; case ErrorHandlingAction.CloseApplication: - CloseApplication(context.JobName, context.JobServices.GetRequiredService(), exception); + CloseApplication( + context.JobName, + context.ServiceProvider.GetRequiredService(), exception); break; case ErrorHandlingAction.StopAllJobs: - StopAllJobs(context.JobName, context.JobServices.GetRequiredService(), exception); + StopAllJobs( + context.JobName, + context.ServiceProvider.GetRequiredService(), exception); break; default: diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index 25f8bb64..a31aa2a2 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -7,11 +7,17 @@ internal sealed class JobFactory( IServiceScopeFactory scopeFactory, IInterceptorSettings interceptorSettings, IJobRunner jobRunner, - TimeProvider timeProvider) : IJobFactory + TimeProvider? timeProvider = null) : IJobFactory { - public IJobController CreateJobController(IJobSettings settings) - => new JobController(settings, interceptorSettings, scopeFactory, timeProvider); - public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(jobRunner, CreateJobController(settings)); + => new JobScheduler(jobRunner, () => CreateController(settings)); + + private JobController CreateController(IJobSettings settings) + { + return new( + settings, + interceptorSettings, + scopeFactory, + timeProvider ?? TimeProvider.System); + } } diff --git a/src/Sa.Schedule/Engine/JobPipeline.cs b/src/Sa.Schedule/Engine/JobPipeline.cs index 1ff5ac8b..657fe040 100644 --- a/src/Sa.Schedule/Engine/JobPipeline.cs +++ b/src/Sa.Schedule/Engine/JobPipeline.cs @@ -45,7 +45,7 @@ public JobPipeline( } } - public IServiceProvider JobServices => _scope.ServiceProvider; + public IServiceProvider ServiceProvider => _scope.ServiceProvider; public void Dispose() => _scope.Dispose(); diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 63a3461f..62b58068 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -1,33 +1,44 @@ using Microsoft.Extensions.Primitives; +using Sa.Classes; namespace Sa.Schedule.Engine; -internal sealed class JobScheduler(IJobRunner runner, IJobController controller) - : IJobScheduler, IDisposable, IAsyncDisposable +internal sealed class JobScheduler( + IJobRunner runner, + Func createController) : IJobScheduler { private readonly static IChangeToken NoneChangeToken = new CancellationChangeToken(CancellationToken.None); - private readonly Lock _locked = new(); - - private TaskCompletionSource? _stoppingTask; - + private readonly Lock _lock = new(); private CancellationTokenSource _stoppingToken = new(); private CancellationToken _originalToken; + private bool _disposed; - public IJobContext Context => controller.Context; + private readonly WorkQueue _jobs = new( + WorkQueueOptions.Create( + (controller, ct) => runner.Run(controller, ct)) + .WithQueueCapacity(40) + .WithConcurrencyLimit(1)); - public bool IsActive => _stoppingTask?.Task.Status - == TaskStatus.WaitingForActivation; + public bool IsActive + { + get + { + lock (_lock) + { + return !_disposed && _jobs.ActiveTasks > 0; + } + } + } public IChangeToken GetActiveChangeToken() { - if (_disposed) return NoneChangeToken; - - lock (_locked) + lock (_lock) { + if (_disposed) return NoneChangeToken; return new CancellationChangeToken(_stoppingToken.Token); } } @@ -35,68 +46,90 @@ public IChangeToken GetActiveChangeToken() /// /// Start all jobs /// - public bool Start(CancellationToken cancellationToken) + public async Task Start(CancellationToken cancellationToken) { if (IsActive) return false; - lock (_locked) + CancellationTokenSource tokenToStope; + + lock (_lock) { - _stoppingToken.Cancel(); - _stoppingToken.Dispose(); + if (IsActive) return false; + + tokenToStope = _stoppingToken; _originalToken = cancellationToken; _stoppingToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + } - _stoppingTask = new TaskCompletionSource(); + await tokenToStope.CancelAsync(); + tokenToStope.Dispose(); - _ = runner - .Run(controller, _stoppingToken.Token) - .ContinueWith(Done, CancellationToken.None); + await ExecuteJobAsync(_stoppingToken.Token); + return true; + } - return true; + private async Task ExecuteJobAsync(CancellationToken ct) + { + try + { + await _jobs.WaitForIdleAsync(_originalToken); + await _jobs.Enqueue(createController(), ct); + } + catch + { + // ignore } } - public bool Restart() => Start(_originalToken); + public async Task Restart() + { + await Stop(); + return await Start(_originalToken); + } - public Task Stop() + public async Task Stop() { - if (IsActive) + CancellationTokenSource? tokenToCancel; + + lock (_lock) { - lock (_locked) - { - _stoppingToken?.Cancel(); - } + if (_disposed) return; + tokenToCancel = _stoppingToken; } - return _stoppingTask?.Task ?? Task.CompletedTask; - } - private void Done(Task task) - { - lock (_locked) + // Отмена вне блокировки для минимизации времени удержания лога + if (tokenToCancel != null) { - _stoppingTask?.TrySetResult(); + await tokenToCancel.CancelAsync(); } + + await _jobs.WaitForIdleAsync(CancellationToken.None); } + public void Dispose() { - if (!_disposed) + CancellationTokenSource? tokenToDispose; + + lock (_lock) { + if (_disposed) return; _disposed = true; - _ = Stop(); - _stoppingToken.Dispose(); + tokenToDispose = _stoppingToken; + _stoppingToken.Cancel(); } + + _jobs.Dispose(); + tokenToDispose?.Dispose(); } public async ValueTask DisposeAsync() { - if (!_disposed) - { - _disposed = true; - await Stop(); - _stoppingToken.Dispose(); - } + // Корректное ожидание завершения фоновых задач + await Stop(); + await _jobs.DisposeAsync(); + _stoppingToken?.Dispose(); } } diff --git a/src/Sa.Schedule/Engine/ScheduleHost.cs b/src/Sa.Schedule/Engine/ScheduleHost.cs index 0920b6c0..d3c73089 100644 --- a/src/Sa.Schedule/Engine/ScheduleHost.cs +++ b/src/Sa.Schedule/Engine/ScheduleHost.cs @@ -4,10 +4,9 @@ namespace Sa.Schedule.Engine; internal sealed class ScheduleHost(IScheduler controller) : IHostedService { - public Task StartAsync(CancellationToken cancellationToken) + public async Task StartAsync(CancellationToken cancellationToken) { - controller.Start(cancellationToken); - return Task.CompletedTask; + await controller.Start(cancellationToken); } public async Task StopAsync(CancellationToken cancellationToken) diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index 6acf8676..e987cce0 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -14,23 +14,42 @@ internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) /// /// Start all jobs /// - public int Start(CancellationToken cancellationToken) - => Schedules.Count(c => c.Start(cancellationToken)); + public async Task Start(CancellationToken cancellationToken) + { + var results = await Task.WhenAll(Schedules.Select(c => c.Start(cancellationToken))); + return results.Count(r => r); + } - public int Restart() - => Schedules.Count(c => c.Restart()); + public async Task Restart() + { + var results = await Task.WhenAll(Schedules.Select(c => c.Restart())); + return results.Count(r => r); + } public async Task Stop() - => await Task.WhenAll(Schedules.Select(c => c.Stop())); + { + await Task.WhenAll(Schedules.Select(c => c.Stop())); + } public void Dispose() { if (!_disposed) { _disposed = true; - _ = Stop(); + + foreach (var job in Schedules) + { + job.Dispose(); + } } } - public async ValueTask DisposeAsync() => await Stop(); + public async ValueTask DisposeAsync() + { + await Stop(); + foreach (var job in Schedules) + { + await job.DisposeAsync(); + } + } } diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index a00680d7..200ed236 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -65,14 +65,16 @@ public interface IJobBuilder /// /// The number of seconds (default is 1). /// The current builder instance. - IJobBuilder EverySeconds(int seconds = 1) => EveryTime(TimeSpan.FromSeconds(seconds), $"every {seconds} seconds"); + IJobBuilder EverySeconds(int seconds = 1) + => EveryTime(TimeSpan.FromSeconds(seconds), $"every {seconds} seconds"); /// /// Configures the job to run every specified number of minutes. /// /// The number of minutes (default is 1). /// The current builder instance. - IJobBuilder EveryMinutes(int minutes = 1) => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); + IJobBuilder EveryMinutes(int minutes = 1) + => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); /// /// Merges the specified job properties into the current job configuration. diff --git a/src/Sa.Schedule/IJobContext.cs b/src/Sa.Schedule/IJobContext.cs index 11b0a919..52fa5bec 100644 --- a/src/Sa.Schedule/IJobContext.cs +++ b/src/Sa.Schedule/IJobContext.cs @@ -12,9 +12,8 @@ namespace Sa.Schedule; /// public interface IJobContext { - Guid JobId => Settings.JobId; string JobName { get; } - JobStatus Status { get; } + IJobSettings Settings { get; } ulong NumIterations { get; } ulong FailedIterations { get; } @@ -23,10 +22,9 @@ public interface IJobContext DateTimeOffset CreatedAt { get; } DateTimeOffset? ExecuteAt { get; } JobException? LastError { get; } - IServiceProvider JobServices { get; } + IServiceProvider ServiceProvider { get; } IEnumerable Stack { get; } ILogger Logger { get; } IJobContext Clone(); - bool Active => Status == JobStatus.Running || Status == JobStatus.WaitingToRun; } diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index b85a3d26..0f06e53a 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -5,18 +5,13 @@ namespace Sa.Schedule; /// /// This individual task scheduler is responsible for managing specific tasks. /// -public interface IJobScheduler +public interface IJobScheduler: IDisposable, IAsyncDisposable { /// /// Gets a value indicating whether the job scheduler is currently active. /// bool IsActive { get; } - /// - /// Gets the context associated with the job scheduler. - /// - IJobContext Context { get; } - /// /// Gets a change token that can be used to track changes to the active state of the scheduler. /// @@ -25,12 +20,12 @@ public interface IJobScheduler /// /// Starts the job scheduler asynchronously. /// - bool Start(CancellationToken cancellationToken); + Task Start(CancellationToken cancellationToken); /// /// Restarts the job scheduler asynchronously. /// - bool Restart(); + Task Restart(); /// /// Stops the job scheduler asynchronously. diff --git a/src/Sa.Schedule/IScheduler.cs b/src/Sa.Schedule/IScheduler.cs index fb3cb5ee..2ce46be9 100644 --- a/src/Sa.Schedule/IScheduler.cs +++ b/src/Sa.Schedule/IScheduler.cs @@ -20,13 +20,13 @@ public interface IScheduler /// /// The cancellation token. /// The number of jobs started. - int Start(CancellationToken cancellationToken); + Task Start(CancellationToken cancellationToken); /// /// Restarts the scheduler. /// /// The number of jobs restarted. - int Restart(); + Task Restart(); /// /// Stops the scheduler. diff --git a/src/Sa.Schedule/JobStatus.cs b/src/Sa.Schedule/JobStatus.cs deleted file mode 100644 index bb67c901..00000000 --- a/src/Sa.Schedule/JobStatus.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Sa.Schedule; - -public enum JobStatus -{ - WaitingToRun = 2, - Running = 3, - Completed = 5, - Cancelled = 6, - Failed = 7 -} diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index fbc50cd7..5a9120d1 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -1,27 +1,28 @@  - + - - 0.8.0 - Execute jobs on a schedule - + + 0.9.0 + Execute jobs on a schedule + - - 1701;1702;CS8602; - + + 1701;1702;CS8602; + - - 1701;1702;CS8602; - + + 1701;1702;CS8602; + - - - + + + + - - - - + + + + diff --git a/src/Sa.Schedule/Settings/JobSettings.cs b/src/Sa.Schedule/Settings/JobSettings.cs index 86c1f09d..15f52876 100644 --- a/src/Sa.Schedule/Settings/JobSettings.cs +++ b/src/Sa.Schedule/Settings/JobSettings.cs @@ -5,7 +5,7 @@ internal sealed class JobSettings(Type jobType, Guid jobId) : IJobSettings /// /// handler id /// - public Guid JobId { get; } = jobId; + public Guid JobId => jobId; public Type JobType => jobType; diff --git a/src/Sa.Schedule/Settings/ScheduleBuilder.cs b/src/Sa.Schedule/Settings/ScheduleBuilder.cs index 42a1f220..3954aeaa 100644 --- a/src/Sa.Schedule/Settings/ScheduleBuilder.cs +++ b/src/Sa.Schedule/Settings/ScheduleBuilder.cs @@ -20,7 +20,12 @@ public ScheduleBuilder(IServiceCollection services) _services.TryAddSingleton(sp => { IEnumerable jobSettings = sp.GetServices(); - ScheduleSettings settings = ScheduleSettings.Create(jobSettings, _isHostedService, _handleError); + + ScheduleSettings settings = ScheduleSettings.Create( + jobSettings, + _isHostedService, + _handleError); + return settings; }); @@ -46,7 +51,8 @@ public ScheduleBuilder(IServiceCollection services) } public IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( - Action configure, Guid? jobId = null) + Action configure, + Guid? jobId = null) where T : class, IJob { Guid id = GetId(jobId); diff --git a/src/Sa.Schedule/Settings/ScheduleSettings.cs b/src/Sa.Schedule/Settings/ScheduleSettings.cs index f7240292..9f5cbd9f 100644 --- a/src/Sa.Schedule/Settings/ScheduleSettings.cs +++ b/src/Sa.Schedule/Settings/ScheduleSettings.cs @@ -2,17 +2,25 @@ internal sealed class ScheduleSettings : IScheduleSettings { - private Dictionary _storage = []; + private readonly IReadOnlyDictionary _storage; - public bool IsHostedService { get; private set; } + private ScheduleSettings( + IReadOnlyDictionary storage, + Func? handleError, + bool isHostedService) + { + _storage = storage; + IsHostedService = isHostedService; + HandleError = handleError; + } + + public bool IsHostedService { get; } - public Func? HandleError { get; private set; } + public Func? HandleError { get; } public IEnumerable GetJobSettings() => _storage.Values.Where(c => c.Properties.Disabled != true); - public void UseHostedService() => IsHostedService = true; - internal static ScheduleSettings Create( IEnumerable jobSettings, bool isHostedService, @@ -20,14 +28,15 @@ internal static ScheduleSettings Create( { IEnumerable items = jobSettings.GroupBy( c => (c.JobId, c.JobType) - , (k, items) => items.Aggregate(seed: new JobSettings(k.JobType, k.JobId), (s1, s2) => s1.Merge(s2)) + , (k, items) => items.Aggregate( + seed: new JobSettings(k.JobType, k.JobId), + (s1, s2) => s1.Merge(s2)) ); - return new ScheduleSettings - { - HandleError = handleError, - IsHostedService = isHostedService, - _storage = items.ToDictionary(c => c.JobId) - }; + return new ScheduleSettings( + storage: items.ToDictionary(c => c.JobId), + handleError: handleError, + isHostedService: isHostedService + ); } } diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 22d8a119..97bbd705 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -107,23 +107,16 @@ internal sealed class WorkQueue : IWorkQueue private long _taskIdCounter = 0; private bool _disposed = false; - public WorkQueue( - IWork processor, - IWorkObserver? observer = null, - TimeProvider? timeProvider = null, - int? maxQueueCapacity = null, - int? initialConcurrencyLimit = null, - Action? statusChanged = null, - ILogger>? logger = null) + public WorkQueue(WorkQueueOptions options) { - _processor = processor; - _watcher = observer ?? processor as IWorkObserver; - _timeProvider = timeProvider ?? TimeProvider.System; - _maxConcurrency = initialConcurrencyLimit ?? Environment.ProcessorCount; - _statusChanged = statusChanged; - _logger = logger; - - _queue = Channel.CreateBounded(new BoundedChannelOptions(maxQueueCapacity ?? 1000) + _processor = options.Processor; + _watcher = options.Observer ?? _processor as IWorkObserver; + _timeProvider = options.TimeProvider ?? TimeProvider.System; + _maxConcurrency = options.ConcurrencyLimit ?? Environment.ProcessorCount; + _statusChanged = options.StatusChanged; + _logger = options.Logger; + + _queue = Channel.CreateBounded(new BoundedChannelOptions(options.MaxQueueCapacity) { AllowSynchronousContinuations = true, SingleReader = false, @@ -462,3 +455,54 @@ private sealed class WorkItem(TModel item, CancellationToken token) public WorkInfo ToInfo() => new(Id, Status, EnqueuedTime, StartedTime, EndedTime, LastError); } } + + +internal sealed record WorkQueueOptions +( + // Required + IWork Processor, + // Optional with defaults + IWorkObserver? Observer = null, + TimeProvider? TimeProvider = null, + int MaxQueueCapacity = 20, + int? ConcurrencyLimit = null, + Action? StatusChanged = null, + ILogger? Logger = null +) +{ + public WorkQueueOptions WithObserver(IWorkObserver observer) => + this with { Observer = observer }; + + public WorkQueueOptions WithTimeProvider(TimeProvider timeProvider) => + this with { TimeProvider = timeProvider }; + + public WorkQueueOptions WithQueueCapacity(int capacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + return this with { MaxQueueCapacity = capacity }; + } + + public WorkQueueOptions WithConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + return this with { ConcurrencyLimit = limit }; + } + + public WorkQueueOptions WithStatusCallback(Action callback) => + this with { StatusChanged = callback }; + + public WorkQueueOptions WithLogger(ILogger> logger) => + this with { Logger = logger }; + + public static WorkQueueOptions Create(IWork processor) => + new(processor); + + public static WorkQueueOptions Create(Func process) => + new(new WrapProcessor(process)); + + sealed class WrapProcessor(Func process) : IWork + { + public Task Execute(TModel model, CancellationToken cancellationToken) + => process(model, cancellationToken); + } +} diff --git a/src/Samples/Schedule.Console/Program.cs b/src/Samples/Schedule.Console/Program.cs index 0aed5e0a..7d7a89d5 100644 --- a/src/Samples/Schedule.Console/Program.cs +++ b/src/Samples/Schedule.Console/Program.cs @@ -46,7 +46,7 @@ } else { - controller.Start(cts.Token); + await controller.Start(cts.Token); } _ = Task.Run(async () => @@ -55,7 +55,7 @@ await controller.Stop(); Console.WriteLine($"*** stopped & restart after 2 sec"); await Task.Delay(2000); - controller.Restart(); + await controller.Restart(); }); _ = Task.Run(async () => @@ -93,7 +93,11 @@ public async Task Execute(IJobContext context, CancellationToken cancellationTok public class SomeInterceptor : IJobInterceptor { - public async Task OnHandle(IJobContext context, Func next, object? key, CancellationToken cancellationToken) + public async Task OnHandle( + IJobContext context, + Func next, + object? key, + CancellationToken cancellationToken) { System.Console.WriteLine($""); await next(); diff --git a/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs b/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs index 42267c45..281c7b20 100644 --- a/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs +++ b/src/Tests/Fixtures/Sa.Fixture/SaFixture.cs @@ -43,7 +43,8 @@ protected SaFixture() public async virtual ValueTask DisposeAsync() { - if (_serviceProvider.IsValueCreated) await _serviceProvider.Value.DisposeAsync(); + if (_serviceProvider.IsValueCreated) + await _serviceProvider.Value.DisposeAsync(); } } diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs index de33a9aa..580a1399 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxParallelMessagingTests.cs @@ -130,7 +130,7 @@ public async Task ParallelMessaging_MustBeProcessed() // start cron schedules IScheduler scheduler = ServiceProvider.GetRequiredService(); - int i = scheduler.Start(CancellationToken.None); + int i = await scheduler.Start(CancellationToken.None); Assert.True(i > 0); // start delivery message diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs index 9678be1b..166d06b9 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTenantParallelismTests.cs @@ -142,7 +142,7 @@ public async Task Outbox_TenantParallelism_ShouldProcessTenantsConcurrently() ParallelTestConsumer.Clear(); var scheduler = ServiceProvider.GetRequiredService(); - int startedSchedules = scheduler.Start(CancellationToken.None); + int startedSchedules = await scheduler.Start(CancellationToken.None); Assert.True(startedSchedules >= 1); var publisher = ServiceProvider.GetRequiredService(); diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs index 6639554f..b854e8d0 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTests.cs @@ -79,7 +79,7 @@ public async Task OutBoxTest() // start cron schedules var scheduler = ServiceProvider.GetRequiredService(); - int i = scheduler.Start(CancellationToken.None); + int i = await scheduler.Start(CancellationToken.None); Assert.True(i > 0); // start delivery message diff --git a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs index 17f441cb..d3d2d8ea 100644 --- a/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs +++ b/src/Tests/Sa.Outbox.PostgreSqlTests/OutboxTwoGroupsTests.cs @@ -99,7 +99,7 @@ public async Task OutBox_TwoGroups_ShouldProcessSeparately() SomeMessageConsumerGr2.Counter = 0; var scheduler = ServiceProvider.GetRequiredService(); - int startedSchedules = scheduler.Start(CancellationToken.None); + int startedSchedules = await scheduler.Start(CancellationToken.None); Assert.True(startedSchedules >= 2, "Ожидалось как минимум 2 расписания (по одному на группу)"); var publisher = ServiceProvider.GetRequiredService(); diff --git a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs index 7e4651c6..415627d1 100644 --- a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs +++ b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs @@ -6,7 +6,8 @@ namespace Sa.Partitional.PostgreSqlTests; -public class PartitionAsJobTests(PartitionAsJobTests.Fixture fixture) : IClassFixture +public class PartitionAsJobTests(PartitionAsJobTests.Fixture fixture) + : IClassFixture { public class Fixture : PgDataSourceFixture @@ -53,12 +54,17 @@ public async Task MigrateAsJobTest() { Console.WriteLine(fixture.ConnectionString); - int i = fixture.ServiceProvider.GetRequiredService().Start(CancellationToken.None); + int i = await fixture.ServiceProvider + .GetRequiredService().Start(CancellationToken.None); Assert.True(i > 0); await Task.Delay(800, TestContext.Current.CancellationToken); - var list = await Sub.GetPartsFromDate("customer", StartOfDay(DateTimeOffset.Now), TestContext.Current.CancellationToken); + var list = await Sub.GetPartsFromDate( + "customer", + StartOfDay(DateTimeOffset.Now), + TestContext.Current.CancellationToken); + Assert.NotEmpty(list); } } diff --git a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs index 2fe992cf..d9b5fe91 100644 --- a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs @@ -4,7 +4,8 @@ namespace Sa.ScheduleTests; -public class SchedulePostSetupTests(SchedulePostSetupTests.Fixture fixture) : IClassFixture +public class SchedulePostSetupTests(SchedulePostSetupTests.Fixture fixture) + : IClassFixture { public class Fixture : SaFixture { @@ -28,7 +29,7 @@ public Fixture() { Services.AddSaSchedule(b => { - b.AddJob((sp, builder) => + b.AddJob((_, builder) => { builder .EveryTime(TimeSpan.FromMilliseconds(100)) @@ -41,7 +42,7 @@ public Fixture() Services.AddSaSchedule(b => { - b.AddJob((sp, builder) => + b.AddJob((_, builder) => { builder .EveryTime(TimeSpan.FromMilliseconds(100)) @@ -62,7 +63,7 @@ public Fixture() [Fact] public async Task Check_Executing_RunOnce_ForMultiJobs() { - int i = Sub.Start(CancellationToken.None); + int i = await Sub.Start(CancellationToken.None); Assert.Equal(2, i); diff --git a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs index e1c19114..544d9be1 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleSetupTests.cs @@ -4,7 +4,8 @@ namespace Sa.ScheduleTests; -public class ScheduleSetupTests(ScheduleSetupTests.Fixture fixture) : IClassFixture +public sealed class ScheduleSetupTests(ScheduleSetupTests.Fixture fixture) + : IClassFixture { public class Fixture : SaFixture { @@ -45,7 +46,7 @@ public Fixture() [Fact] public async Task Check_ExecuteCounterJob() { - int i = Sub.Start(CancellationToken.None); + int i = await Sub.Start(CancellationToken.None); Assert.NotEqual(0, i); diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index 49b265a7..a72aad3d 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -59,8 +59,8 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() { // Arrange var model = new TestModel(); - var processor = new TestWork(); - using var queue = new WorkQueue(processor); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(50)); + using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); // Act await queue.Enqueue(model, cancellationToken: TestToken); @@ -72,12 +72,34 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() Assert.Equal(0, queue.QueuedTasks); } + + [Fact] + public async Task Enqueue_WithWaitForIdle_Completed() + { + // Arrange + var model = new TestModel(); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(10)); + using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); + + await queue.WaitForIdleAsync(cancellationToken: TestToken); + // Act + await queue.Enqueue(model, cancellationToken: TestToken); + await queue.WaitForIdleAsync(cancellationToken: TestToken); + + await queue.WaitForIdleAsync(cancellationToken: TestToken); + + // Assert + Assert.True(model.WasProcessed); + Assert.Equal(0, queue.ActiveTasks); + Assert.Equal(0, queue.QueuedTasks); + } + [Fact] public async Task Enqueue_MultipleTasks_AllExecuted() { // Arrange - var processor = new TestWork(); - using var queue = new WorkQueue(processor); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(30)); + using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); var models = new List { new(), new(), new() @@ -103,7 +125,9 @@ public async Task ConcurrencyLimit_Respected() var processor = new TestWorkWithDelay(delay); var concurrencyLimit = 3; - using var queue = new WorkQueue(processor, initialConcurrencyLimit: concurrencyLimit); + using var queue = new WorkQueue(WorkQueueOptions + .Create(processor) + .WithConcurrencyLimit(concurrencyLimit)); var models = new List { new(), new(), new(), new(), new(), new(), new() }; @@ -129,7 +153,9 @@ public async Task FaultedTask_StatusIsFaulted() // Arrange var processor = new TestWorkThatThrows(); var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + using var queue = new WorkQueue(WorkQueueOptions + .Create(processor) + .WithObserver(observer)); var model = new TestModel(); @@ -152,7 +178,9 @@ public async Task CancelledTask_StatusIsCancelled() using var cts = new CancellationTokenSource(); var processor = new TestWorkWithDelay(TimeSpan.FromSeconds(5)); var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + using var queue = new WorkQueue(WorkQueueOptions + .Create(processor) + .WithObserver(observer)); var model = new TestModel(); @@ -175,7 +203,9 @@ public async Task ShutdownAsync_StopsProcessing() // Arrange var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(500)); var observer = new TestObserver(); - using var queue = new WorkQueue(processor, observer); + using var queue = new WorkQueue(WorkQueueOptions + .Create(processor) + .WithObserver(observer)); for (int i = 0; i < 5; i++) await queue.Enqueue(new TestModel(), cancellationToken: TestToken); @@ -208,13 +238,15 @@ public async Task StatusChanged_Event_FiresOnStateChange() var eventTcs = new TaskCompletionSource(); using var queue = new WorkQueue( - processor, - statusChanged: (info) => - { - lastInfo = info; - if (info.Status == WorkStatus.Completed) - eventTcs.TrySetResult(info); - }); + WorkQueueOptions + .Create(processor) + .WithStatusCallback( + (info) => + { + lastInfo = info; + if (info.Status == WorkStatus.Completed) + eventTcs.TrySetResult(info); + })); var model = new TestModel(); @@ -232,7 +264,10 @@ public async Task Observer_InvokedOnStateChange() // Arrange var observer = new TestObserver(); var processor = new TestWork(); - using var queue = new WorkQueue(processor, observer); + using var queue = new WorkQueue( + WorkQueueOptions + .Create(processor) + .WithObserver(observer)); var model = new TestModel(); @@ -252,7 +287,8 @@ public async Task DisposeAsync_CleansUpResources() { // Arrange var processor = new TestWork(); - var queue = new WorkQueue(processor); + var queue = new WorkQueue( + WorkQueueOptions.Create(processor)); await queue.Enqueue(new TestModel(), cancellationToken: TestToken); await queue.DisposeAsync(); @@ -267,7 +303,8 @@ await Assert.ThrowsAsync(() [Fact] public void ConcurrencyLimit_InvalidValue_Throws() { - var queue = new WorkQueue(new TestWork()); + var queue = new WorkQueue( + WorkQueueOptions.Create(new TestWork())); Assert.Throws(() => queue.ConcurrencyLimit = 0); Assert.Throws(() => queue.ConcurrencyLimit = -1); @@ -278,7 +315,8 @@ public void ConcurrencyLimit_InvalidValue_Throws() [Fact] public async Task Enqueue_Disposed_Throws() { - var queue = new WorkQueue(new TestWork()); + var queue = new WorkQueue( + WorkQueueOptions.Create(new TestWork())); await queue.DisposeAsync(); await Assert.ThrowsAsync(() From 79dddb8ecf4d5e131f39d873900da15374a5587f Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 10 Apr 2026 21:23:48 +0300 Subject: [PATCH 17/28] ~ work Queue --- src/Sa.Schedule/Engine/JobScheduler.cs | 49 ++++++++++++++++---------- src/Sa.Schedule/IScheduleBuilder.cs | 9 +++-- src/Sa/Classes/WorkQueue.cs | 40 +++++++++------------ 3 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 62b58068..86a624f3 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -11,7 +11,7 @@ private readonly static IChangeToken NoneChangeToken = new CancellationChangeToken(CancellationToken.None); private readonly Lock _lock = new(); - private CancellationTokenSource _stoppingToken = new(); + private CancellationTokenSource _stoppingTokenSource = new(); private CancellationToken _originalToken; @@ -39,7 +39,7 @@ public IChangeToken GetActiveChangeToken() lock (_lock) { if (_disposed) return NoneChangeToken; - return new CancellationChangeToken(_stoppingToken.Token); + return new CancellationChangeToken(_stoppingTokenSource.Token); } } @@ -51,21 +51,23 @@ public async Task Start(CancellationToken cancellationToken) if (IsActive) return false; CancellationTokenSource tokenToStope; + CancellationTokenSource newTokenSource; lock (_lock) { if (IsActive) return false; - tokenToStope = _stoppingToken; + tokenToStope = _stoppingTokenSource; _originalToken = cancellationToken; - _stoppingToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _stoppingTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + newTokenSource = _stoppingTokenSource; } await tokenToStope.CancelAsync(); tokenToStope.Dispose(); - await ExecuteJobAsync(_stoppingToken.Token); + await ExecuteJobAsync(newTokenSource.Token); return true; } @@ -90,46 +92,57 @@ public async Task Restart() public async Task Stop() { - CancellationTokenSource? tokenToCancel; + CancellationTokenSource? tokenSourceToCancel; lock (_lock) { if (_disposed) return; - tokenToCancel = _stoppingToken; + tokenSourceToCancel = _stoppingTokenSource; } - - // Отмена вне блокировки для минимизации времени удержания лога - if (tokenToCancel != null) + if (tokenSourceToCancel != null) { - await tokenToCancel.CancelAsync(); + await tokenSourceToCancel.CancelAsync(); } - await _jobs.WaitForIdleAsync(CancellationToken.None); + await _jobs.WaitForIdleAsync(_originalToken); } public void Dispose() { - CancellationTokenSource? tokenToDispose; + if (_disposed) + return; + + CancellationTokenSource tokenToDispose; lock (_lock) { if (_disposed) return; _disposed = true; - tokenToDispose = _stoppingToken; - _stoppingToken.Cancel(); + tokenToDispose = _stoppingTokenSource; + } + + try + { + tokenToDispose.Cancel(); + } + catch (ObjectDisposedException) + { + // ignore } _jobs.Dispose(); - tokenToDispose?.Dispose(); + tokenToDispose.Dispose(); } public async ValueTask DisposeAsync() { - // Корректное ожидание завершения фоновых задач + if (_disposed) + return; + await Stop(); await _jobs.DisposeAsync(); - _stoppingToken?.Dispose(); + _stoppingTokenSource.Dispose(); } } diff --git a/src/Sa.Schedule/IScheduleBuilder.cs b/src/Sa.Schedule/IScheduleBuilder.cs index 16beded6..f71e5728 100644 --- a/src/Sa.Schedule/IScheduleBuilder.cs +++ b/src/Sa.Schedule/IScheduleBuilder.cs @@ -10,7 +10,8 @@ public interface IScheduleBuilder /// The type of job to add. /// The ID of the job. If not specified, a new ID will be generated. /// A builder for the added job. - IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Guid? jobId = null) where T : class, IJob; + IJobBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (Guid? jobId = null) where T : class, IJob; /// /// Adds a job with the specified action to the schedule. @@ -27,7 +28,8 @@ public interface IScheduleBuilder /// An action to configure the job. /// The ID of the job. If not specified, a new ID will be generated. /// The schedule builder. - IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(Action configure, Guid? jobId = null) where T : class, IJob; + IScheduleBuilder AddJob<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (Action configure, Guid? jobId = null) where T : class, IJob; /// /// Adds an interceptor of type to the schedule. @@ -35,7 +37,8 @@ public interface IScheduleBuilder /// The type of interceptor to add. /// The key to use for the interceptor. If not specified, a default key will be used. /// The schedule builder. - IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>(object? key = null) where T : class, IJobInterceptor; + IScheduleBuilder AddInterceptor<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T> + (object? key = null) where T : class, IJobInterceptor; /// /// Configures the schedule to use a hosted service. diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 97bbd705..ddff59d4 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -59,6 +59,11 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable /// int ActiveTasks { get; } + /// + /// Gets the current number of items available from this channel reader. + /// + int QueuedTasks { get; } + /// /// Определяет, свободна ли очередь (нет активных и нет в очереди). /// @@ -100,7 +105,7 @@ internal sealed class WorkQueue : IWorkQueue private readonly Action? _statusChanged; private volatile TaskCompletionSource _idleTcs = new(); - private volatile int _activeCount = 0; + private int _activeCount = 0; private volatile bool _isEnabled = true; private int _maxConcurrency; @@ -187,8 +192,11 @@ public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken public async Task ShutdownAsync() { - if (!_isEnabled || _disposed) return; - _isEnabled = false; + lock (_rootSync) + { + if (!_isEnabled || _disposed) return; + _isEnabled = false; + } await _shutdownCts.CancelAsync(); @@ -209,11 +217,7 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default while (!IsIdle()) { var currentTcs = _idleTcs; - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - - // Ждем сигнал от TCS или отмену. - // Если ActivateItem вызовется, currentTcs будет завершен -> цикл повторится. - await Task.WhenAny(currentTcs.Task, Task.Delay(Timeout.Infinite, cts.Token)); + await currentTcs.Task.WaitAsync(cancellationToken); } } @@ -334,15 +338,18 @@ private async Task ExecuteAsync(WorkItem item) private void ActivateItem(WorkItem item) { + TaskCompletionSource oldTcs; + lock (_rootSync) { _activeItemIds.Add(item.Id); _activeCount = _activeItemIds.Count; - var oldTcs = _idleTcs; + oldTcs = _idleTcs; _idleTcs = new TaskCompletionSource(); - oldTcs.TrySetResult(); // Пробуждаем ожидающие потоки } + + oldTcs.TrySetResult(); } private void DeactivateItem(WorkItem item) @@ -403,19 +410,6 @@ public void Dispose() _queue.Writer.TryComplete(); _shutdownCts.Cancel(); - try - { - _processingTask.Wait(TimeSpan.FromSeconds(5)); - } - catch (OperationCanceledException) - { - // ignore - } - catch (Exception ex) - { - _logger?.LogError(ex, "Error during synchronous disposal"); - } - _concurrencySemaphore.Dispose(); _shutdownCts.Dispose(); } From ef6d9328a3ca98df90a727e183c7710d4c9aac62 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 10 Apr 2026 21:37:18 +0300 Subject: [PATCH 18/28] ~ sked job id --- src/Sa.Schedule/Engine/JobFactory.cs | 2 +- src/Sa.Schedule/Engine/JobScheduler.cs | 6 +++++- src/Sa.Schedule/IJobScheduler.cs | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index a31aa2a2..a4f807e8 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -10,7 +10,7 @@ internal sealed class JobFactory( TimeProvider? timeProvider = null) : IJobFactory { public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(jobRunner, () => CreateController(settings)); + => new JobScheduler(settings.JobId, jobRunner, () => CreateController(settings)); private JobController CreateController(IJobSettings settings) { diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 86a624f3..a7b81088 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -4,6 +4,7 @@ namespace Sa.Schedule.Engine; internal sealed class JobScheduler( + Guid jobId, IJobRunner runner, Func createController) : IJobScheduler { @@ -20,9 +21,12 @@ private readonly static IChangeToken NoneChangeToken private readonly WorkQueue _jobs = new( WorkQueueOptions.Create( (controller, ct) => runner.Run(controller, ct)) - .WithQueueCapacity(40) + .WithQueueCapacity(100) .WithConcurrencyLimit(1)); + + public Guid JobId => jobId; + public bool IsActive { get diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index 0f06e53a..e206b3f1 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -7,6 +7,11 @@ namespace Sa.Schedule; /// public interface IJobScheduler: IDisposable, IAsyncDisposable { + /// + /// Gets the unique identifier of the job. + /// + Guid JobId { get; } + /// /// Gets a value indicating whether the job scheduler is currently active. /// From cf6b3c920755e218a9951645f41afd7f0139afc7 Mon Sep 17 00:00:00 2001 From: dundich Date: Mon, 13 Apr 2026 14:49:45 +0300 Subject: [PATCH 19/28] sked job --- src/Sa.Schedule/Engine/JobFactory.cs | 2 +- src/Sa.Schedule/Engine/JobScheduler.cs | 94 +++++++++++++----- src/Sa.Schedule/IJobBuilder.cs | 9 ++ src/Sa.Schedule/IJobProperties.cs | 9 ++ src/Sa.Schedule/IJobScheduler.cs | 5 + src/Sa.Schedule/IScheduler.cs | 3 + src/Sa.Schedule/Sa.Schedule.csproj | 4 + src/Sa.Schedule/Settings/JobBuilder.cs | 24 +++++ src/Sa.Schedule/Settings/JobProperies.cs | 98 +++++++++++++++++-- src/Sa/Classes/WorkQueue.cs | 50 +++++++--- .../PartitionAsJobTests.cs | 2 +- .../Sa.ScheduleTests/JobSchedulerTests.cs | 77 +++++++++++++++ .../Sa.ScheduleTests/Sa.ScheduleTests.csproj | 17 ++-- .../ScheduleConcurrencyTests.cs | 83 ++++++++++++++++ .../SchedulePostSetupTests.cs | 2 + src/Tests/SaTests/Classes/WorkQueueTests.cs | 2 +- 16 files changed, 422 insertions(+), 59 deletions(-) create mode 100644 src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs create mode 100644 src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index a4f807e8..a153525b 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -10,7 +10,7 @@ internal sealed class JobFactory( TimeProvider? timeProvider = null) : IJobFactory { public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(settings.JobId, jobRunner, () => CreateController(settings)); + => new JobScheduler(settings, jobRunner, () => CreateController(settings)); private JobController CreateController(IJobSettings settings) { diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index a7b81088..c37fa5cf 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -4,7 +4,7 @@ namespace Sa.Schedule.Engine; internal sealed class JobScheduler( - Guid jobId, + IJobSettings settings, IJobRunner runner, Func createController) : IJobScheduler { @@ -15,17 +15,40 @@ private readonly static IChangeToken NoneChangeToken private CancellationTokenSource _stoppingTokenSource = new(); private CancellationToken _originalToken; + private bool _started = false; + private bool _disposed; private readonly WorkQueue _jobs = new( - WorkQueueOptions.Create( - (controller, ct) => runner.Run(controller, ct)) - .WithQueueCapacity(100) - .WithConcurrencyLimit(1)); + WorkQueueOptions.Create((controller, ct) => runner.Run(controller, ct)) + .WithQueueCapacity(settings.Properties.MaxConcurrencyLimit + ?? Environment.ProcessorCount) + .WithMaxConcurrencyLimit(settings.Properties.MaxConcurrencyLimit + ?? Environment.ProcessorCount) + .WithConcurrencyLimit(settings.Properties.ConcurrencyLimit + ?? 1) + .WithSingleWriter(settings.Properties.SingleWriter + ?? false) + ); + + + public Guid JobId => settings.JobId; - public Guid JobId => jobId; + public int ConcurrencyLimit + { + get => _jobs.ConcurrencyLimit; + set + { + if (_jobs.ConcurrencyLimit != value) + { + _jobs.ConcurrencyLimit = Math.Min(value, _jobs.MaxConcurrency); + _ = Restart(); + } + } + } + public bool IsActive { @@ -33,7 +56,7 @@ public bool IsActive { lock (_lock) { - return !_disposed && _jobs.ActiveTasks > 0; + return !_disposed && _started; } } } @@ -54,38 +77,47 @@ public async Task Start(CancellationToken cancellationToken) { if (IsActive) return false; - CancellationTokenSource tokenToStope; - CancellationTokenSource newTokenSource; + return await StartAsync(cancellationToken); + } + private ValueTask StartAsync(CancellationToken cancellationToken) + { lock (_lock) { - if (IsActive) return false; - tokenToStope = _stoppingTokenSource; + if (!_disposed && _started) return ValueTask.FromResult(false); + + + _started = true; + + _stoppingTokenSource.Cancel(); + _stoppingTokenSource.Dispose(); _originalToken = cancellationToken; _stoppingTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - newTokenSource = _stoppingTokenSource; - } - - await tokenToStope.CancelAsync(); - tokenToStope.Dispose(); - await ExecuteJobAsync(newTokenSource.Token); - return true; + return StartConcurrencyAsync(_stoppingTokenSource.Token); + } } - private async Task ExecuteJobAsync(CancellationToken ct) + private async ValueTask StartConcurrencyAsync(CancellationToken ct) { try { - await _jobs.WaitForIdleAsync(_originalToken); - await _jobs.Enqueue(createController(), ct); + var limit = _jobs.ConcurrencyLimit; + + await _jobs.WaitForIdleAsync(ct); + + for (int i = 0; i < limit; i++) + { + await _jobs.Enqueue(createController(), ct); + } } catch { // ignore } + return IsActive; } public async Task Restart() @@ -110,13 +142,16 @@ public async Task Stop() } await _jobs.WaitForIdleAsync(_originalToken); + + lock (_lock) + { + _started = false; + } } public void Dispose() { - if (_disposed) - return; CancellationTokenSource tokenToDispose; @@ -136,16 +171,25 @@ public void Dispose() // ignore } + _jobs.Dispose(); tokenToDispose.Dispose(); } public async ValueTask DisposeAsync() { - if (_disposed) - return; + lock (_lock) + { + if (_disposed) return; + } await Stop(); + + lock (_lock) + { + _disposed = true; + } + await _jobs.DisposeAsync(); _stoppingTokenSource.Dispose(); } diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index 200ed236..f21109ab 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -76,6 +76,15 @@ IJobBuilder EverySeconds(int seconds = 1) IJobBuilder EveryMinutes(int minutes = 1) => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); + + IJobBuilder WithQueueCapacity(int capacity); + + IJobBuilder WithConcurrencyLimit(int limit); + + IJobBuilder WithMaxConcurrencyLimit(int limit); + + IJobBuilder WithSingleWriter(bool singleWriter); + /// /// Merges the specified job properties into the current job configuration. /// diff --git a/src/Sa.Schedule/IJobProperties.cs b/src/Sa.Schedule/IJobProperties.cs index 47cdbce8..daefcbc6 100644 --- a/src/Sa.Schedule/IJobProperties.cs +++ b/src/Sa.Schedule/IJobProperties.cs @@ -44,4 +44,13 @@ public interface IJobProperties /// Gets an optional tag associated with the job. /// object? Tag { get; } + + + int? QueueCapacity { get; } + + int? ConcurrencyLimit { get; } + + int? MaxConcurrencyLimit { get; } + + bool? SingleWriter { get; } } diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index e206b3f1..5472bdfc 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -17,6 +17,11 @@ public interface IJobScheduler: IDisposable, IAsyncDisposable /// bool IsActive { get; } + /// + /// + /// + int ConcurrencyLimit { get; set; } + /// /// Gets a change token that can be used to track changes to the active state of the scheduler. /// diff --git a/src/Sa.Schedule/IScheduler.cs b/src/Sa.Schedule/IScheduler.cs index 2ce46be9..6ab22755 100644 --- a/src/Sa.Schedule/IScheduler.cs +++ b/src/Sa.Schedule/IScheduler.cs @@ -33,4 +33,7 @@ public interface IScheduler /// /// A task representing the asynchronous operation. Task Stop(); + + + IJobScheduler? GetSchedule(Guid jobId) => Schedules.FirstOrDefault(c => c.JobId == jobId); } diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index 5a9120d1..5b56126c 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -25,4 +25,8 @@ + + + + diff --git a/src/Sa.Schedule/Settings/JobBuilder.cs b/src/Sa.Schedule/Settings/JobBuilder.cs index 7a0aaa40..2c6121ca 100644 --- a/src/Sa.Schedule/Settings/JobBuilder.cs +++ b/src/Sa.Schedule/Settings/JobBuilder.cs @@ -62,6 +62,30 @@ public IJobBuilder WithContextStackSize(int size) return this; } + public IJobBuilder WithQueueCapacity(int capacity) + { + settings.Properties.WithQueueCapacity(capacity); + return this; + } + + public IJobBuilder WithConcurrencyLimit(int limit) + { + settings.Properties.WithConcurrencyLimit(limit); + return this; + } + + public IJobBuilder WithMaxConcurrencyLimit(int limit) + { + settings.Properties.WithMaxConcurrencyLimit(limit); + return this; + } + + public IJobBuilder WithSingleWriter(bool singleWriter) + { + settings.Properties.WithSingleWriter(singleWriter); + return this; + } + public IJobBuilder Disabled() { settings.Properties.SetDisabled(); diff --git a/src/Sa.Schedule/Settings/JobProperies.cs b/src/Sa.Schedule/Settings/JobProperies.cs index f637b8cf..f2fad2ac 100644 --- a/src/Sa.Schedule/Settings/JobProperies.cs +++ b/src/Sa.Schedule/Settings/JobProperies.cs @@ -12,18 +12,91 @@ internal sealed class JobProperies : IJobProperties public IJobTiming? Timing { get; private set; } public object? Tag { get; private set; } public int? ContextStackSize { get; private set; } + public int? QueueCapacity { get; private set; } + public int? ConcurrencyLimit { get; private set; } + public int? MaxConcurrencyLimit { get; private set; } + public bool? SingleWriter { get; private set; } - public void WithName(string name) => JobName = name; - public void RunOnce() => IsRunOnce = true; - public void StartImmediate() => Immediate = true; - public void WithInitialDelay(TimeSpan time) => InitialDelay = time; - public void WithTiming(IJobTiming timing) => Timing = timing; - public void SetDisabled() => Disabled = true; - public void WithContextStackSize(int size) => ContextStackSize = size; - public void WithTag(object tag) => Tag = tag; + public IJobProperties WithName(string name) + { + JobName = name; + return this; + } + + public IJobProperties RunOnce() + { + IsRunOnce = true; + return this; + } + + public IJobProperties StartImmediate() + { + Immediate = true; + return this; + } + + public IJobProperties WithInitialDelay(TimeSpan time) + { + InitialDelay = time; + return this; + } - public void EveryTime(TimeSpan timeSpan, string? name = null) - => Timing = JobTiming.EveryTime(timeSpan, name); + public IJobProperties WithTiming(IJobTiming timing) + { + Timing = timing; + return this; + } + + public IJobProperties SetDisabled() + { + Disabled = true; + return this; + } + + public IJobProperties WithContextStackSize(int size) + { + ContextStackSize = size; + return this; + } + + public IJobProperties WithTag(object tag) + { + Tag = tag; + return this; + } + + public IJobProperties EveryTime(TimeSpan timeSpan, string? name = null) + { + Timing = JobTiming.EveryTime(timeSpan, name); + return this; + } + + public IJobProperties WithQueueCapacity(int capacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + QueueCapacity = capacity; + return this; + } + + public IJobProperties WithConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + ConcurrencyLimit = limit; + return this; + } + + public IJobProperties WithMaxConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + MaxConcurrencyLimit = limit; + return this; + } + + public IJobProperties WithSingleWriter(bool singleWriter) + { + SingleWriter = singleWriter; + return this; + } internal JobProperies Merge(IJobProperties props) { @@ -35,6 +108,11 @@ internal JobProperies Merge(IJobProperties props) InitialDelay ??= props.InitialDelay; ContextStackSize ??= props.ContextStackSize; Tag ??= props.Tag; + QueueCapacity ??= props.QueueCapacity; + ConcurrencyLimit ??= props.ConcurrencyLimit; + MaxConcurrencyLimit ??= props.MaxConcurrencyLimit; + SingleWriter ??= props.SingleWriter; + return this; } } diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index ddff59d4..804c9ef8 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -74,6 +74,12 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable /// int ConcurrencyLimit { get; set; } + + public int MaxConcurrency { get; } + + + public int QueueCapacity { get; } + /// /// Добавляет задачу в очередь асинхронно. /// @@ -108,7 +114,10 @@ internal sealed class WorkQueue : IWorkQueue private int _activeCount = 0; private volatile bool _isEnabled = true; - private int _maxConcurrency; + private int _сoncurrency; + private readonly int _maxConcurrency; + private readonly int _queueCapacity; + private long _taskIdCounter = 0; private bool _disposed = false; @@ -117,19 +126,23 @@ public WorkQueue(WorkQueueOptions options) _processor = options.Processor; _watcher = options.Observer ?? _processor as IWorkObserver; _timeProvider = options.TimeProvider ?? TimeProvider.System; - _maxConcurrency = options.ConcurrencyLimit ?? Environment.ProcessorCount; + _сoncurrency = options.ConcurrencyLimit ?? Environment.ProcessorCount; _statusChanged = options.StatusChanged; _logger = options.Logger; - _queue = Channel.CreateBounded(new BoundedChannelOptions(options.MaxQueueCapacity) + _maxConcurrency = Math.Max(options.MaxConcurrencyLimit ?? Environment.ProcessorCount, _сoncurrency); + _queueCapacity = options.QueueCapacity ?? _maxConcurrency; + + _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) { AllowSynchronousContinuations = true, SingleReader = false, - SingleWriter = false, + SingleWriter = options.SingleWriter ?? false, FullMode = BoundedChannelFullMode.Wait }); - _concurrencySemaphore = new SemaphoreSlim(_maxConcurrency, _maxConcurrency); + + _concurrencySemaphore = new SemaphoreSlim(_сoncurrency, _maxConcurrency); _processingTask = LoopAsync(); } @@ -148,13 +161,13 @@ public bool IsIdle() public int ConcurrencyLimit { - get => _maxConcurrency; + get => _сoncurrency; set { ObjectDisposedException.ThrowIf(_disposed, this); ArgumentOutOfRangeException.ThrowIfLessThan(value, 1); - var oldValue = Interlocked.Exchange(ref _maxConcurrency, value); + var oldValue = Interlocked.Exchange(ref _сoncurrency, value); int delta = value - oldValue; if (delta > 0) @@ -167,12 +180,14 @@ public int ConcurrencyLimit { // Игнорируем, если семафор уже на максимуме } - // Если уменьшаем — новые задачи просто не смогут захватить слот, - // а текущие завершатся естественным образом } } } + public int MaxConcurrency => _maxConcurrency; + + public int QueueCapacity => _queueCapacity; + public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); @@ -458,8 +473,10 @@ internal sealed record WorkQueueOptions // Optional with defaults IWorkObserver? Observer = null, TimeProvider? TimeProvider = null, - int MaxQueueCapacity = 20, + int? QueueCapacity = null, int? ConcurrencyLimit = null, + int? MaxConcurrencyLimit = null, + bool? SingleWriter = false, Action? StatusChanged = null, ILogger? Logger = null ) @@ -473,15 +490,24 @@ public WorkQueueOptions WithTimeProvider(TimeProvider timeProvider) => public WorkQueueOptions WithQueueCapacity(int capacity) { ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); - return this with { MaxQueueCapacity = capacity }; + return this with { QueueCapacity = capacity }; } public WorkQueueOptions WithConcurrencyLimit(int limit) { - ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); return this with { ConcurrencyLimit = limit }; } + public WorkQueueOptions WithMaxConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + return this with { MaxConcurrencyLimit = limit }; + } + + public WorkQueueOptions WithSingleWriter(bool singleWriter) => + this with { SingleWriter = singleWriter }; + public WorkQueueOptions WithStatusCallback(Action callback) => this with { StatusChanged = callback }; diff --git a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs index 415627d1..79bddf0e 100644 --- a/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs +++ b/src/Tests/Sa.Partitional.PostgreSqlTests/PartitionAsJobTests.cs @@ -10,7 +10,7 @@ public class PartitionAsJobTests(PartitionAsJobTests.Fixture fixture) : IClassFixture { - public class Fixture : PgDataSourceFixture + public sealed class Fixture : PgDataSourceFixture { public Fixture() { diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs new file mode 100644 index 00000000..ad37e0df --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -0,0 +1,77 @@ +using Sa.Schedule; +using Sa.Schedule.Engine; +using Sa.Schedule.Settings; +using System.Collections.Concurrent; + +namespace Sa.ScheduleTests; + +public class JobSchedulerTests +{ + + class TestJob : IJob + { + public async Task Execute(IJobContext context, CancellationToken cancellationToken) + => await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken); + } + + + class TestJobRunner : IJobRunner + { + public Task Run(IJobController controller, CancellationToken cancellationToken) => Task.CompletedTask; + } + + + class TestJobController : IJobController + { + public ValueTask CanExecute(CancellationToken cancellationToken) + { + return ValueTask.FromResult(CanJobExecuteResult.Ok); + } + + public Task Execute(CancellationToken cancellationToken) => Task.CompletedTask; + + public void ExecutionCompleted() + { + // + } + + public void ExecutionFailed(Exception exception) + { + // + } + + public void Running() + { + // + } + + public void Stopped(TaskStatus status) + { + // + } + + public ValueTask WaitToRun(CancellationToken cancellationToken) => ValueTask.CompletedTask; + } + + + [Fact] + public async Task Start_ConcurrentCalls_OnlyOneSucceeds() + { + + var settings = JobSettings.Create(Guid.NewGuid()); + + + var scheduler = new JobScheduler(settings, new TestJobRunner(), () => new TestJobController()); + var results = new ConcurrentBag(); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => results.Add( + await scheduler.Start(TestContext.Current.CancellationToken)) + )); + + await Task.WhenAll(tasks); + + Assert.Single(results, r => r); + Assert.Equal(9, results.Count(r => !r)); + } +} diff --git a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj index 47dc9d6f..7a9e09e3 100644 --- a/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj +++ b/src/Tests/Sa.ScheduleTests/Sa.ScheduleTests.csproj @@ -1,15 +1,14 @@ - + - - true - - - - - - + + true + + + + + diff --git a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs new file mode 100644 index 00000000..f357b5ec --- /dev/null +++ b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs @@ -0,0 +1,83 @@ +using Sa.Fixture; +using Sa.Schedule; + +namespace Sa.ScheduleTests; + + + +public sealed class ScheduleConcurrencyTests(ScheduleConcurrencyTests.Fixture fixture) + : IClassFixture +{ + public class Fixture : SaFixture + { + static class Counter + { + private readonly static HashSet jobs = []; + public static int Total => jobs.Count; + public static void Inc(SomeJob job) + { + jobs.Add(job); + } + } + + class SomeJob : IJob + { + public async Task Execute(IJobContext context, CancellationToken cancellationToken) + { + Counter.Inc(this); + await Task.Delay(10, cancellationToken); + } + } + + public Fixture() + { + Services.AddSaSchedule(b => + { + b + .AddJob(JobId) + .EveryTime(TimeSpan.FromMilliseconds(50)) + .StartImmediate() + .WithConcurrencyLimit(1) + ; + }); + } + + public static int Count => Counter.Total; + + public readonly static Guid JobId = Guid.NewGuid(); + } + + private IScheduler Sub => fixture.Sub; + + + + [Fact] + public async Task Check_ExecuteCounterJob() + { + int i = await Sub.Start(CancellationToken.None); + + Assert.NotEqual(0, i); + + await Task.Delay(100, TestContext.Current.CancellationToken); + + Assert.Equal(1, Fixture.Count); + + var j = Sub.GetSchedule(Fixture.JobId); + + Assert.NotNull(j); + + j.ConcurrencyLimit = 10; + + await Task.Delay(100, TestContext.Current.CancellationToken); + + Assert.Equal(11, Fixture.Count); + + await Task.Delay(100, TestContext.Current.CancellationToken); + Assert.Equal(11, Fixture.Count); + + j.ConcurrencyLimit = 4; + await Task.Delay(100, TestContext.Current.CancellationToken); + + Assert.Equal(15, Fixture.Count); + } +} diff --git a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs index d9b5fe91..ed6e5253 100644 --- a/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs +++ b/src/Tests/Sa.ScheduleTests/SchedulePostSetupTests.cs @@ -35,6 +35,7 @@ public Fixture() .EveryTime(TimeSpan.FromMilliseconds(100)) .RunOnce() .StartImmediate() + //.WithMaxConcurrencyLimit(1) ; }); @@ -48,6 +49,7 @@ public Fixture() .EveryTime(TimeSpan.FromMilliseconds(100)) .RunOnce() .StartImmediate() + //.WithMaxConcurrencyLimit(1) ; }); diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index a72aad3d..7fa1206b 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -74,7 +74,7 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() [Fact] - public async Task Enqueue_WithWaitForIdle_Completed() + public async Task Enqueue_WaitForIdle_Completed() { // Arrange var model = new TestModel(); From c205f09d41ad789e2ae16a414baa7112aadf746a Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 15 Apr 2026 14:37:39 +0300 Subject: [PATCH 20/28] IWorkQueue --- src/Sa.Schedule/Engine/IJobController.cs | 25 +- src/Sa.Schedule/Engine/JobController.cs | 62 +++- .../Engine/{JobPipeline.cs => JobExecutor.cs} | 4 +- src/Sa.Schedule/Engine/JobFactory.cs | 5 +- src/Sa.Schedule/Engine/JobRunner.cs | 57 ++- src/Sa.Schedule/Engine/JobScheduler.cs | 174 ++++++---- src/Sa.Schedule/Engine/Scheduler.cs | 13 +- src/Sa.Schedule/IJobBuilder.cs | 4 +- src/Sa.Schedule/IJobProperties.cs | 5 +- src/Sa.Schedule/IJobScheduler.cs | 14 +- src/Sa.Schedule/IScheduler.cs | 2 +- src/Sa.Schedule/Settings/JobBuilder.cs | 8 +- src/Sa.Schedule/Settings/JobProperies.cs | 40 +-- src/Sa/Classes/WorkQueue.cs | 324 +++++++++++------- src/Samples/Schedule.Console/Program.cs | 5 +- .../Sa.ScheduleTests/JobSchedulerTests.cs | 136 ++++++-- .../ScheduleConcurrencyTests.cs | 46 ++- src/Tests/SaTests/Classes/WorkQueueTests.cs | 25 +- 18 files changed, 623 insertions(+), 326 deletions(-) rename src/Sa.Schedule/Engine/{JobPipeline.cs => JobExecutor.cs} (95%) diff --git a/src/Sa.Schedule/Engine/IJobController.cs b/src/Sa.Schedule/Engine/IJobController.cs index 4934ea0b..5d4723e3 100644 --- a/src/Sa.Schedule/Engine/IJobController.cs +++ b/src/Sa.Schedule/Engine/IJobController.cs @@ -13,19 +13,32 @@ internal enum CanJobExecuteResult /// internal interface IJobController { - // scope events + int Index { get; } + ValueTask WaitToRun(CancellationToken cancellationToken); - void Running(); + void Init(); + + void Free(); + + + + // Методы управления паузой + bool IsPaused { get; } + + void Pause(); + + void Resume(); + + ValueTask WaitIfPaused(CancellationToken cancellationToken); + - void Stopped(TaskStatus status); // iteration events + ValueTask CanExecute(CancellationToken cancellationToken); Task Execute(CancellationToken cancellationToken); - - void ExecutionFailed(Exception exception); - void ExecutionCompleted(); + void ExecutionFailed(Exception exception); } diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index 944f004a..fb4c6997 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -9,6 +9,7 @@ namespace Sa.Schedule.Engine; /// job lifecycly controller with context /// internal sealed partial class JobController( + int index, IJobSettings settings, IInterceptorSettings interceptorSettings, IServiceScopeFactory scopeFactory, @@ -17,7 +18,17 @@ internal sealed partial class JobController( private readonly JobContext _context = new(settings); - private volatile JobPipeline? _job; + private readonly SemaphoreSlim _pauseSemaphore = new(1, 1); + + + private volatile JobExecutor? _job; + + private volatile bool _isPaused; + + + public int Index => index; + public bool IsPaused => _isPaused; + public async ValueTask WaitToRun(CancellationToken cancellationToken) { @@ -29,9 +40,51 @@ public async ValueTask WaitToRun(CancellationToken cancellationToken) } } - public void Running() + + public void Pause() + { + if (_isPaused) return; + + _isPaused = true; + _pauseSemaphore.Wait(); // Блокируем семафор + } + + public void Resume() + { + if (!_isPaused) return; + + _isPaused = false; + + try + { + _pauseSemaphore.Release(); + } + catch (SemaphoreFullException) + { + // Уже разблокирован + } + } + + public async ValueTask WaitIfPaused(CancellationToken cancellationToken) + { + if (!_isPaused) return; + + try + { + await _pauseSemaphore.WaitAsync(cancellationToken); + } + finally + { + if (!_isPaused) + { + _pauseSemaphore.Release(); + } + } + } + + public void Init() { - _job = new JobPipeline(settings, interceptorSettings, scopeFactory); + _job = new JobExecutor(settings, interceptorSettings, scopeFactory); _context.ServiceProvider = _job.ServiceProvider; @@ -39,10 +92,11 @@ public void Running() _context.NumRuns++; } - public void Stopped(TaskStatus status) + public void Free() { _context.ServiceProvider = NullJobServices.Instance; _job?.Dispose(); + _pauseSemaphore.Dispose(); } public async ValueTask CanExecute(CancellationToken cancellationToken) diff --git a/src/Sa.Schedule/Engine/JobPipeline.cs b/src/Sa.Schedule/Engine/JobExecutor.cs similarity index 95% rename from src/Sa.Schedule/Engine/JobPipeline.cs rename to src/Sa.Schedule/Engine/JobExecutor.cs index 657fe040..d5ba9b6e 100644 --- a/src/Sa.Schedule/Engine/JobPipeline.cs +++ b/src/Sa.Schedule/Engine/JobExecutor.cs @@ -3,7 +3,7 @@ namespace Sa.Schedule.Engine; -internal sealed class JobPipeline : IJob, IDisposable +internal sealed class JobExecutor : IJob, IDisposable { #region proxy class JobProxy(IJob job, IJobInterceptor interceptor, object? key) : IJob @@ -20,7 +20,7 @@ public Task Execute(IJobContext context, CancellationToken cancellationToken) private readonly IServiceScope _scope; private readonly IJob _job; - public JobPipeline( + public JobExecutor( IJobSettings settings, IInterceptorSettings interceptorSettings, IServiceScopeFactory scopeFactory) diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index a153525b..8b6cde32 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -10,11 +10,12 @@ internal sealed class JobFactory( TimeProvider? timeProvider = null) : IJobFactory { public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(settings, jobRunner, () => CreateController(settings)); + => new JobScheduler(settings, jobRunner, i => CreateController(i, settings)); - private JobController CreateController(IJobSettings settings) + private JobController CreateController(int index, IJobSettings settings) { return new( + index, settings, interceptorSettings, scopeFactory, diff --git a/src/Sa.Schedule/Engine/JobRunner.cs b/src/Sa.Schedule/Engine/JobRunner.cs index 9482e792..f6438ebd 100644 --- a/src/Sa.Schedule/Engine/JobRunner.cs +++ b/src/Sa.Schedule/Engine/JobRunner.cs @@ -9,10 +9,16 @@ public async Task Run(IJobController controller, CancellationToken cancellationT { await controller.WaitToRun(cancellationToken); - controller.Running(); + controller.Init(); - await RunLoop(controller, cancellationToken) - .ContinueWith(t => controller.Stopped(t.Status), CancellationToken.None); + try + { + await RunLoop(controller, cancellationToken); + } + finally + { + controller.Free(); + } } [StackTraceHidden] @@ -20,24 +26,41 @@ private static async Task RunLoop(IJobController controller, CancellationToken c { while (!cancellationToken.IsCancellationRequested) { - CanJobExecuteResult next = await controller.CanExecute(cancellationToken); + await controller.WaitIfPaused(cancellationToken); - if (next == CanJobExecuteResult.Abort) break; - if (next == CanJobExecuteResult.Skip) continue; + CanJobExecuteResult next = await controller.CanExecute(cancellationToken); - try + switch (next) { - await controller.Execute(cancellationToken); - controller.ExecutionCompleted(); - } - catch (OperationCanceledException ex) when (ex.CancellationToken == cancellationToken) - { - // skip - } - catch (Exception ex) - { - controller.ExecutionFailed(ex); + case CanJobExecuteResult.Abort: + return; + + case CanJobExecuteResult.Skip: + continue; + + case CanJobExecuteResult.Ok: + await ExecuteIteration(controller, cancellationToken); + break; } } } + + private static async Task ExecuteIteration( + IJobController controller, + CancellationToken cancellationToken) + { + try + { + await controller.Execute(cancellationToken); + controller.ExecutionCompleted(); + } + catch (OperationCanceledException ex) when (ex.CancellationToken == cancellationToken) + { + // Expected cancellation - silently exit + } + catch (Exception ex) + { + controller.ExecutionFailed(ex); + } + } } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index c37fa5cf..37951220 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -3,65 +3,83 @@ namespace Sa.Schedule.Engine; -internal sealed class JobScheduler( - IJobSettings settings, - IJobRunner runner, - Func createController) : IJobScheduler +internal sealed class JobScheduler : IJobScheduler { private readonly static IChangeToken NoneChangeToken = new CancellationChangeToken(CancellationToken.None); private readonly Lock _lock = new(); + private CancellationTokenSource _stoppingTokenSource = new(); private CancellationToken _originalToken; - private bool _started = false; - + private bool? _started = false; private bool _disposed; - private readonly WorkQueue _jobs = new( - WorkQueueOptions.Create((controller, ct) => runner.Run(controller, ct)) - .WithQueueCapacity(settings.Properties.MaxConcurrencyLimit - ?? Environment.ProcessorCount) - .WithMaxConcurrencyLimit(settings.Properties.MaxConcurrencyLimit - ?? Environment.ProcessorCount) - .WithConcurrencyLimit(settings.Properties.ConcurrencyLimit - ?? 1) - .WithSingleWriter(settings.Properties.SingleWriter - ?? false) - ); + private readonly WorkQueue _jobs; + + private readonly Func _createController; + + private readonly IJobRunner _runner; + + private IReadOnlyList _jobControllers = []; + + + public JobScheduler( + IJobSettings settings, + IJobRunner runner, + Func createController) + { + _runner = runner; + _createController = createController; - public Guid JobId => settings.JobId; + JobId = settings.JobId; + var concurrency = settings.Properties.ConcurrencyLimit ?? 1; + var maxConcurrency = settings.Properties.MaxConcurrency ?? concurrency; + + _jobs = new(WorkQueueOptions.Create(CreateJob) + .WithQueueCapacity(maxConcurrency) + .WithMaxConcurrency(maxConcurrency) + .WithConcurrencyLimit(concurrency) + .WithSingleWriter(true) + ); + } + + private Task CreateJob(IJobController controller, CancellationToken ct) + => _runner.Run(controller, ct); + + public Guid JobId { get; } public int ConcurrencyLimit { get => _jobs.ConcurrencyLimit; set { - if (_jobs.ConcurrencyLimit != value) - { - _jobs.ConcurrencyLimit = Math.Min(value, _jobs.MaxConcurrency); - _ = Restart(); - } + if (_jobs.ConcurrencyLimit == value) return; + + _jobs.ConcurrencyLimit = value; + RefreshConcurrency(); } } - - public bool IsActive + public bool IsStarted { get { lock (_lock) { - return !_disposed && _started; + return !_disposed && _started.GetValueOrDefault(); } } } - public IChangeToken GetActiveChangeToken() + + public int ActiveTasks => _jobs.QueueCount; + + public IChangeToken StartChangeToken() { lock (_lock) { @@ -75,20 +93,16 @@ public IChangeToken GetActiveChangeToken() /// public async Task Start(CancellationToken cancellationToken) { - if (IsActive) return false; - - return await StartAsync(cancellationToken); - } + CancellationToken stoppingToken; - private ValueTask StartAsync(CancellationToken cancellationToken) - { lock (_lock) { + if (_disposed || (_started == null || _started == true)) + { + return false; + } - if (!_disposed && _started) return ValueTask.FromResult(false); - - - _started = true; + _started = null; _stoppingTokenSource.Cancel(); _stoppingTokenSource.Dispose(); @@ -96,84 +110,101 @@ private ValueTask StartAsync(CancellationToken cancellationToken) _originalToken = cancellationToken; _stoppingTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - return StartConcurrencyAsync(_stoppingTokenSource.Token); + stoppingToken = _stoppingTokenSource.Token; } - } - private async ValueTask StartConcurrencyAsync(CancellationToken ct) - { + var maxCapacity = _jobs.MaxConcurrency; + + List controllers = new(capacity: maxCapacity); + try { - var limit = _jobs.ConcurrencyLimit; + await _jobs.WaitForIdleAsync(_originalToken); - await _jobs.WaitForIdleAsync(ct); - - for (int i = 0; i < limit; i++) + for (var i = 0; i < maxCapacity; i++) { - await _jobs.Enqueue(createController(), ct); + IJobController controller = _createController(i); + controller.Pause(); + controllers.Add(controller); + await _jobs.Enqueue(controller, stoppingToken); } + } - catch + finally { - // ignore + lock (_lock) + { + _jobControllers = controllers; + _started = true; + } } - return IsActive; - } - public async Task Restart() - { - await Stop(); - return await Start(_originalToken); + RefreshConcurrency(); + + return true; } - public async Task Stop() + private void RefreshConcurrency() { - CancellationTokenSource? tokenSourceToCancel; + IReadOnlyList controllers; lock (_lock) { - if (_disposed) return; - tokenSourceToCancel = _stoppingTokenSource; + controllers = _jobControllers; } - if (tokenSourceToCancel != null) + int limit = _jobs.ConcurrencyLimit; + + for (int i = 0; i < controllers.Count; i++) { - await tokenSourceToCancel.CancelAsync(); + if (i < limit) + { + controllers[i].Resume(); + } + else + { + controllers[i].Pause(); + } } + } - await _jobs.WaitForIdleAsync(_originalToken); - + public async Task Stop() + { lock (_lock) { + if (_disposed || !_started.GetValueOrDefault()) return; + + _stoppingTokenSource.Cancel(); _started = false; } - } + await _jobs.WaitForIdleAsync(_originalToken); + } public void Dispose() { - CancellationTokenSource tokenToDispose; + CancellationTokenSource ctsStopping; lock (_lock) { if (_disposed) return; _disposed = true; - tokenToDispose = _stoppingTokenSource; + ctsStopping = _stoppingTokenSource; } try { - tokenToDispose.Cancel(); + ctsStopping.Cancel(); } catch (ObjectDisposedException) { // ignore } - _jobs.Dispose(); - tokenToDispose.Dispose(); + + ctsStopping.Dispose(); } public async ValueTask DisposeAsync() @@ -181,16 +212,11 @@ public async ValueTask DisposeAsync() lock (_lock) { if (_disposed) return; - } - - await Stop(); - - lock (_lock) - { _disposed = true; } await _jobs.DisposeAsync(); + _stoppingTokenSource.Dispose(); } } diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index e987cce0..9a753818 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -20,9 +20,18 @@ public async Task Start(CancellationToken cancellationToken) return results.Count(r => r); } - public async Task Restart() + public async Task Restart(CancellationToken cancellationToken) { - var results = await Task.WhenAll(Schedules.Select(c => c.Restart())); + var results = await Task + .WhenAll(Schedules + .Where(c => c.IsStarted) + .Select(c => Task.Run(async () => + { + await c.Stop(); + await c.Start(cancellationToken); + return true; + }))); + return results.Count(r => r); } diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index f21109ab..413ac716 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -77,11 +77,9 @@ IJobBuilder EveryMinutes(int minutes = 1) => EveryTime(TimeSpan.FromMinutes(minutes), $"every {minutes} minutes"); - IJobBuilder WithQueueCapacity(int capacity); - IJobBuilder WithConcurrencyLimit(int limit); - IJobBuilder WithMaxConcurrencyLimit(int limit); + IJobBuilder WithMaxConcurrency(int limit); IJobBuilder WithSingleWriter(bool singleWriter); diff --git a/src/Sa.Schedule/IJobProperties.cs b/src/Sa.Schedule/IJobProperties.cs index daefcbc6..a894ba50 100644 --- a/src/Sa.Schedule/IJobProperties.cs +++ b/src/Sa.Schedule/IJobProperties.cs @@ -45,12 +45,9 @@ public interface IJobProperties /// object? Tag { get; } - - int? QueueCapacity { get; } - int? ConcurrencyLimit { get; } - int? MaxConcurrencyLimit { get; } + int? MaxConcurrency { get; } bool? SingleWriter { get; } } diff --git a/src/Sa.Schedule/IJobScheduler.cs b/src/Sa.Schedule/IJobScheduler.cs index 5472bdfc..b9842a97 100644 --- a/src/Sa.Schedule/IJobScheduler.cs +++ b/src/Sa.Schedule/IJobScheduler.cs @@ -15,7 +15,12 @@ public interface IJobScheduler: IDisposable, IAsyncDisposable /// /// Gets a value indicating whether the job scheduler is currently active. /// - bool IsActive { get; } + bool IsStarted { get; } + + /// + /// Gets the number of tasks that are contained in the Job. + /// + int ActiveTasks { get; } /// /// @@ -25,18 +30,13 @@ public interface IJobScheduler: IDisposable, IAsyncDisposable /// /// Gets a change token that can be used to track changes to the active state of the scheduler. /// - IChangeToken GetActiveChangeToken(); + IChangeToken StartChangeToken(); /// /// Starts the job scheduler asynchronously. /// Task Start(CancellationToken cancellationToken); - /// - /// Restarts the job scheduler asynchronously. - /// - Task Restart(); - /// /// Stops the job scheduler asynchronously. /// diff --git a/src/Sa.Schedule/IScheduler.cs b/src/Sa.Schedule/IScheduler.cs index 6ab22755..1002c779 100644 --- a/src/Sa.Schedule/IScheduler.cs +++ b/src/Sa.Schedule/IScheduler.cs @@ -26,7 +26,7 @@ public interface IScheduler /// Restarts the scheduler. /// /// The number of jobs restarted. - Task Restart(); + Task Restart(CancellationToken cancellationToken); /// /// Stops the scheduler. diff --git a/src/Sa.Schedule/Settings/JobBuilder.cs b/src/Sa.Schedule/Settings/JobBuilder.cs index 2c6121ca..ac956fdb 100644 --- a/src/Sa.Schedule/Settings/JobBuilder.cs +++ b/src/Sa.Schedule/Settings/JobBuilder.cs @@ -62,19 +62,13 @@ public IJobBuilder WithContextStackSize(int size) return this; } - public IJobBuilder WithQueueCapacity(int capacity) - { - settings.Properties.WithQueueCapacity(capacity); - return this; - } - public IJobBuilder WithConcurrencyLimit(int limit) { settings.Properties.WithConcurrencyLimit(limit); return this; } - public IJobBuilder WithMaxConcurrencyLimit(int limit) + public IJobBuilder WithMaxConcurrency(int limit) { settings.Properties.WithMaxConcurrencyLimit(limit); return this; diff --git a/src/Sa.Schedule/Settings/JobProperies.cs b/src/Sa.Schedule/Settings/JobProperies.cs index f2fad2ac..658615d6 100644 --- a/src/Sa.Schedule/Settings/JobProperies.cs +++ b/src/Sa.Schedule/Settings/JobProperies.cs @@ -12,87 +12,79 @@ internal sealed class JobProperies : IJobProperties public IJobTiming? Timing { get; private set; } public object? Tag { get; private set; } public int? ContextStackSize { get; private set; } - public int? QueueCapacity { get; private set; } public int? ConcurrencyLimit { get; private set; } - public int? MaxConcurrencyLimit { get; private set; } + public int? MaxConcurrency { get; private set; } public bool? SingleWriter { get; private set; } - public IJobProperties WithName(string name) + public JobProperies WithName(string name) { JobName = name; return this; } - public IJobProperties RunOnce() + public JobProperies RunOnce() { IsRunOnce = true; return this; } - public IJobProperties StartImmediate() + public JobProperies StartImmediate() { Immediate = true; return this; } - public IJobProperties WithInitialDelay(TimeSpan time) + public JobProperies WithInitialDelay(TimeSpan time) { InitialDelay = time; return this; } - public IJobProperties WithTiming(IJobTiming timing) + public JobProperies WithTiming(IJobTiming timing) { Timing = timing; return this; } - public IJobProperties SetDisabled() + public JobProperies SetDisabled() { Disabled = true; return this; } - public IJobProperties WithContextStackSize(int size) + public JobProperies WithContextStackSize(int size) { ContextStackSize = size; return this; } - public IJobProperties WithTag(object tag) + public JobProperies WithTag(object tag) { Tag = tag; return this; } - public IJobProperties EveryTime(TimeSpan timeSpan, string? name = null) + public JobProperies EveryTime(TimeSpan timeSpan, string? name = null) { Timing = JobTiming.EveryTime(timeSpan, name); return this; } - public IJobProperties WithQueueCapacity(int capacity) - { - ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); - QueueCapacity = capacity; - return this; - } - - public IJobProperties WithConcurrencyLimit(int limit) + public JobProperies WithConcurrencyLimit(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); ConcurrencyLimit = limit; return this; } - public IJobProperties WithMaxConcurrencyLimit(int limit) + public JobProperies WithMaxConcurrencyLimit(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); - MaxConcurrencyLimit = limit; + MaxConcurrency = limit; return this; } - public IJobProperties WithSingleWriter(bool singleWriter) + public JobProperies WithSingleWriter(bool singleWriter) { SingleWriter = singleWriter; return this; @@ -108,9 +100,9 @@ internal JobProperies Merge(IJobProperties props) InitialDelay ??= props.InitialDelay; ContextStackSize ??= props.ContextStackSize; Tag ??= props.Tag; - QueueCapacity ??= props.QueueCapacity; + ConcurrencyLimit ??= props.ConcurrencyLimit; - MaxConcurrencyLimit ??= props.MaxConcurrencyLimit; + MaxConcurrency ??= props.MaxConcurrency; SingleWriter ??= props.SingleWriter; return this; diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 804c9ef8..d02114e0 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -55,14 +55,9 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable bool IsEnabled { get; } /// - /// Текущее количество активно выполняющихся задач. + /// Gets the current number of items available from queue. /// - int ActiveTasks { get; } - - /// - /// Gets the current number of items available from this channel reader. - /// - int QueuedTasks { get; } + int QueueCount { get; } /// /// Определяет, свободна ли очередь (нет активных и нет в очереди). @@ -99,110 +94,116 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable internal sealed class WorkQueue : IWorkQueue { private readonly Channel _queue; - private readonly HashSet _activeItemIds = []; + private readonly HashSet _queueItems = []; + private readonly Lock _rootSync = new(); - private readonly SemaphoreSlim _concurrencySemaphore; + private readonly Lock _activeReadersSync = new(); + private readonly ILogger? _logger = null; private readonly TimeProvider _timeProvider; public readonly IWork _processor; public readonly IWorkObserver? _watcher; + private readonly Action? _statusChanged; + private readonly CancellationTokenSource _shutdownCts = new(); private readonly Task _processingTask; - private readonly Action? _statusChanged; - private volatile TaskCompletionSource _idleTcs = new(); - private int _activeCount = 0; + + private volatile CancellationTokenSource? _throttleCts = new(); + private volatile TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); private volatile bool _isEnabled = true; + private volatile bool _disposed = false; - private int _сoncurrency; + + private int _concurrency; private readonly int _maxConcurrency; private readonly int _queueCapacity; + private int _queueCount; + private long _taskIdCounter; + + private readonly List _activeReaders = []; - private long _taskIdCounter = 0; - private bool _disposed = false; public WorkQueue(WorkQueueOptions options) { + ArgumentNullException.ThrowIfNull(options.Processor); + _processor = options.Processor; _watcher = options.Observer ?? _processor as IWorkObserver; _timeProvider = options.TimeProvider ?? TimeProvider.System; - _сoncurrency = options.ConcurrencyLimit ?? Environment.ProcessorCount; _statusChanged = options.StatusChanged; _logger = options.Logger; - _maxConcurrency = Math.Max(options.MaxConcurrencyLimit ?? Environment.ProcessorCount, _сoncurrency); + _maxConcurrency = options.MaxConcurrencyLimit ?? Environment.ProcessorCount; + + _concurrency = Math.Clamp( + options.ConcurrencyLimit ?? Environment.ProcessorCount, + 0, + _maxConcurrency); + _queueCapacity = options.QueueCapacity ?? _maxConcurrency; _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) { - AllowSynchronousContinuations = true, + AllowSynchronousContinuations = false, SingleReader = false, SingleWriter = options.SingleWriter ?? false, - FullMode = BoundedChannelFullMode.Wait + FullMode = options.FullMode }); - - _concurrencySemaphore = new SemaphoreSlim(_сoncurrency, _maxConcurrency); - - _processingTask = LoopAsync(); + _processingTask = ProcessQueueAsync(); } public bool IsEnabled => _isEnabled; - public int ActiveTasks => _activeCount; - public int QueuedTasks => _queue.Reader.Count; + + public int QueueCount => _queueCount; public bool IsIdle() { lock (_rootSync) { - return _activeItemIds.Count == 0 && _queue.Reader.Count == 0; + return _queueItems.Count == 0 && _queue.Reader.Count == 0; } } public int ConcurrencyLimit { - get => _сoncurrency; + get => _concurrency; set { ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentOutOfRangeException.ThrowIfLessThan(value, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(value, 0); - var oldValue = Interlocked.Exchange(ref _сoncurrency, value); - int delta = value - oldValue; + var newLimit = Math.Clamp(value, 0, _maxConcurrency); - if (delta > 0) - { - try - { - _concurrencySemaphore.Release(delta); - } - catch (SemaphoreFullException) - { - // Игнорируем, если семафор уже на максимуме - } - } + Interlocked.Exchange(ref _concurrency, newLimit); } } + public int MaxConcurrency => _maxConcurrency; public int QueueCapacity => _queueCapacity; + + public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); if (!_isEnabled) - throw new InvalidOperationException("Task scheduler has been stopped."); + throw new InvalidOperationException("Queue has been stopped."); - var snapshot = new WorkItem(model, cancellationToken) + var item = new WorkItem(model, cancellationToken) { Id = Interlocked.Increment(ref _taskIdCounter), Status = WorkStatus.Queued, EnqueuedTime = _timeProvider.GetUtcNow(), }; - await OnStatusChangedAsync(snapshot); - await _queue.Writer.WriteAsync(snapshot, cancellationToken); + await OnStatusChangedAsync(item).ConfigureAwait(false); + await _queue.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false); + + ActivateItem(item); } public async Task ShutdownAsync() @@ -213,11 +214,18 @@ public async Task ShutdownAsync() _isEnabled = false; } - await _shutdownCts.CancelAsync(); + await _shutdownCts.CancelAsync().ConfigureAwait(false); try { - await _processingTask; + await _processingTask.ConfigureAwait(false); + + Task[] readersCopy; + lock (_activeReadersSync) + { + readersCopy = [.. _activeReaders]; + } + await Task.WhenAll(readersCopy).ConfigureAwait(false); } catch (Exception ex) { @@ -231,94 +239,137 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default while (!IsIdle()) { - var currentTcs = _idleTcs; - await currentTcs.Task.WaitAsync(cancellationToken); + TaskCompletionSource tcs; + lock (_rootSync) + { + if (IsIdle()) break; // Повторная проверка после входа в блокировку + tcs = _idleTcs; + } + await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); } } - private async Task LoopAsync() + private async Task ProcessQueueAsync() { - var ct = _shutdownCts.Token; try { - while (!ct.IsCancellationRequested && await _queue.Reader.WaitToReadAsync(ct)) - { - // Каждый поток пытается читать, пока есть данные - while (_isEnabled && _queue.Reader.TryRead(out var item)) - { - // SemaphoreSlim автоматически ограничит реальную параллельность - _ = ExecuteWithConcurrencyControlAsync(item, ct); - } - } + // Запускаем начальных reader'ов + await SpawnAdditionalReaders(_concurrency).ConfigureAwait(false); + await MonitorQueueAsync().ConfigureAwait(false); } catch (OperationCanceledException) { - /* ignore */ + // Normal shutdown } catch (Exception ex) { - _logger?.LogError(ex, "Unhandled error in queue processing loop"); + _logger?.LogError(ex, "Queue processing failed"); } } - // Обёртка: захват слота -> выполнение -> освобождение - private async Task ExecuteWithConcurrencyControlAsync(WorkItem item, CancellationToken loopCt) + private async Task MonitorQueueAsync() { - // Ждём доступного слота (с учётом отмены) - try - { - await _concurrencySemaphore.WaitAsync(loopCt); - } - catch (OperationCanceledException) - { - // Если отменили ожидание слота — возвращаем задачу в очередь или отменяем - await CancelWorkItemAsync(item); - return; - } - + var shutdownToken = _shutdownCts.Token; - try + while (!shutdownToken.IsCancellationRequested) { - // Проверяем, не остановили ли очередь пока ждали слот - if (!_isEnabled) + // Ждём появления элементов или изменения состояния + if (await _queue.Reader.WaitToReadAsync(shutdownToken).ConfigureAwait(false)) { - await CancelWorkItemAsync(item); - return; + // Убеждаемся что есть активные reader'ы + await EnsureReadersActive().ConfigureAwait(false); } - await ExecuteAsync(item); + // Небольшая задержка чтобы не спамить + await Task.Delay(100, shutdownToken).ConfigureAwait(false); } - catch (Exception ex) + } + + private Task EnsureReadersActive() + { + lock (_activeReadersSync) { - _logger?.LogError(ex, "Unhandled error executing work item #{WorkItemId}", item.Id); + var activeCount = _activeReaders.Count(t => !t.IsCompleted); + var targetCount = _concurrency; - if (item.Status is WorkStatus.Queued or WorkStatus.Running) + if (activeCount < targetCount && _isEnabled && !_disposed) { - item.Status = WorkStatus.Faulted; - item.LastError = ex; - item.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(item); + return SpawnAdditionalReaders(targetCount - activeCount); } } - finally + + return Task.CompletedTask; + } + + private Task SpawnAdditionalReaders(int count) + { + var tasks = new List(capacity: count); + + lock (_activeReadersSync) { - _concurrencySemaphore.Release(); // Освобождаем слот + for (int i = 0; i < count; i++) + { + Task readerTask = ReaderLoopAsync(_shutdownCts.Token); + _activeReaders.Add(readerTask); + tasks.Add(readerTask); + } } + + // Не ждём запуска + var _ = Task.WhenAll(tasks).ContinueWith(t => + { + lock (_activeReadersSync) + { + foreach (var task in tasks) + { + _activeReaders.Remove(task); + } + } + }); + + return Task.CompletedTask; } - private async Task CancelWorkItemAsync(WorkItem item) + + private async Task ReaderLoopAsync(CancellationToken shutdownToken) { - item.Status = WorkStatus.Cancelled; - item.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(item); + + while (!shutdownToken.IsCancellationRequested && _isEnabled) + { + + // Создаём токен с учётом троттлинга + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + shutdownToken, + _throttleCts?.Token ?? CancellationToken.None); + + try + { + var item = await _queue.Reader.ReadAsync(linkedCts.Token).ConfigureAwait(false); + + // Запускаем выполнение без ожидания + _ = ExecuteItemAsync(item, shutdownToken); + } + catch (OperationCanceledException) when (shutdownToken.IsCancellationRequested) + { + return; + } + catch (OperationCanceledException) + { + // Троттлинг — перепроверяем условия + } + catch (Exception ex) + { + _logger?.LogError(ex, "Reader error"); + await Task.Delay(100, shutdownToken).ConfigureAwait(false); + } + } } - private async Task ExecuteAsync(WorkItem item) - { - ActivateItem(item); + private async Task ExecuteItemAsync(WorkItem item, CancellationToken shutdownToken) + { using var cts = CancellationTokenSource.CreateLinkedTokenSource( - _shutdownCts.Token, + shutdownToken, item.CancellationToken); var ct = cts.Token; @@ -329,9 +380,10 @@ private async Task ExecuteAsync(WorkItem item) item.StartedTime = _timeProvider.GetUtcNow(); item.Status = WorkStatus.Running; - await OnStatusChangedAsync(item); + await OnStatusChangedAsync(item).ConfigureAwait(false); + + await _processor.Execute(item.Model, ct).ConfigureAwait(false); - await _processor.Execute(item.Model, ct); item.Status = WorkStatus.Completed; } catch (OperationCanceledException) when (ct.IsCancellationRequested) @@ -342,12 +394,16 @@ private async Task ExecuteAsync(WorkItem item) { item.Status = WorkStatus.Faulted; item.LastError = ex; + _logger?.LogError(ex, "Item {ItemId} execution failed", item.Id); } finally { - item.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(item); - DeactivateItem(item); + if (!_disposed) + { + item.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(item).ConfigureAwait(false); + DeactivateItem(item); + } } } @@ -357,11 +413,11 @@ private void ActivateItem(WorkItem item) lock (_rootSync) { - _activeItemIds.Add(item.Id); - _activeCount = _activeItemIds.Count; + _queueItems.Add(item); + _queueCount = _queueItems.Count; oldTcs = _idleTcs; - _idleTcs = new TaskCompletionSource(); + _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } oldTcs.TrySetResult(); @@ -373,10 +429,10 @@ private void DeactivateItem(WorkItem item) lock (_rootSync) { - _activeItemIds.Remove(item.Id); - _activeCount = _activeItemIds.Count; + _queueItems.Remove(item); + _queueCount = _queueItems.Count; - if (_activeItemIds.Count == 0 && _queue.Reader.Count == 0) + if (_queueItems.Count == 0 && _queue.Reader.Count == 0) { tcsToNotify = _idleTcs; } @@ -394,7 +450,7 @@ private async Task OnStatusChangedAsync(WorkItem item) { try { - await watcher.HandleChanges(item.Model, info, _shutdownCts.Token); + await watcher.HandleChanges(item.Model, info, _shutdownCts.Token).ConfigureAwait(false); } catch (Exception ex) { @@ -420,34 +476,56 @@ public void Dispose() { if (_disposed) return; - _disposed = true; _isEnabled = false; _queue.Writer.TryComplete(); + _shutdownCts.Cancel(); - _concurrencySemaphore.Dispose(); + var throttle = Interlocked.Exchange(ref _throttleCts, null); + throttle?.Cancel(); + + _disposed = true; + _shutdownCts.Dispose(); + throttle?.Dispose(); } public async ValueTask DisposeAsync() { if (_disposed) return; - _disposed = true; _isEnabled = false; _queue.Writer.TryComplete(); - await _shutdownCts.CancelAsync(); + await _shutdownCts.CancelAsync().ConfigureAwait(false); + + var throttle = Interlocked.Exchange(ref _throttleCts, null); + if (throttle != null) + { + await throttle.CancelAsync().ConfigureAwait(false); + } + try { - await _processingTask; + await _processingTask.ConfigureAwait(false); + + Task[] readersCopy; + lock (_activeReadersSync) + { + readersCopy = [.. _activeReaders]; + } + await Task.WhenAll(readersCopy).ConfigureAwait(false); } catch (Exception ex) { _logger?.LogError(ex, "Error during synchronous disposal"); } - _concurrencySemaphore.Dispose(); - _shutdownCts.Dispose(); + finally + { + _disposed = true; + _shutdownCts.Dispose(); + throttle?.Dispose(); + } } private sealed class WorkItem(TModel item, CancellationToken token) @@ -472,13 +550,15 @@ internal sealed record WorkQueueOptions IWork Processor, // Optional with defaults IWorkObserver? Observer = null, + TimeProvider? TimeProvider = null, int? QueueCapacity = null, int? ConcurrencyLimit = null, int? MaxConcurrencyLimit = null, bool? SingleWriter = false, Action? StatusChanged = null, - ILogger? Logger = null + ILogger? Logger = null, + BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait ) { public WorkQueueOptions WithObserver(IWorkObserver observer) => @@ -499,7 +579,7 @@ public WorkQueueOptions WithConcurrencyLimit(int limit) return this with { ConcurrencyLimit = limit }; } - public WorkQueueOptions WithMaxConcurrencyLimit(int limit) + public WorkQueueOptions WithMaxConcurrency(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); return this with { MaxConcurrencyLimit = limit }; @@ -514,6 +594,10 @@ public WorkQueueOptions WithStatusCallback(Action callback) => public WorkQueueOptions WithLogger(ILogger> logger) => this with { Logger = logger }; + + public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) => + this with { FullMode = mode }; + public static WorkQueueOptions Create(IWork processor) => new(processor); diff --git a/src/Samples/Schedule.Console/Program.cs b/src/Samples/Schedule.Console/Program.cs index 7d7a89d5..3285c3d1 100644 --- a/src/Samples/Schedule.Console/Program.cs +++ b/src/Samples/Schedule.Console/Program.cs @@ -53,9 +53,10 @@ { await Task.Delay(5000); await controller.Stop(); - Console.WriteLine($"*** stopped & restart after 2 sec"); + Console.WriteLine($"*** stopped & start after 2 sec"); + await Task.Delay(2000); - await controller.Restart(); + await controller.Start(cts.Token); }); _ = Task.Run(async () => diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs index ad37e0df..20c183aa 100644 --- a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -8,6 +8,95 @@ namespace Sa.ScheduleTests; public class JobSchedulerTests { + [Fact] + public async Task Start_ConcurrentCalls_OnlyOneSucceeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + var results = new ConcurrentBag(); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => results.Add( + await scheduler.Start(TestContext.Current.CancellationToken)) + )); + + await Task.WhenAll(tasks); + + Assert.Single(results, r => r); + } + + + [Fact] + public async Task Stop_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => + await scheduler.Stop() + )); + + await Task.WhenAll(tasks); + + + Assert.False(scheduler.IsStarted); + Assert.Equal(0, scheduler.ActiveTasks); + } + + + [Fact] + public async Task Dispose_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(async () => await scheduler.DisposeAsync())); + + await Task.WhenAll(tasks); + + Assert.False(scheduler.IsStarted); + Assert.Equal(0, scheduler.ActiveTasks); + } + + + [Fact] + public async Task ConcurrencyLimit_ConcurrentCalls_Succeeds() + { + var settings = JobSettings.Create(Guid.NewGuid()); + settings.Properties + .WithMaxConcurrencyLimit(30) + .WithConcurrencyLimit(1); + + var scheduler = new JobScheduler(settings, new TestJobRunner(), i => new TestJobController(i)); + + await scheduler.Start(TestContext.Current.CancellationToken); + + Assert.True(scheduler.IsStarted); + + var tasks = Enumerable.Range(0, 10).Select(_ => + Task.Run(() => scheduler.ConcurrencyLimit = Random.Shared.Next(2, 45))); + + await Task.WhenAll(tasks); + + Assert.True(scheduler.IsStarted); + Assert.InRange(scheduler.ConcurrencyLimit, 2, 30); + + await scheduler.DisposeAsync(); + } + + + class TestJob : IJob { public async Task Execute(IJobContext context, CancellationToken cancellationToken) @@ -21,8 +110,12 @@ class TestJobRunner : IJobRunner } - class TestJobController : IJobController + class TestJobController(int index) : IJobController { + public bool IsPaused => false; + + public int Index => index; + public ValueTask CanExecute(CancellationToken cancellationToken) { return ValueTask.FromResult(CanJobExecuteResult.Ok); @@ -40,38 +133,31 @@ public void ExecutionFailed(Exception exception) // } - public void Running() + public void Pause() { - // - } - public void Stopped(TaskStatus status) - { - // } - public ValueTask WaitToRun(CancellationToken cancellationToken) => ValueTask.CompletedTask; - } - - - [Fact] - public async Task Start_ConcurrentCalls_OnlyOneSucceeds() - { - - var settings = JobSettings.Create(Guid.NewGuid()); + public void Resume() + { + } - var scheduler = new JobScheduler(settings, new TestJobRunner(), () => new TestJobController()); - var results = new ConcurrentBag(); + public void Init() + { + // + } - var tasks = Enumerable.Range(0, 10).Select(_ => - Task.Run(async () => results.Add( - await scheduler.Start(TestContext.Current.CancellationToken)) - )); + public void Free() + { + // + } - await Task.WhenAll(tasks); + public ValueTask WaitIfPaused(CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } - Assert.Single(results, r => r); - Assert.Equal(9, results.Count(r => !r)); + public ValueTask WaitToRun(CancellationToken cancellationToken) => ValueTask.CompletedTask; } } diff --git a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs index f357b5ec..730852e1 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs @@ -12,11 +12,30 @@ public class Fixture : SaFixture { static class Counter { + private static readonly Lock @lock = new(); + private readonly static HashSet jobs = []; - public static int Total => jobs.Count; + public static int Total + { + get + { + lock (@lock) + { + return jobs.Count; + } + } + } public static void Inc(SomeJob job) { - jobs.Add(job); + lock (@lock) + { + jobs.Add(job); + } + } + + public static void Clear() + { + lock (@lock) { jobs.Clear(); } } } @@ -35,9 +54,10 @@ public Fixture() { b .AddJob(JobId) - .EveryTime(TimeSpan.FromMilliseconds(50)) + .EveryTime(TimeSpan.FromMilliseconds(10)) .StartImmediate() .WithConcurrencyLimit(1) + .WithMaxConcurrency(10) ; }); } @@ -45,6 +65,7 @@ public Fixture() public static int Count => Counter.Total; public readonly static Guid JobId = Guid.NewGuid(); + public static void Reset() => Counter.Clear(); } private IScheduler Sub => fixture.Sub; @@ -54,30 +75,33 @@ public Fixture() [Fact] public async Task Check_ExecuteCounterJob() { + Fixture.Reset(); + int i = await Sub.Start(CancellationToken.None); Assert.NotEqual(0, i); - await Task.Delay(100, TestContext.Current.CancellationToken); + await Task.Delay(200, TestContext.Current.CancellationToken); Assert.Equal(1, Fixture.Count); + Fixture.Reset(); var j = Sub.GetSchedule(Fixture.JobId); - Assert.NotNull(j); j.ConcurrencyLimit = 10; await Task.Delay(100, TestContext.Current.CancellationToken); - Assert.Equal(11, Fixture.Count); - - await Task.Delay(100, TestContext.Current.CancellationToken); - Assert.Equal(11, Fixture.Count); + Assert.Equal(10, Fixture.Count); j.ConcurrencyLimit = 4; - await Task.Delay(100, TestContext.Current.CancellationToken); - Assert.Equal(15, Fixture.Count); + await Task.Delay(200, TestContext.Current.CancellationToken); + Fixture.Reset(); + + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.Equal(4, Fixture.Count); } } diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index 7fa1206b..3e000294 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -68,8 +68,8 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() // Assert Assert.True(model.WasProcessed); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueCount); + } @@ -90,8 +90,7 @@ public async Task Enqueue_WaitForIdle_Completed() // Assert Assert.True(model.WasProcessed); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueCount); } [Fact] @@ -113,8 +112,7 @@ public async Task Enqueue_MultipleTasks_AllExecuted() // Assert Assert.All(models, m => Assert.True(m.WasProcessed)); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueCount); } [Fact] @@ -138,13 +136,14 @@ public async Task ConcurrencyLimit_Respected() await Task.Delay(50, TestToken); // Assert - Assert.True(queue.ActiveTasks <= concurrencyLimit, "Concurrency limit violated"); + Assert.True(queue.ConcurrencyLimit <= concurrencyLimit, "Concurrency limit violated"); + + Assert.False(queue.IsIdle()); await queue.WaitForIdleAsync(cancellationToken: TestToken); - Assert.Equal(0, queue.ActiveTasks); - Assert.Equal(0, queue.QueuedTasks); + Assert.Equal(0, queue.QueueCount); } [Fact] @@ -215,7 +214,7 @@ public async Task ShutdownAsync_StopsProcessing() // Assert Assert.False(queue.IsEnabled); - Assert.True(queue.ActiveTasks <= 5); + Assert.True(queue.QueueCount <= 5); await Assert.ThrowsAsync(async () => { @@ -223,7 +222,7 @@ await Assert.ThrowsAsync(async () => }); await queue.WaitForIdleAsync(TestToken); - Assert.Equal(0, queue.ActiveTasks); + Assert.Equal(0, queue.QueueCount); var canceled = observer.Changes.Count(c => c.Status == WorkStatus.Cancelled); Assert.Equal(5, canceled); @@ -293,9 +292,6 @@ public async Task DisposeAsync_CleansUpResources() await queue.Enqueue(new TestModel(), cancellationToken: TestToken); await queue.DisposeAsync(); - // Act & Assert - Assert.False(queue.IsEnabled); - Assert.Equal(0, queue.QueuedTasks); // Writer completed await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); } @@ -306,7 +302,6 @@ public void ConcurrencyLimit_InvalidValue_Throws() var queue = new WorkQueue( WorkQueueOptions.Create(new TestWork())); - Assert.Throws(() => queue.ConcurrencyLimit = 0); Assert.Throws(() => queue.ConcurrencyLimit = -1); queue.Dispose(); From 7eded5e0161c445c07fe9981d469c97de2db86d9 Mon Sep 17 00:00:00 2001 From: dundich Date: Thu, 16 Apr 2026 15:44:46 +0300 Subject: [PATCH 21/28] WorkQueueConcurrencyTests --- src/Sa.Schedule/Engine/JobScheduler.cs | 2 +- src/Sa/Classes/WorkQueue.cs | 312 ++++++----- .../Classes/WorkQueueConcurrencyTests.cs | 518 ++++++++++++++++++ src/Tests/SaTests/Classes/WorkQueueTests.cs | 23 +- 4 files changed, 715 insertions(+), 140 deletions(-) create mode 100644 src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 37951220..d549622d 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -77,7 +77,7 @@ public bool IsStarted } - public int ActiveTasks => _jobs.QueueCount; + public int ActiveTasks => _jobs.QueueTasks; public IChangeToken StartChangeToken() { diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index d02114e0..144449a0 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -7,17 +7,17 @@ namespace Sa.Classes; /// /// Интерфейс для выполнения бизнес-логики. /// -internal interface IWork +internal interface IWork { - Task Execute(TModel model, CancellationToken cancellationToken); + Task Execute(TInput input, CancellationToken cancellationToken); } /// /// Интерфейс для наблюдения за изменениями состояния задач в очереди. /// -internal interface IWorkObserver +internal interface IWorkObserver { - Task HandleChanges(TModel model, WorkInfo work, CancellationToken cancellationToken); + Task HandleChanges(TInput input, WorkInfo work, CancellationToken cancellationToken); } /// @@ -47,7 +47,7 @@ internal enum WorkStatus /// /// Асинхронная очередь задач с ограничением параллелизма. /// -internal interface IWorkQueue : IDisposable, IAsyncDisposable +internal interface IWorkQueue : IDisposable, IAsyncDisposable { /// /// Показывает, включена ли обработка задач. @@ -55,9 +55,10 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable bool IsEnabled { get; } /// - /// Gets the current number of items available from queue. + /// Gets the current inputs from queue. /// - int QueueCount { get; } + int QueueTasks { get; } + /// /// Определяет, свободна ли очередь (нет активных и нет в очереди). @@ -78,7 +79,7 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable /// /// Добавляет задачу в очередь асинхронно. /// - ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default); + ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default); /// /// Блокируется до момента, когда очередь станет полностью пустой. @@ -91,25 +92,24 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable Task ShutdownAsync(); } -internal sealed class WorkQueue : IWorkQueue +internal sealed class WorkQueue : IWorkQueue { private readonly Channel _queue; - private readonly HashSet _queueItems = []; + private readonly HashSet _workItems = []; private readonly Lock _rootSync = new(); private readonly Lock _activeReadersSync = new(); private readonly ILogger? _logger = null; private readonly TimeProvider _timeProvider; - public readonly IWork _processor; - public readonly IWorkObserver? _watcher; + public readonly IWork _processor; + public readonly IWorkObserver? _watcher; private readonly Action? _statusChanged; private readonly CancellationTokenSource _shutdownCts = new(); private readonly Task _processingTask; - private volatile CancellationTokenSource? _throttleCts = new(); private volatile TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); private volatile bool _isEnabled = true; private volatile bool _disposed = false; @@ -118,23 +118,25 @@ internal sealed class WorkQueue : IWorkQueue private int _concurrency; private readonly int _maxConcurrency; private readonly int _queueCapacity; - private int _queueCount; + private int _workItemsCount; private long _taskIdCounter; - private readonly List _activeReaders = []; + private readonly List _activeReaders = []; - public WorkQueue(WorkQueueOptions options) + public WorkQueue(WorkQueueOptions options) { ArgumentNullException.ThrowIfNull(options.Processor); _processor = options.Processor; - _watcher = options.Observer ?? _processor as IWorkObserver; + _watcher = options.Observer ?? _processor as IWorkObserver; _timeProvider = options.TimeProvider ?? TimeProvider.System; _statusChanged = options.StatusChanged; _logger = options.Logger; - _maxConcurrency = options.MaxConcurrencyLimit ?? Environment.ProcessorCount; + _maxConcurrency = options.MaxConcurrency > 0 + ? options.MaxConcurrency.Value + : Environment.ProcessorCount; _concurrency = Math.Clamp( options.ConcurrencyLimit ?? Environment.ProcessorCount, @@ -156,13 +158,14 @@ public WorkQueue(WorkQueueOptions options) public bool IsEnabled => _isEnabled; - public int QueueCount => _queueCount; + public int QueueTasks => _workItemsCount; + public bool IsIdle() { lock (_rootSync) { - return _queueItems.Count == 0 && _queue.Reader.Count == 0; + return _workItems.Count == 0 && _queue.Reader.Count == 0; } } @@ -172,10 +175,8 @@ public int ConcurrencyLimit set { ObjectDisposedException.ThrowIf(_disposed, this); - ArgumentOutOfRangeException.ThrowIfLessThan(value, 0); var newLimit = Math.Clamp(value, 0, _maxConcurrency); - Interlocked.Exchange(ref _concurrency, newLimit); } } @@ -187,13 +188,13 @@ public int ConcurrencyLimit - public async ValueTask Enqueue(TModel model, CancellationToken cancellationToken = default) + public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); if (!_isEnabled) throw new InvalidOperationException("Queue has been stopped."); - var item = new WorkItem(model, cancellationToken) + var item = new WorkItem(input, cancellationToken) { Id = Interlocked.Increment(ref _taskIdCounter), Status = WorkStatus.Queued, @@ -220,41 +221,66 @@ public async Task ShutdownAsync() { await _processingTask.ConfigureAwait(false); - Task[] readersCopy; + Task[] readers; lock (_activeReadersSync) { - readersCopy = [.. _activeReaders]; + readers = [.. _activeReaders.Select(c => c.Task)]; } - await Task.WhenAll(readersCopy).ConfigureAwait(false); + await Task.WhenAll(readers).ConfigureAwait(false); } catch (Exception ex) { _logger?.LogError(ex, "Error during shutdown"); } + finally + { + await ClearAll().ConfigureAwait(false); + } + } + + private async Task ClearAll() + { + while (_queue.Reader.TryRead(out var input)) + { + input.Status = WorkStatus.Faulted; + input.LastError = ShutdownAsyncException.Default; + input.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(input).ConfigureAwait(false); + DeactivateItem(input); + } } public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + _shutdownCts.Token, + cancellationToken); + while (!IsIdle()) { TaskCompletionSource tcs; lock (_rootSync) { - if (IsIdle()) break; // Повторная проверка после входа в блокировку + if (IsIdle()) break; tcs = _idleTcs; } - await tcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + + await Task.WhenAny( + tcs.Task.WaitAsync(cts.Token), + Task.Delay(1000, cts.Token)).ConfigureAwait(false); } } + /// + /// init readers + /// private async Task ProcessQueueAsync() { try { - // Запускаем начальных reader'ов - await SpawnAdditionalReaders(_concurrency).ConfigureAwait(false); + SpawnAdditionalReaders(_concurrency); await MonitorQueueAsync().ConfigureAwait(false); } catch (OperationCanceledException) @@ -273,107 +299,109 @@ private async Task MonitorQueueAsync() while (!shutdownToken.IsCancellationRequested) { - // Ждём появления элементов или изменения состояния if (await _queue.Reader.WaitToReadAsync(shutdownToken).ConfigureAwait(false)) { - // Убеждаемся что есть активные reader'ы - await EnsureReadersActive().ConfigureAwait(false); + EnsureReadersActive(); } - // Небольшая задержка чтобы не спамить await Task.Delay(100, shutdownToken).ConfigureAwait(false); } } - private Task EnsureReadersActive() + private void EnsureReadersActive() { lock (_activeReadersSync) { - var activeCount = _activeReaders.Count(t => !t.IsCompleted); + if (!_isEnabled && _disposed) return; + + var activeCount = GetActiveCount(); + var targetCount = _concurrency; - if (activeCount < targetCount && _isEnabled && !_disposed) + var delta = targetCount - activeCount; + + if (delta > 0) + { + SpawnAdditionalReaders(delta); + } + else if (delta < 0) { - return SpawnAdditionalReaders(targetCount - activeCount); + foreach (var item in _activeReaders.Take(Math.Abs(delta))) + { + item.TokenSource.Cancel(); + } } } - - return Task.CompletedTask; } - private Task SpawnAdditionalReaders(int count) + private int GetActiveCount() { - var tasks = new List(capacity: count); + return _activeReaders + .Count(t => !t.Task.IsCompleted || !t.TokenSource.IsCancellationRequested); + } - lock (_activeReadersSync) + private void SpawnAdditionalReaders(int count) + { + void Delete(ReaderTask reader) { - for (int i = 0; i < count; i++) + lock (_activeReadersSync) { - Task readerTask = ReaderLoopAsync(_shutdownCts.Token); - _activeReaders.Add(readerTask); - tasks.Add(readerTask); + _activeReaders.Remove(reader); + reader.Dispose(); } } - // Не ждём запуска - var _ = Task.WhenAll(tasks).ContinueWith(t => + lock (_activeReadersSync) { - lock (_activeReadersSync) + for (int i = 0; i < count; i++) { - foreach (var task in tasks) - { - _activeReaders.Remove(task); - } - } - }); + CancellationTokenSource tokenSource = new(); + Task readerTask = ReaderLoopAsync(tokenSource.Token); + + ReaderTask reader = new(readerTask, tokenSource); + + _activeReaders.Add(reader); - return Task.CompletedTask; + readerTask.ContinueWith(t => Delete(reader)); + } + } } - private async Task ReaderLoopAsync(CancellationToken shutdownToken) + private async Task ReaderLoopAsync(CancellationToken cancellationToken) { + var shutdownToken = _shutdownCts.Token; - while (!shutdownToken.IsCancellationRequested && _isEnabled) + while (!cancellationToken.IsCancellationRequested + && !shutdownToken.IsCancellationRequested + && _isEnabled) { - - // Создаём токен с учётом троттлинга - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - shutdownToken, - _throttleCts?.Token ?? CancellationToken.None); - try { - var item = await _queue.Reader.ReadAsync(linkedCts.Token).ConfigureAwait(false); + WorkItem item = await _queue.Reader.ReadAsync(shutdownToken).ConfigureAwait(false); - // Запускаем выполнение без ожидания - _ = ExecuteItemAsync(item, shutdownToken); - } - catch (OperationCanceledException) when (shutdownToken.IsCancellationRequested) - { - return; + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + shutdownToken, + cancellationToken, + item.CancellationToken); + + await ExecuteItemAsync(item, cts.Token).ConfigureAwait(false); } catch (OperationCanceledException) { - // Троттлинг — перепроверяем условия + // ignore } catch (Exception ex) { - _logger?.LogError(ex, "Reader error"); + _logger?.LogError(ex, "ReaderTask error"); await Task.Delay(100, shutdownToken).ConfigureAwait(false); } } } - private async Task ExecuteItemAsync(WorkItem item, CancellationToken shutdownToken) + private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) { - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - shutdownToken, - item.CancellationToken); - - var ct = cts.Token; - try { ct.ThrowIfCancellationRequested(); @@ -382,13 +410,18 @@ private async Task ExecuteItemAsync(WorkItem item, CancellationToken shutdownTok item.Status = WorkStatus.Running; await OnStatusChangedAsync(item).ConfigureAwait(false); - await _processor.Execute(item.Model, ct).ConfigureAwait(false); + await _processor.Execute(item.Input, ct).ConfigureAwait(false); item.Status = WorkStatus.Completed; } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (OperationCanceledException ex) when (item.CancellationToken.IsCancellationRequested) { item.Status = WorkStatus.Cancelled; + + if (_logger != null && _logger.IsEnabled(LogLevel.Warning)) + { + _logger?.LogWarning(ex, "Item {ItemId} processing was cancelled", item.Id); + } } catch (Exception ex) { @@ -413,8 +446,8 @@ private void ActivateItem(WorkItem item) lock (_rootSync) { - _queueItems.Add(item); - _queueCount = _queueItems.Count; + _workItems.Add(item); + _workItemsCount = _workItems.Count; oldTcs = _idleTcs; _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -429,10 +462,10 @@ private void DeactivateItem(WorkItem item) lock (_rootSync) { - _queueItems.Remove(item); - _queueCount = _queueItems.Count; + _workItems.Remove(item); + _workItemsCount = _workItems.Count; - if (_queueItems.Count == 0 && _queue.Reader.Count == 0) + if (_workItems.Count == 0 && _queue.Reader.Count == 0) { tcsToNotify = _idleTcs; } @@ -450,7 +483,7 @@ private async Task OnStatusChangedAsync(WorkItem item) { try { - await watcher.HandleChanges(item.Model, info, _shutdownCts.Token).ConfigureAwait(false); + await watcher.HandleChanges(item.Input, info, _shutdownCts.Token).ConfigureAwait(false); } catch (Exception ex) { @@ -481,13 +514,9 @@ public void Dispose() _shutdownCts.Cancel(); - var throttle = Interlocked.Exchange(ref _throttleCts, null); - throttle?.Cancel(); - _disposed = true; _shutdownCts.Dispose(); - throttle?.Dispose(); } public async ValueTask DisposeAsync() @@ -499,22 +528,16 @@ public async ValueTask DisposeAsync() await _shutdownCts.CancelAsync().ConfigureAwait(false); - var throttle = Interlocked.Exchange(ref _throttleCts, null); - if (throttle != null) - { - await throttle.CancelAsync().ConfigureAwait(false); - } - try { await _processingTask.ConfigureAwait(false); - Task[] readersCopy; + Task[] readers; lock (_activeReadersSync) { - readersCopy = [.. _activeReaders]; + readers = [.. _activeReaders.Select(c => c.Task)]; } - await Task.WhenAll(readersCopy).ConfigureAwait(false); + await Task.WhenAll(readers).ConfigureAwait(false); } catch (Exception ex) { @@ -524,14 +547,13 @@ public async ValueTask DisposeAsync() { _disposed = true; _shutdownCts.Dispose(); - throttle?.Dispose(); } } - private sealed class WorkItem(TModel item, CancellationToken token) + private sealed class WorkItem(TInput item, CancellationToken token) { public long Id { get; init; } - public TModel Model => item; + public TInput Input => item; public CancellationToken CancellationToken => token; public WorkStatus Status { get; set; } public DateTimeOffset EnqueuedTime { get; init; } @@ -541,72 +563,104 @@ private sealed class WorkItem(TModel item, CancellationToken token) public WorkInfo ToInfo() => new(Id, Status, EnqueuedTime, StartedTime, EndedTime, LastError); } + + internal sealed record ReaderTask(Task Task, CancellationTokenSource TokenSource) : IDisposable + { + public void Dispose() + { + TokenSource.Dispose(); + } + } } -internal sealed record WorkQueueOptions +internal sealed record WorkQueueOptions ( // Required - IWork Processor, + IWork Processor, // Optional with defaults - IWorkObserver? Observer = null, + IWorkObserver? Observer = null, TimeProvider? TimeProvider = null, int? QueueCapacity = null, int? ConcurrencyLimit = null, - int? MaxConcurrencyLimit = null, + int? MaxConcurrency = null, bool? SingleWriter = false, Action? StatusChanged = null, ILogger? Logger = null, BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait ) { - public WorkQueueOptions WithObserver(IWorkObserver observer) => - this with { Observer = observer }; - - public WorkQueueOptions WithTimeProvider(TimeProvider timeProvider) => + public WorkQueueOptions WithTimeProvider(TimeProvider timeProvider) => this with { TimeProvider = timeProvider }; - public WorkQueueOptions WithQueueCapacity(int capacity) + public WorkQueueOptions WithQueueCapacity(int capacity) { ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); return this with { QueueCapacity = capacity }; } - public WorkQueueOptions WithConcurrencyLimit(int limit) + public WorkQueueOptions WithConcurrencyLimit(int limit) { ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); return this with { ConcurrencyLimit = limit }; } - public WorkQueueOptions WithMaxConcurrency(int limit) + public WorkQueueOptions WithMaxConcurrency(int limit) { - ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); - return this with { MaxConcurrencyLimit = limit }; + if (limit < 1) + { + limit = Environment.ProcessorCount; + } + return this with { MaxConcurrency = limit }; } - public WorkQueueOptions WithSingleWriter(bool singleWriter) => + public WorkQueueOptions WithSingleWriter(bool singleWriter) => this with { SingleWriter = singleWriter }; - public WorkQueueOptions WithStatusCallback(Action callback) => + public WorkQueueOptions WithStatusCallback(Action callback) => this with { StatusChanged = callback }; - public WorkQueueOptions WithLogger(ILogger> logger) => + public WorkQueueOptions WithLogger(ILogger> logger) => this with { Logger = logger }; - public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) => + public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) => this with { FullMode = mode }; - public static WorkQueueOptions Create(IWork processor) => + public WorkQueueOptions WithObserver(Func process) => + this with { Observer = new WorkObserverProcessor(process) }; + + public WorkQueueOptions WithObserver(IWorkObserver observer) => + this with { Observer = observer }; + + + // -- ct-or + public static WorkQueueOptions Create(IWork processor) => new(processor); - public static WorkQueueOptions Create(Func process) => - new(new WrapProcessor(process)); + public static WorkQueueOptions Create(Func process) => + new(new WorkProcessor(process)); - sealed class WrapProcessor(Func process) : IWork + + + // -- wrap classes + sealed class WorkProcessor(Func process) : IWork + { + public Task Execute(TInput input, CancellationToken cancellationToken) + => process(input, cancellationToken); + } + + sealed class WorkObserverProcessor(Func process) : IWorkObserver { - public Task Execute(TModel model, CancellationToken cancellationToken) - => process(model, cancellationToken); + public Task HandleChanges(TInput input, WorkInfo work, CancellationToken cancellationToken) + => process(input, work, cancellationToken); } } + + + +public sealed class ShutdownAsyncException : Exception +{ + public static ShutdownAsyncException Default => new(); +} diff --git a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs new file mode 100644 index 00000000..bebc446d --- /dev/null +++ b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs @@ -0,0 +1,518 @@ +using Sa.Classes; +using System.Collections.Concurrent; + +namespace SaTests.Classes; + + +internal sealed class TrackingWork( + Func? executeFunc = null, + Action? onStatusChanged = null) : IWork, IWorkObserver +{ + private readonly Func? _executeFunc = executeFunc; + private readonly Action? _onStatusChanged = onStatusChanged; + + // Thread-safe + private int _maxParallelObserved; + private int _currentParallel; + private long _completedCount; + private long _failedCount; + private long _cancelledCount; + + public int MaxParallelObserved => Volatile.Read(ref _maxParallelObserved); + public long CompletedCount => Interlocked.Read(ref _completedCount); + public long FailedCount => Interlocked.Read(ref _failedCount); + public long CancelledCount => Interlocked.Read(ref _cancelledCount); + public long TotalProcessed => CompletedCount + FailedCount + CancelledCount; + + public async Task Execute(BlockingTaskModel model, CancellationToken cancellationToken) + { + int current = Interlocked.Increment(ref _currentParallel); + try + { + int max = Volatile.Read(ref _maxParallelObserved); + if (current > max) + { + Interlocked.CompareExchange(ref _maxParallelObserved, current, max); + } + + await model.AllowCompletion.Task; + + + if (_executeFunc != null) + await _executeFunc(model, cancellationToken); + else + await Task.Delay(50, cancellationToken); // Default delay + } + finally + { + Interlocked.Decrement(ref _currentParallel); + } + } + + public Task HandleChanges( + BlockingTaskModel model, + WorkInfo work, + CancellationToken cancellationToken) + { + _onStatusChanged?.Invoke(work); + + switch (work.Status) + { + case WorkStatus.Completed: + Interlocked.Increment(ref _completedCount); + break; + case WorkStatus.Faulted: + Interlocked.Increment(ref _failedCount); + break; + case WorkStatus.Cancelled: + Interlocked.Increment(ref _cancelledCount); + break; + } + + return Task.CompletedTask; + } + + public void Reset() + { + Volatile.Write(ref _maxParallelObserved, 0); + Volatile.Write(ref _currentParallel, 0); + Interlocked.Exchange(ref _completedCount, 0); + Interlocked.Exchange(ref _failedCount, 0); + Interlocked.Exchange(ref _cancelledCount, 0); + } +} + +internal sealed record BlockingTaskModel( + TimeSpan? Delay = null, + bool ShouldFail = false, + string? Id = null +) +{ + public TaskCompletionSource AllowCompletion { get; } + = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public BlockingTaskModel Unlock() + { + AllowCompletion.SetResult(); + return this; + } +}; + + +public sealed class WorkQueueConcurrencyTests : IAsyncLifetime +{ + private readonly List _disposables = []; + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public async ValueTask DisposeAsync() + { + foreach (var d in _disposables) + { + if (d is IAsyncDisposable ad) + await ad.DisposeAsync(); + else + d.Dispose(); + } + _disposables.Clear(); + } + + private WorkQueue CreateQueue( + TrackingWork work, + int concurrencyLimit, + int? queueCapacity = null, + int? maxConcurrency = null) + { + var options = new WorkQueueOptions( + Processor: work, + ConcurrencyLimit: concurrencyLimit, + MaxConcurrency: maxConcurrency ?? concurrencyLimit * 2, + QueueCapacity: queueCapacity ?? 100 + ); + + var queue = new WorkQueue(options); + _disposables.Add(queue); + return queue; + } + + + + [Fact] + public async Task ConcurrencyLimit_Respected_AtStartup() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3); + + var blockers = new List(); + + // Act: enqueue 10 tasks that block until we allow them + for (int i = 0; i < 10; i++) + { + var model = new BlockingTaskModel(); + blockers.Add(model.AllowCompletion); + await queue.Enqueue(model, TestContext.Current.CancellationToken); + } + + await Task.Delay(200, TestContext.Current.CancellationToken); + + + Assert.True(work.MaxParallelObserved <= 3, + $"Expected max parallel <= 3, but observed {work.MaxParallelObserved}"); + + Assert.True(work.MaxParallelObserved >= 1, "At least one task should have started"); + + + foreach (var tcs in blockers) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + Assert.Equal(10, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_Zero_StopsProcessing() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 0); + + // Enqueue tasks + for (int i = 0; i < 5; i++) + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromMilliseconds(10)); + await queue.Enqueue(model, TestContext.Current.CancellationToken); + model.Unlock(); // Сразу разрешаем завершение + } + + Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); + + await Task.Delay(300, TestContext.Current.CancellationToken); + + Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); + Assert.Equal(0, work.TotalProcessed); + + // Восстанавливаем лимит и ждём завершения + queue.ConcurrencyLimit = 2; + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + Assert.Equal(5, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_DynamicIncrease_Works() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 2); + + var phase1Blockers = new List(); + var phase2Blockers = new List(); + + // Act Phase 1: enqueue 4 tasks with limit=2 + for (int i = 0; i < 4; i++) + { + var model = new BlockingTaskModel(Id: $"P1-{i}"); + phase1Blockers.Add(model.AllowCompletion); + await queue.Enqueue(model, TestContext.Current.CancellationToken); + } + await Task.Delay(150, TestContext.Current.CancellationToken); + + int maxAtPhase1 = work.MaxParallelObserved; + Assert.True(maxAtPhase1 <= 2, $"Phase 1: expected <=2, got {maxAtPhase1}"); + + // Act Phase 2: increase limit to 4 and enqueue more + queue.ConcurrencyLimit = 4; + + for (int i = 0; i < 4; i++) + { + var model2 = new BlockingTaskModel(Id: $"P2-{i}"); + phase2Blockers.Add(model2.AllowCompletion); + await queue.Enqueue(model2, TestContext.Current.CancellationToken); + } + await Task.Delay(150, TestContext.Current.CancellationToken); + + // Assert: после увеличения лимита должно быть возможно до 4 параллельных + Assert.True(work.MaxParallelObserved <= 4, + $"Expected max parallel <= 4, but observed {work.MaxParallelObserved}"); + Assert.True(work.MaxParallelObserved > maxAtPhase1, + "Max parallel should have increased after raising limit"); + + // Завершаем все задачи + foreach (var tcs in phase1Blockers.Concat(phase2Blockers)) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + Assert.Equal(8, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_DynamicDecrease_ThrottlesReaders() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 4); + + var blockers = new List(); + var startedEvents = new List(); + + // Act: enqueue 8 tasks + for (int i = 0; i < 8; i++) + { + var model = new BlockingTaskModel(Id: $"T-{i}"); + var startedTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + blockers.Add(model.AllowCompletion); + startedEvents.Add(startedTcs); + + await queue.Enqueue(model, TestContext.Current.CancellationToken); + } + + + for (int i = 0; i < 4; i++) + blockers[i].SetResult(); + + await Task.Delay(200, TestContext.Current.CancellationToken); + int beforeDecrease = work.MaxParallelObserved; + Assert.True(beforeDecrease <= 4, $"Before decrease: expected <=4, got {beforeDecrease}"); + + Assert.Equal(4, queue.QueueTasks); + + // Act: decrease limit to 2 + queue.ConcurrencyLimit = 2; + await Task.Delay(200, TestContext.Current.CancellationToken); + + Assert.True(work.MaxParallelObserved <= 4, + $"After decrease: observed spike to {work.MaxParallelObserved}"); + + // Завершаем остальные + for (int i = 4; i < 8; i++) + blockers[i].SetResult(); + + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + Assert.Equal(8, work.TotalProcessed); + Assert.Equal(0, queue.QueueTasks); + } + + + + [Theory] + [InlineData(1, 10)] + [InlineData(5, 25)] + [InlineData(10, 50)] + public async Task ConcurrencyLimit_NeverExceeded_UnderLoad(int limit, int taskCount) + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: limit, maxConcurrency: limit * 2); + + var blockers = new List(); + + // Act: rapid enqueue + var enqueueTasks = Enumerable.Range(0, taskCount).Select(async i => + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromMilliseconds(20)); + blockers.Add(model.AllowCompletion); + await queue.Enqueue(model); + }); + + await Task.WhenAll(enqueueTasks); + + await Task.Delay(300, TestContext.Current.CancellationToken); + + Assert.True(work.MaxParallelObserved <= limit); + + foreach (var tcs in blockers) + tcs.SetResult(); + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + Assert.Equal(taskCount, work.TotalProcessed); + } + + [Fact] + public async Task ConcurrencyLimit_RapidChanges_DoesNotBreak() + { + int totals = 30; + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3, maxConcurrency: 6, queueCapacity: totals); + + using var cts = new CancellationTokenSource(); + var blockers = new ConcurrentQueue(); + + // Background enqueuer + var enqueueTask = Task.Run(async () => + { + int i = 0; + while (!cts.Token.IsCancellationRequested && i < totals) + { + var model = new BlockingTaskModel(Id: $"T-{i++}"); + blockers.Enqueue(model.AllowCompletion); + await queue.Enqueue(model, cts.Token); + await Task.Delay(20, cts.Token); + } + }, cts.Token); + + // Limiter changer + var changeTask = Task.Run(async () => + { + int[] limits = [2, 5, 1, 4, 3, 6, 2]; + foreach (var limit in limits) + { + if (cts.Token.IsCancellationRequested) break; + queue.ConcurrencyLimit = limit; + await Task.Delay(100, cts.Token); + } + // cts.Cancel(); // Stop enqueuer + }, cts.Token); + + // Processor: complete tasks with small delay + var processTask = Task.Run(async () => + { + + int i = 0; + + try + { + while (!cts.Token.IsCancellationRequested || !blockers.IsEmpty) + { + if (blockers.TryDequeue(out var tcs)) + { + i++; + await Task.Delay(10); + + tcs.TrySetResult(); + } + else + { + await Task.Delay(100, cts.Token); + if (i >= totals) + { + return; + } + } + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + }, cts.Token); + + await Task.WhenAll(enqueueTask, changeTask, processTask); + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + // Assert: no exceptions, all tasks processed, limit respected at peaks + Assert.True(work.MaxParallelObserved <= 6, + $"Max observed {work.MaxParallelObserved} exceeded max configured limit 6"); + + Assert.Equal(totals, work.TotalProcessed); // All accounted for + } + + [Fact] + public async Task ConcurrencyLimit_SetToMaxAllowed_ClampsCorrectly() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 2, maxConcurrency: 5); + + // Act & Assert + queue.ConcurrencyLimit = 10; // Above max + Assert.Equal(5, queue.ConcurrencyLimit); // Should be clamped + + queue.ConcurrencyLimit = -1; // Below min + Assert.Equal(0, queue.ConcurrencyLimit); // Should be clamped to 0 + + queue.ConcurrencyLimit = 3; // Valid + Assert.Equal(3, queue.ConcurrencyLimit); + } + + // ───────────────────────────────────────────────────────────── + // 🔹 Тесты на корректность обработки после изменений лимита + // ───────────────────────────────────────────────────────────── + + [Fact] + public async Task ConcurrencyLimit_Changes_DontLoseTasks() + { + // Arrange + var completedIds = new ConcurrentBag(); + var work = new TrackingWork( + executeFunc: async (model, ct) => + { + await Task.Delay(10, ct); + if (!string.IsNullOrEmpty(model.Id)) + completedIds.Add(model.Id); + }); + + var queue = CreateQueue(work, concurrencyLimit: 2); + + + List models = []; + + // Act: enqueue with interleaved limit changes + var tasks = new List(); + for (int i = 0; i < 20; i++) + { + if (i == 5) queue.ConcurrencyLimit = 4; + if (i == 10) queue.ConcurrencyLimit = 1; + if (i == 15) queue.ConcurrencyLimit = 3; + + var model = new BlockingTaskModel(Id: $"T-{i:D3}"); + models.Add(model); + var t = queue.Enqueue(model, TestContext.Current.CancellationToken).AsTask(); + + tasks.Add(t); + } + + await Task.WhenAll(tasks); + + foreach (var model in models) model.Unlock(); + + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + // Assert: all tasks completed, no duplicates, no losses + Assert.Equal(20, completedIds.Count); + Assert.Equal(20, completedIds.Distinct().Count()); + Assert.All(Enumerable.Range(0, 20), i => + Assert.Contains($"T-{i:D3}", completedIds)); + } + + [Fact] + public async Task ConcurrencyLimit_WithCancellation_HandlesGracefully() + { + // Arrange + var work = new TrackingWork(); + var queue = CreateQueue(work, concurrencyLimit: 3); + + using var cts = new CancellationTokenSource(); + + // Act: enqueue cancellable tasks + var enqueueTasks = Enumerable.Range(0, 10).Select(i => + { + var model = new BlockingTaskModel(Delay: TimeSpan.FromSeconds(20)); + model.Unlock(); + return queue.Enqueue(model, cts.Token).AsTask(); + }); + + await Task.WhenAll(enqueueTasks); + + await Task.Delay(100, TestContext.Current.CancellationToken); + + await cts.CancelAsync(); + + // Change limit during cancellation + queue.ConcurrencyLimit = 1; + queue.ConcurrencyLimit = 5; + + // Shutdown should complete without hanging + await queue.ShutdownAsync(); + + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + Assert.True(work.TotalProcessed > 0); + } +} + diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index 3e000294..7de6d35c 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -68,7 +68,7 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() // Assert Assert.True(model.WasProcessed); - Assert.Equal(0, queue.QueueCount); + Assert.Equal(0, queue.QueueTasks); } @@ -90,7 +90,7 @@ public async Task Enqueue_WaitForIdle_Completed() // Assert Assert.True(model.WasProcessed); - Assert.Equal(0, queue.QueueCount); + Assert.Equal(0, queue.QueueTasks); } [Fact] @@ -112,7 +112,7 @@ public async Task Enqueue_MultipleTasks_AllExecuted() // Assert Assert.All(models, m => Assert.True(m.WasProcessed)); - Assert.Equal(0, queue.QueueCount); + Assert.Equal(0, queue.QueueTasks); } [Fact] @@ -143,7 +143,7 @@ public async Task ConcurrencyLimit_Respected() await queue.WaitForIdleAsync(cancellationToken: TestToken); - Assert.Equal(0, queue.QueueCount); + Assert.Equal(0, queue.QueueTasks); } [Fact] @@ -214,7 +214,7 @@ public async Task ShutdownAsync_StopsProcessing() // Assert Assert.False(queue.IsEnabled); - Assert.True(queue.QueueCount <= 5); + Assert.True(queue.QueueTasks <= 5); await Assert.ThrowsAsync(async () => { @@ -222,9 +222,9 @@ await Assert.ThrowsAsync(async () => }); await queue.WaitForIdleAsync(TestToken); - Assert.Equal(0, queue.QueueCount); + Assert.Equal(0, queue.QueueTasks); - var canceled = observer.Changes.Count(c => c.Status == WorkStatus.Cancelled); + var canceled = observer.Changes.Count(c => c.LastError is OperationCanceledException); Assert.Equal(5, canceled); } @@ -297,12 +297,15 @@ await Assert.ThrowsAsync(() } [Fact] - public void ConcurrencyLimit_InvalidValue_Throws() + public void ConcurrencyLimit_InvalidValue_Clamp() { var queue = new WorkQueue( - WorkQueueOptions.Create(new TestWork())); + WorkQueueOptions.Create(new TestWork())) + { + ConcurrencyLimit = -1 + }; - Assert.Throws(() => queue.ConcurrencyLimit = -1); + Assert.Equal(0, queue.ConcurrencyLimit); queue.Dispose(); } From e3ec12057e353f6be78b42712072450152744043 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 17 Apr 2026 11:33:11 +0300 Subject: [PATCH 22/28] tests --- src/Sa.Schedule/Engine/IJobController.cs | 2 +- src/Sa.Schedule/Engine/JobController.cs | 2 +- src/Sa.Schedule/Engine/JobRunner.cs | 2 +- src/Sa.Schedule/Engine/JobScheduler.cs | 4 +- src/Sa/Classes/WorkQueue.cs | 105 ++++++++---------- .../Sa.ScheduleTests/JobSchedulerTests.cs | 2 +- .../ScheduleConcurrencyTests.cs | 2 +- .../Classes/WorkQueueConcurrencyTests.cs | 15 +-- 8 files changed, 58 insertions(+), 76 deletions(-) diff --git a/src/Sa.Schedule/Engine/IJobController.cs b/src/Sa.Schedule/Engine/IJobController.cs index 5d4723e3..64b86721 100644 --- a/src/Sa.Schedule/Engine/IJobController.cs +++ b/src/Sa.Schedule/Engine/IJobController.cs @@ -19,7 +19,7 @@ internal interface IJobController void Init(); - void Free(); + void Finish(); diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index fb4c6997..25c0a92d 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -92,7 +92,7 @@ public void Init() _context.NumRuns++; } - public void Free() + public void Finish() { _context.ServiceProvider = NullJobServices.Instance; _job?.Dispose(); diff --git a/src/Sa.Schedule/Engine/JobRunner.cs b/src/Sa.Schedule/Engine/JobRunner.cs index f6438ebd..6f9d7a55 100644 --- a/src/Sa.Schedule/Engine/JobRunner.cs +++ b/src/Sa.Schedule/Engine/JobRunner.cs @@ -17,7 +17,7 @@ public async Task Run(IJobController controller, CancellationToken cancellationT } finally { - controller.Free(); + controller.Finish(); } } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index d549622d..33f470c7 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -88,9 +88,7 @@ public IChangeToken StartChangeToken() } } - /// - /// Start all jobs - /// + public async Task Start(CancellationToken cancellationToken) { CancellationToken stoppingToken; diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 144449a0..5180e2f1 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -207,49 +207,6 @@ public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken ActivateItem(item); } - public async Task ShutdownAsync() - { - lock (_rootSync) - { - if (!_isEnabled || _disposed) return; - _isEnabled = false; - } - - await _shutdownCts.CancelAsync().ConfigureAwait(false); - - try - { - await _processingTask.ConfigureAwait(false); - - Task[] readers; - lock (_activeReadersSync) - { - readers = [.. _activeReaders.Select(c => c.Task)]; - } - await Task.WhenAll(readers).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Error during shutdown"); - } - finally - { - await ClearAll().ConfigureAwait(false); - } - } - - private async Task ClearAll() - { - while (_queue.Reader.TryRead(out var input)) - { - input.Status = WorkStatus.Faulted; - input.LastError = ShutdownAsyncException.Default; - input.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(input).ConfigureAwait(false); - DeactivateItem(input); - } - } - public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); @@ -505,6 +462,49 @@ private async Task OnStatusChangedAsync(WorkItem item) } } + public async Task ShutdownAsync() + { + lock (_rootSync) + { + if (!_isEnabled || _disposed) return; + _isEnabled = false; + } + + await _shutdownCts.CancelAsync().ConfigureAwait(false); + + try + { + await _processingTask.ConfigureAwait(false); + + Task[] readers; + lock (_activeReadersSync) + { + readers = [.. _activeReaders.Select(c => c.Task)]; + } + await Task.WhenAll(readers).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger?.LogError(ex, "Error during shutdown"); + } + finally + { + await ClearAll().ConfigureAwait(false); + } + } + + private async Task ClearAll() + { + while (_queue.Reader.TryRead(out var input)) + { + input.Status = WorkStatus.Faulted; + input.LastError = ShutdownAsyncException.Default; + input.EndedTime = _timeProvider.GetUtcNow(); + await OnStatusChangedAsync(input).ConfigureAwait(false); + DeactivateItem(input); + } + } + public void Dispose() { if (_disposed) return; @@ -523,25 +523,12 @@ public async ValueTask DisposeAsync() { if (_disposed) return; - _isEnabled = false; - _queue.Writer.TryComplete(); - - await _shutdownCts.CancelAsync().ConfigureAwait(false); - try { - await _processingTask.ConfigureAwait(false); + _isEnabled = false; + _queue.Writer.TryComplete(); + await ShutdownAsync().ConfigureAwait(false); - Task[] readers; - lock (_activeReadersSync) - { - readers = [.. _activeReaders.Select(c => c.Task)]; - } - await Task.WhenAll(readers).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Error during synchronous disposal"); } finally { diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs index 20c183aa..ce9d67c4 100644 --- a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -148,7 +148,7 @@ public void Init() // } - public void Free() + public void Finish() { // } diff --git a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs index 730852e1..257a4ef2 100644 --- a/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs +++ b/src/Tests/Sa.ScheduleTests/ScheduleConcurrencyTests.cs @@ -91,7 +91,7 @@ public async Task Check_ExecuteCounterJob() j.ConcurrencyLimit = 10; - await Task.Delay(100, TestContext.Current.CancellationToken); + await Task.Delay(250, TestContext.Current.CancellationToken); Assert.Equal(10, Fixture.Count); diff --git a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs index bebc446d..7bc37507 100644 --- a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs @@ -332,7 +332,7 @@ public async Task ConcurrencyLimit_NeverExceeded_UnderLoad(int limit, int taskCo } [Fact] - public async Task ConcurrencyLimit_RapidChanges_DoesNotBreak() + public async Task ConcurrencyLimitRapidChangesDoesNotBreak() { int totals = 30; // Arrange @@ -429,12 +429,9 @@ public async Task ConcurrencyLimit_SetToMaxAllowed_ClampsCorrectly() Assert.Equal(3, queue.ConcurrencyLimit); } - // ───────────────────────────────────────────────────────────── - // 🔹 Тесты на корректность обработки после изменений лимита - // ───────────────────────────────────────────────────────────── - + [Fact] - public async Task ConcurrencyLimit_Changes_DontLoseTasks() + public async Task ConcurrencyLimit_UpChanges_DontLoseTasks() { // Arrange var completedIds = new ConcurrentBag(); @@ -455,9 +452,9 @@ public async Task ConcurrencyLimit_Changes_DontLoseTasks() var tasks = new List(); for (int i = 0; i < 20; i++) { - if (i == 5) queue.ConcurrencyLimit = 4; - if (i == 10) queue.ConcurrencyLimit = 1; - if (i == 15) queue.ConcurrencyLimit = 3; + if (i == 5) queue.ConcurrencyLimit = 3; + if (i == 10) queue.ConcurrencyLimit = 4; + if (i == 15) queue.ConcurrencyLimit = 5; var model = new BlockingTaskModel(Id: $"T-{i:D3}"); models.Add(model); From 400c6a2604e36aff577fdd46738df43a93ce758c Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 17 Apr 2026 16:00:29 +0300 Subject: [PATCH 23/28] little fix --- src/Sa.Outbox/Delivery/ConsumeSettings.cs | 7 ++++++- src/Sa.Outbox/Delivery/Job/Setup.cs | 2 ++ src/Sa.Outbox/Delivery/ScheduleSettings.cs | 7 ++++++- src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs | 12 ++++++++++++ src/Sa.Schedule/Engine/JobScheduler.cs | 6 ++++-- src/Sa.Schedule/IJobBuilder.cs | 1 - src/Sa.Schedule/IJobProperties.cs | 11 +++++++++-- src/Sa.Schedule/Settings/JobBuilder.cs | 6 ------ src/Sa.Schedule/Settings/JobProperies.cs | 9 +-------- src/Sa.Schedule/Setup.cs | 4 +++- src/Sa/Classes/WorkQueue.cs | 2 +- 11 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/Sa.Outbox/Delivery/ConsumeSettings.cs b/src/Sa.Outbox/Delivery/ConsumeSettings.cs index 07bc6732..d64f9fbf 100644 --- a/src/Sa.Outbox/Delivery/ConsumeSettings.cs +++ b/src/Sa.Outbox/Delivery/ConsumeSettings.cs @@ -11,8 +11,13 @@ public sealed class ConsumeSettings { /// /// Maximum number of processing iterations when greedy mode is enabled. - /// -1 means unlimited iterations. + /// -1 means unlimited iterations (greedy mode). /// + /// + /// Consumes data in chunks of size ConsumeBatchSize repeatedly, up to a total of MaxBatchSize. + /// The MaxProcessingIterations limits the number of these consumption cycles. + /// When set to -1 (greedy mode), there is no iteration limit — the system will continue consuming batches until it reaches MaxBatchSize or runs out of data. + /// public int MaxProcessingIterations { get; set; } = 10; /// diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index 34e69834..a6d01539 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -42,6 +42,8 @@ public static IServiceCollection AddDeliveryJob< .EveryTime(scheduleSettings.Interval) .WithInitialDelay(scheduleSettings.InitialDelay) .WithTag(settings) + .WithConcurrencyLimit(scheduleSettings.ConcurrencyLimit) + .WithMaxConcurrency(scheduleSettings.MaxConcurrency) .WithName(scheduleSettings.Name ?? typeof(TConsumer).Name) .ConfigureErrorHandling(c => c .IfErrorRetry(scheduleSettings.RetryCountOnError) diff --git a/src/Sa.Outbox/Delivery/ScheduleSettings.cs b/src/Sa.Outbox/Delivery/ScheduleSettings.cs index 9a704924..792426c7 100644 --- a/src/Sa.Outbox/Delivery/ScheduleSettings.cs +++ b/src/Sa.Outbox/Delivery/ScheduleSettings.cs @@ -19,5 +19,10 @@ public sealed class ScheduleSettings /// public TimeSpan InitialDelay { get; internal set; } = TimeSpan.FromSeconds(10); - public int RetryCountOnError { get; internal set; } = 2; + public int RetryCountOnError { get; internal set; } = 1; + + public int ConcurrencyLimit {get; internal set; } = 1; + + public int MaxConcurrency {get; internal set; } = 48; + } diff --git a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs b/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs index 50353fe3..f5543753 100644 --- a/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs +++ b/src/Sa.Outbox/Delivery/ScheduleSettingsExtensions.cs @@ -39,6 +39,17 @@ public static ScheduleSettings WithImmediate(this ScheduleSettings settings) return settings; } + public static ScheduleSettings WithConcurrencyLimit(this ScheduleSettings settings, int limit) + { + settings.ConcurrencyLimit = limit; + return settings; + } + public static ScheduleSettings WithMaxConcurrency(this ScheduleSettings settings, int maxLimit) + { + settings.MaxConcurrency = maxLimit; + return settings; + } + /// /// Sets the number of retry attempts on error. /// @@ -92,4 +103,5 @@ public static ScheduleSettings WithInitialDelaySeconds(this ScheduleSettings set public static ScheduleSettings WithInitialDelayMinutes(this ScheduleSettings settings, int minutes) => settings.WithInitialDelay(TimeSpan.FromMinutes(minutes)); + } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 33f470c7..1df784d5 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -37,8 +37,10 @@ public JobScheduler( JobId = settings.JobId; - var concurrency = settings.Properties.ConcurrencyLimit ?? 1; - var maxConcurrency = settings.Properties.MaxConcurrency ?? concurrency; + var concurrency = Math.Max(0, settings.Properties.ConcurrencyLimit ?? 1); + var maxConcurrency = Math.Max(1, settings.Properties.MaxConcurrency ?? concurrency); + + concurrency = Math.Clamp(concurrency, 0, maxConcurrency); _jobs = new(WorkQueueOptions.Create(CreateJob) .WithQueueCapacity(maxConcurrency) diff --git a/src/Sa.Schedule/IJobBuilder.cs b/src/Sa.Schedule/IJobBuilder.cs index 413ac716..4543323e 100644 --- a/src/Sa.Schedule/IJobBuilder.cs +++ b/src/Sa.Schedule/IJobBuilder.cs @@ -81,7 +81,6 @@ IJobBuilder EveryMinutes(int minutes = 1) IJobBuilder WithMaxConcurrency(int limit); - IJobBuilder WithSingleWriter(bool singleWriter); /// /// Merges the specified job properties into the current job configuration. diff --git a/src/Sa.Schedule/IJobProperties.cs b/src/Sa.Schedule/IJobProperties.cs index a894ba50..cd45c526 100644 --- a/src/Sa.Schedule/IJobProperties.cs +++ b/src/Sa.Schedule/IJobProperties.cs @@ -45,9 +45,16 @@ public interface IJobProperties /// object? Tag { get; } + /// + /// Gets the current concurrency limit for the job when it starts. + /// This value cannot exceed . + /// Null means use the default limit = 1. + /// int? ConcurrencyLimit { get; } + /// + /// Gets the absolute maximum concurrency limit allowed for this job. + /// Null means = ConcurrencyLimit. + /// int? MaxConcurrency { get; } - - bool? SingleWriter { get; } } diff --git a/src/Sa.Schedule/Settings/JobBuilder.cs b/src/Sa.Schedule/Settings/JobBuilder.cs index ac956fdb..fcce2385 100644 --- a/src/Sa.Schedule/Settings/JobBuilder.cs +++ b/src/Sa.Schedule/Settings/JobBuilder.cs @@ -74,12 +74,6 @@ public IJobBuilder WithMaxConcurrency(int limit) return this; } - public IJobBuilder WithSingleWriter(bool singleWriter) - { - settings.Properties.WithSingleWriter(singleWriter); - return this; - } - public IJobBuilder Disabled() { settings.Properties.SetDisabled(); diff --git a/src/Sa.Schedule/Settings/JobProperies.cs b/src/Sa.Schedule/Settings/JobProperies.cs index 658615d6..6c483650 100644 --- a/src/Sa.Schedule/Settings/JobProperies.cs +++ b/src/Sa.Schedule/Settings/JobProperies.cs @@ -14,7 +14,6 @@ internal sealed class JobProperies : IJobProperties public int? ContextStackSize { get; private set; } public int? ConcurrencyLimit { get; private set; } public int? MaxConcurrency { get; private set; } - public bool? SingleWriter { get; private set; } public JobProperies WithName(string name) { @@ -79,16 +78,11 @@ public JobProperies WithConcurrencyLimit(int limit) public JobProperies WithMaxConcurrencyLimit(int limit) { - ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); MaxConcurrency = limit; return this; } - public JobProperies WithSingleWriter(bool singleWriter) - { - SingleWriter = singleWriter; - return this; - } internal JobProperies Merge(IJobProperties props) { @@ -103,7 +97,6 @@ internal JobProperies Merge(IJobProperties props) ConcurrencyLimit ??= props.ConcurrencyLimit; MaxConcurrency ??= props.MaxConcurrency; - SingleWriter ??= props.SingleWriter; return this; } diff --git a/src/Sa.Schedule/Setup.cs b/src/Sa.Schedule/Setup.cs index 4c378176..0613c46f 100644 --- a/src/Sa.Schedule/Setup.cs +++ b/src/Sa.Schedule/Setup.cs @@ -16,7 +16,9 @@ public static class Setup /// The service collection to add the scheduling system to. /// An action to configure the scheduling system. /// The service collection with the scheduling system added. - public static IServiceCollection AddSaSchedule(this IServiceCollection services, Action configure) + public static IServiceCollection AddSaSchedule( + this IServiceCollection services, + Action configure) { services.TryAddSingleton(TimeProvider.System); services.TryAddSingleton(); diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 5180e2f1..6cc8b9d8 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -226,7 +226,7 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default await Task.WhenAny( tcs.Task.WaitAsync(cts.Token), - Task.Delay(1000, cts.Token)).ConfigureAwait(false); + Task.Delay(3000, cts.Token)).ConfigureAwait(false); } } From d2a8b392789beb34dc2a3ea5a8a765316f17c8cb Mon Sep 17 00:00:00 2001 From: dundich Date: Wed, 22 Apr 2026 10:13:37 +0300 Subject: [PATCH 24/28] IDeliveryScheduleProvider --- .../Delivery/Job/DeliveryScheduleProvider.cs | 9 +++++++++ .../Delivery/Job/IDeliveryScheduleProvider.cs | 11 +++++++++++ src/Sa.Outbox/Delivery/Job/Setup.cs | 3 +++ src/Sa.Outbox/Delivery/ScheduleSettings.cs | 4 ++-- 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs create mode 100644 src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs diff --git a/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs b/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs new file mode 100644 index 00000000..9e1fe3b4 --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/DeliveryScheduleProvider.cs @@ -0,0 +1,9 @@ +using Sa.Schedule; + +namespace Sa.Outbox.Delivery.Job; + +internal sealed class DeliveryScheduleProvider(IScheduler scheduler) : IDeliveryScheduleProvider +{ + public IJobScheduler GetJob(Guid jobId) + => scheduler.GetSchedule(jobId) ?? throw new InvalidOperationException(); +} diff --git a/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs new file mode 100644 index 00000000..28b27938 --- /dev/null +++ b/src/Sa.Outbox/Delivery/Job/IDeliveryScheduleProvider.cs @@ -0,0 +1,11 @@ +using Sa.Schedule; + +namespace Sa.Outbox.Delivery.Job; + +public interface IDeliveryScheduleProvider +{ + IJobScheduler GetJob(Guid jobId); + + int GetInstanceCount(Guid jobId) => GetJob(jobId).ConcurrencyLimit; + void SetInstanceCount(Guid jobId, int count) => GetJob(jobId).ConcurrencyLimit = count; +} diff --git a/src/Sa.Outbox/Delivery/Job/Setup.cs b/src/Sa.Outbox/Delivery/Job/Setup.cs index a6d01539..098e650a 100644 --- a/src/Sa.Outbox/Delivery/Job/Setup.cs +++ b/src/Sa.Outbox/Delivery/Job/Setup.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Sa.Schedule; using System.Diagnostics.CodeAnalysis; @@ -56,6 +57,8 @@ public static IServiceCollection AddDeliveryJob< builder.AddInterceptor(); }); + services.TryAddSingleton(); + return services; } } diff --git a/src/Sa.Outbox/Delivery/ScheduleSettings.cs b/src/Sa.Outbox/Delivery/ScheduleSettings.cs index 792426c7..a049e1f6 100644 --- a/src/Sa.Outbox/Delivery/ScheduleSettings.cs +++ b/src/Sa.Outbox/Delivery/ScheduleSettings.cs @@ -21,8 +21,8 @@ public sealed class ScheduleSettings public int RetryCountOnError { get; internal set; } = 1; - public int ConcurrencyLimit {get; internal set; } = 1; + public int ConcurrencyLimit { get; internal set; } = 1; - public int MaxConcurrency {get; internal set; } = 48; + public int MaxConcurrency { get; internal set; } = 48; } From d3bec010b3a0dbb60bc5f4dcf4609071fea518ff Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 24 Apr 2026 15:23:03 +0300 Subject: [PATCH 25/28] ~ IWorkQueue --- src/Sa.Schedule/Engine/JobFactory.cs | 5 +- src/Sa/Classes/WorkQueue.cs | 656 +++++++----------- .../Classes/WorkQueueConcurrencyTests.cs | 18 +- src/Tests/SaTests/Classes/WorkQueueTests.cs | 90 +-- 4 files changed, 297 insertions(+), 472 deletions(-) diff --git a/src/Sa.Schedule/Engine/JobFactory.cs b/src/Sa.Schedule/Engine/JobFactory.cs index 8b6cde32..0dd03eb2 100644 --- a/src/Sa.Schedule/Engine/JobFactory.cs +++ b/src/Sa.Schedule/Engine/JobFactory.cs @@ -10,7 +10,10 @@ internal sealed class JobFactory( TimeProvider? timeProvider = null) : IJobFactory { public IJobScheduler CreateJobSchedule(IJobSettings settings) - => new JobScheduler(settings, jobRunner, i => CreateController(i, settings)); + => new JobScheduler( + settings, + jobRunner, + i => CreateController(i, settings)); private JobController CreateController(int index, IJobSettings settings) { diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 6cc8b9d8..6cc1e2c2 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -3,7 +3,6 @@ namespace Sa.Classes; - /// /// Интерфейс для выполнения бизнес-логики. /// @@ -12,32 +11,8 @@ internal interface IWork Task Execute(TInput input, CancellationToken cancellationToken); } -/// -/// Интерфейс для наблюдения за изменениями состояния задач в очереди. -/// -internal interface IWorkObserver -{ - Task HandleChanges(TInput input, WorkInfo work, CancellationToken cancellationToken); -} - -/// -/// Представляет состояние и метаданные задачи в очереди. -/// -internal sealed record WorkInfo( - long Id, - WorkStatus Status, - DateTimeOffset EnqueuedTime, - DateTimeOffset? StartedTime = null, - DateTimeOffset? EndedTime = null, - Exception? LastError = null -) -{ - public bool IsEmpty => Id == 0; -} - internal enum WorkStatus { - Queued, Running, Completed, Faulted, @@ -49,103 +24,69 @@ internal enum WorkStatus /// internal interface IWorkQueue : IDisposable, IAsyncDisposable { - /// - /// Показывает, включена ли обработка задач. - /// bool IsEnabled { get; } - - /// - /// Gets the current inputs from queue. - /// int QueueTasks { get; } - - - /// - /// Определяет, свободна ли очередь (нет активных и нет в очереди). - /// bool IsIdle(); - - /// - /// Максимальное количество параллельных задач (динамически изменяется). - /// int ConcurrencyLimit { get; set; } + int MaxConcurrency { get; } + int QueueCapacity { get; } - - public int MaxConcurrency { get; } - - - public int QueueCapacity { get; } - - /// - /// Добавляет задачу в очередь асинхронно. - /// ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default); - - /// - /// Блокируется до момента, когда очередь станет полностью пустой. - /// Task WaitForIdleAsync(CancellationToken cancellationToken = default); - - /// - /// Завершает обработку и отменяет ожидание новых задач. - /// Task ShutdownAsync(); } -internal sealed class WorkQueue : IWorkQueue -{ - private readonly Channel _queue; - private readonly HashSet _workItems = []; - private readonly Lock _rootSync = new(); - private readonly Lock _activeReadersSync = new(); +internal record WorkItem(TInput Input, CancellationToken CancellationToken); - private readonly ILogger? _logger = null; - private readonly TimeProvider _timeProvider; - public readonly IWork _processor; - public readonly IWorkObserver? _watcher; - private readonly Action? _statusChanged; - private readonly CancellationTokenSource _shutdownCts = new(); - private readonly Task _processingTask; +internal sealed partial class WorkQueue : IWorkQueue +{ + private readonly Channel> _queue; + private readonly Lock _rootSync = new(); + private readonly Lock _readersSync = new(); - private volatile TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - private volatile bool _isEnabled = true; - private volatile bool _disposed = false; + private readonly ILogger>? _logger; + private readonly IWork _processor; - private int _concurrency; + private readonly Action? _statusChanged; + + private readonly CancellationTokenSource _shutdownCts = new(); private readonly int _maxConcurrency; private readonly int _queueCapacity; - private int _workItemsCount; - private long _taskIdCounter; - private readonly List _activeReaders = []; + private volatile int _concurrency; + private volatile bool _isEnabled = true; + /// + /// 0: Active, 1: ShuttingDown, 2: Shutdown, 3: Disposed + /// + private int _state; + + // Lock-free счётчик активных + ожидающих задач + private volatile int _pendingAndActiveCount; + private TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private readonly List _activeReaders = []; - public WorkQueue(WorkQueueOptions options) + public WorkQueue(WorkQueueOptions options, ILogger>? logger = null) { ArgumentNullException.ThrowIfNull(options.Processor); - _processor = options.Processor; - _watcher = options.Observer ?? _processor as IWorkObserver; - _timeProvider = options.TimeProvider ?? TimeProvider.System; - _statusChanged = options.StatusChanged; - _logger = options.Logger; + _logger = logger; - _maxConcurrency = options.MaxConcurrency > 0 - ? options.MaxConcurrency.Value - : Environment.ProcessorCount; + _processor = options.Processor; - _concurrency = Math.Clamp( - options.ConcurrencyLimit ?? Environment.ProcessorCount, - 0, - _maxConcurrency); + _statusChanged = options.StatusChanged; + _maxConcurrency = options.MaxConcurrency > 0 ? options.MaxConcurrency.Value : Environment.ProcessorCount; + _concurrency = Math.Clamp(options.ConcurrencyLimit ?? Environment.ProcessorCount, 0, _maxConcurrency); _queueCapacity = options.QueueCapacity ?? _maxConcurrency; - _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) + + _queue = Channel.CreateBounded>(new BoundedChannelOptions(_queueCapacity) { AllowSynchronousContinuations = false, SingleReader = false, @@ -153,63 +94,60 @@ public WorkQueue(WorkQueueOptions options) FullMode = options.FullMode }); - _processingTask = ProcessQueueAsync(); + // Запускаем читателей сразу + SpawnReaders(_concurrency); } public bool IsEnabled => _isEnabled; - public int QueueTasks => _workItemsCount; - + public int QueueTasks => _pendingAndActiveCount; public bool IsIdle() { lock (_rootSync) { - return _workItems.Count == 0 && _queue.Reader.Count == 0; + return _pendingAndActiveCount <= 0 && _queue.Reader.Count == 0; } } + public int MaxConcurrency => _maxConcurrency; + + public int QueueCapacity => _queueCapacity; + public int ConcurrencyLimit { get => _concurrency; set { - ObjectDisposedException.ThrowIf(_disposed, this); - + ObjectDisposedException.ThrowIf(_state == 2, this); var newLimit = Math.Clamp(value, 0, _maxConcurrency); - Interlocked.Exchange(ref _concurrency, newLimit); + var oldLimit = Interlocked.Exchange(ref _concurrency, newLimit); + if (newLimit != oldLimit) AdjustReaders(newLimit, oldLimit); } } - - public int MaxConcurrency => _maxConcurrency; - - public int QueueCapacity => _queueCapacity; - - - public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); - if (!_isEnabled) - throw new InvalidOperationException("Queue has been stopped."); + ObjectDisposedException.ThrowIf(_state == 3, this); - var item = new WorkItem(input, cancellationToken) - { - Id = Interlocked.Increment(ref _taskIdCounter), - Status = WorkStatus.Queued, - EnqueuedTime = _timeProvider.GetUtcNow(), - }; + if (!_isEnabled) ThrowHelper.QueueStopped(); - await OnStatusChangedAsync(item).ConfigureAwait(false); - await _queue.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false); + MarkActive(); - ActivateItem(item); + try + { + await _queue.Writer.WriteAsync(new(input, cancellationToken), cancellationToken).ConfigureAwait(false); + } + catch + { + MarkInactive(); // Откат счётчика при ошибке записи + throw; + } } public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_disposed, this); + ObjectDisposedException.ThrowIf(_state == 3, this); using var cts = CancellationTokenSource.CreateLinkedTokenSource( _shutdownCts.Token, @@ -218,369 +156,319 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default while (!IsIdle()) { TaskCompletionSource tcs; - lock (_rootSync) - { - if (IsIdle()) break; - tcs = _idleTcs; - } + lock (_rootSync) { tcs = _idleTcs; } - await Task.WhenAny( - tcs.Task.WaitAsync(cts.Token), - Task.Delay(3000, cts.Token)).ConfigureAwait(false); + await tcs.Task.WaitAsync(cts.Token).ConfigureAwait(false); } } - /// - /// init readers - /// - private async Task ProcessQueueAsync() - { - try - { - SpawnAdditionalReaders(_concurrency); - await MonitorQueueAsync().ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // Normal shutdown - } - catch (Exception ex) - { - _logger?.LogError(ex, "Queue processing failed"); - } - } + #region Reader Management - private async Task MonitorQueueAsync() + private void SpawnReaders(int count) { - var shutdownToken = _shutdownCts.Token; - - while (!shutdownToken.IsCancellationRequested) + lock (_readersSync) { - if (await _queue.Reader.WaitToReadAsync(shutdownToken).ConfigureAwait(false)) + for (var i = 0; i < count; i++) { - EnsureReadersActive(); + var cts = new CancellationTokenSource(); + var task = ReaderLoopAsync(cts.Token).ContinueWith(c => c.Dispose()); + var ctx = new ReaderContext(task, cts); + _activeReaders.Add(ctx); } - - await Task.Delay(100, shutdownToken).ConfigureAwait(false); } } - private void EnsureReadersActive() + + private void AdjustReaders(int target, int current) { - lock (_activeReadersSync) + lock (_readersSync) { - if (!_isEnabled && _disposed) return; - - var activeCount = GetActiveCount(); - - var targetCount = _concurrency; - - var delta = targetCount - activeCount; + if (_state > 0) return; // Не масштабируем при завершении + var delta = target - current; if (delta > 0) { - SpawnAdditionalReaders(delta); + SpawnReaders(delta); } else if (delta < 0) { - foreach (var item in _activeReaders.Take(Math.Abs(delta))) + // Отменяем лишних читателей с конца списка + var toCancel = Math.Min(-delta, _activeReaders.Count); + for (var i = 0; i < toCancel; i++) { - item.TokenSource.Cancel(); + _activeReaders[^1].StopTokenSource.Cancel(); + _activeReaders.RemoveAt(_activeReaders.Count - 1); } } } } - private int GetActiveCount() + private async Task ReaderLoopAsync(CancellationToken stopToken) { - return _activeReaders - .Count(t => !t.Task.IsCompleted || !t.TokenSource.IsCancellationRequested); - } - - private void SpawnAdditionalReaders(int count) - { - void Delete(ReaderTask reader) - { - lock (_activeReadersSync) - { - _activeReaders.Remove(reader); - reader.Dispose(); - } - } - - lock (_activeReadersSync) + var shutdownToken = _shutdownCts.Token; + try { - for (int i = 0; i < count; i++) + while (!stopToken.IsCancellationRequested + && !shutdownToken.IsCancellationRequested + && _isEnabled) { - CancellationTokenSource tokenSource = new(); - Task readerTask = ReaderLoopAsync(tokenSource.Token); - - ReaderTask reader = new(readerTask, tokenSource); + try + { + var item = await _queue.Reader.ReadAsync(shutdownToken).ConfigureAwait(false); - _activeReaders.Add(reader); + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + shutdownToken, + stopToken, + item.CancellationToken); - readerTask.ContinueWith(t => Delete(reader)); + await ExecuteItemAsync(item, cts.Token).ConfigureAwait(false); + } + catch (ChannelClosedException) { break; } + catch (OperationCanceledException) { break; } + catch (Exception ex) + { + LogReaderError(_logger, ex); + await Task.Delay(100, shutdownToken).ConfigureAwait(false); + } } } + finally + { + UnregisterReader(); + } } - - private async Task ReaderLoopAsync(CancellationToken cancellationToken) + private void UnregisterReader() { - var shutdownToken = _shutdownCts.Token; - - while (!cancellationToken.IsCancellationRequested - && !shutdownToken.IsCancellationRequested - && _isEnabled) + lock (_readersSync) { - try - { - WorkItem item = await _queue.Reader.ReadAsync(shutdownToken).ConfigureAwait(false); - - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - shutdownToken, - cancellationToken, - item.CancellationToken); - - await ExecuteItemAsync(item, cts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // ignore - } - catch (Exception ex) - { - _logger?.LogError(ex, "ReaderTask error"); - await Task.Delay(100, shutdownToken).ConfigureAwait(false); - } + // Удаляем завершившегося читателя (если он ещё не был удалён при масштабировании) + _activeReaders.RemoveAll(c => c.Task.IsCompleted); } } + #endregion - private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) + #region Execution & State + + private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) { try { ct.ThrowIfCancellationRequested(); - - item.StartedTime = _timeProvider.GetUtcNow(); - item.Status = WorkStatus.Running; - await OnStatusChangedAsync(item).ConfigureAwait(false); + OnStatusChanged(item.Input, WorkStatus.Running); await _processor.Execute(item.Input, ct).ConfigureAwait(false); - - item.Status = WorkStatus.Completed; + OnStatusChanged(item.Input, WorkStatus.Completed); } catch (OperationCanceledException ex) when (item.CancellationToken.IsCancellationRequested) { - item.Status = WorkStatus.Cancelled; - - if (_logger != null && _logger.IsEnabled(LogLevel.Warning)) - { - _logger?.LogWarning(ex, "Item {ItemId} processing was cancelled", item.Id); - } + OnStatusChanged(item.Input, WorkStatus.Cancelled); + LogItemCancelled(_logger, ex); } catch (Exception ex) { - item.Status = WorkStatus.Faulted; - item.LastError = ex; - _logger?.LogError(ex, "Item {ItemId} execution failed", item.Id); + OnStatusChanged(item.Input, WorkStatus.Faulted, ex); + LogItemExecutionFailed(_logger, ex); } finally { - if (!_disposed) + if (_state < 2) { - item.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(item).ConfigureAwait(false); - DeactivateItem(item); + MarkInactive(); } } } - private void ActivateItem(WorkItem item) + private void MarkActive() { - TaskCompletionSource oldTcs; - - lock (_rootSync) + if (Interlocked.Increment(ref _pendingAndActiveCount) == 1) { - _workItems.Add(item); - _workItemsCount = _workItems.Count; + TaskCompletionSource tcs; - oldTcs = _idleTcs; - _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - } + lock (_rootSync) + { + tcs = _idleTcs; + // Переход из Idle в Active: создаём новый TCS + _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } - oldTcs.TrySetResult(); + tcs.TrySetResult(); + } } - private void DeactivateItem(WorkItem item) + private void MarkInactive() { - TaskCompletionSource? tcsToNotify = null; - - lock (_rootSync) + if (Interlocked.Decrement(ref _pendingAndActiveCount) == 0) { - _workItems.Remove(item); - _workItemsCount = _workItems.Count; - - if (_workItems.Count == 0 && _queue.Reader.Count == 0) - { - tcsToNotify = _idleTcs; - } + TaskCompletionSource tcs; + lock (_rootSync) { tcs = _idleTcs; } + tcs.TrySetResult(); } - - tcsToNotify?.TrySetResult(); } - private async Task OnStatusChangedAsync(WorkItem item) + private void OnStatusChanged(TInput item, WorkStatus status, Exception? error = null) { - var info = item.ToInfo(); - var watcher = _watcher; - - if (watcher != null) + if (_statusChanged is not null) { - try - { - await watcher.HandleChanges(item.Input, info, _shutdownCts.Token).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Observer failed for work item #{WorkItemId}", info.Id); - } + try { _statusChanged(item, status, error); } + catch (Exception ex) { LogEventHandlerFailed(_logger, ex); } } + } - if (_statusChanged != null) - { - try - { - _statusChanged(info); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Event handler failed for work item #{WorkItemId}", info.Id); - } + #endregion - } - } + #region Shutdown & Dispose public async Task ShutdownAsync() { - lock (_rootSync) - { - if (!_isEnabled || _disposed) return; - _isEnabled = false; - } - - await _shutdownCts.CancelAsync().ConfigureAwait(false); - + if (Interlocked.Exchange(ref _state, 1) != 0) return; // Idempotent + _isEnabled = false; try { - await _processingTask.ConfigureAwait(false); + await _shutdownCts.CancelAsync().ConfigureAwait(false); + _queue.Writer.TryComplete(); Task[] readers; - lock (_activeReadersSync) - { - readers = [.. _activeReaders.Select(c => c.Task)]; - } + lock (_readersSync) { readers = [.. _activeReaders.Select(c => c.Task)]; } await Task.WhenAll(readers).ConfigureAwait(false); + + await ClearRemainingItems().ConfigureAwait(false); + } catch (Exception ex) { - _logger?.LogError(ex, "Error during shutdown"); + // Логируем ошибку, если что-то пошло не так при остановке + LogShutdownError(_logger, ex); } finally { - await ClearAll().ConfigureAwait(false); + _state = 2; // Mark as Disposed + + lock (_rootSync) + { + _idleTcs.TrySetResult(); + } } } - private async Task ClearAll() + private async Task ClearRemainingItems() { - while (_queue.Reader.TryRead(out var input)) + OperationCanceledException? err = null; + + while (_queue.Reader.TryRead(out var item)) { - input.Status = WorkStatus.Faulted; - input.LastError = ShutdownAsyncException.Default; - input.EndedTime = _timeProvider.GetUtcNow(); - await OnStatusChangedAsync(input).ConfigureAwait(false); - DeactivateItem(input); + err ??= new OperationCanceledException("Queue shutdown"); + + OnStatusChanged(item.Input, WorkStatus.Faulted, err); + MarkInactive(); } } public void Dispose() { - if (_disposed) return; - - _isEnabled = false; - _queue.Writer.TryComplete(); - - _shutdownCts.Cancel(); - - _disposed = true; + if (Interlocked.Exchange(ref _state, 2) != 2) + { + _isEnabled = false; + _shutdownCts.Cancel(); + _queue.Writer.TryComplete(); + } + Interlocked.Exchange(ref _state, 3); _shutdownCts.Dispose(); } public async ValueTask DisposeAsync() { - if (_disposed) return; - - try + if (Interlocked.Exchange(ref _state, 2) != 2) { - _isEnabled = false; - _queue.Writer.TryComplete(); await ShutdownAsync().ConfigureAwait(false); - - } - finally - { - _disposed = true; - _shutdownCts.Dispose(); } + Interlocked.Exchange(ref _state, 3); + _shutdownCts.Dispose(); } - private sealed class WorkItem(TInput item, CancellationToken token) + #endregion + + + private sealed record ReaderContext(Task Task, CancellationTokenSource StopTokenSource); + + + #region Logging Definitions (Source Generator) + + private static partial class LogMessages { - public long Id { get; init; } - public TInput Input => item; - public CancellationToken CancellationToken => token; - public WorkStatus Status { get; set; } - public DateTimeOffset EnqueuedTime { get; init; } - public DateTimeOffset? StartedTime { get; set; } = default; - public DateTimeOffset? EndedTime { get; set; } = default; - public Exception? LastError { get; set; } = null; - - public WorkInfo ToInfo() => new(Id, Status, EnqueuedTime, StartedTime, EndedTime, LastError); + [LoggerMessage( + EventId = 1, + Level = LogLevel.Warning, + Message = "Item processing was cancelled")] + public static partial void ItemCancelled(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 2, + Level = LogLevel.Error, + Message = "Item execution failed")] + public static partial void ItemExecutionFailed(ILogger logger, Exception exception); + + + [LoggerMessage( + EventId = 3, + Level = LogLevel.Error, + Message = "Event handler failed for work item")] + public static partial void EventHandlerFailed(ILogger logger, Exception exception); + + [LoggerMessage( + EventId = 4, + Level = LogLevel.Error, + Message = "ReaderTask error")] + public static partial void ReaderError(ILogger logger, Exception exception); + + + [LoggerMessage( + EventId = 5, + Level = LogLevel.Error, + Message = "Error during shutdown")] + public static partial void ShutdownError(ILogger logger, Exception exception); } - internal sealed record ReaderTask(Task Task, CancellationTokenSource TokenSource) : IDisposable + // Статические методы-обертки для удобного вызова из экземпляра класса + private static void LogItemCancelled(ILogger>? logger, Exception ex) { - public void Dispose() - { - TokenSource.Dispose(); - } + if (logger is not null) LogMessages.ItemCancelled(logger, ex); } -} + private static void LogItemExecutionFailed(ILogger>? logger, Exception ex) + { + if (logger is not null) LogMessages.ItemExecutionFailed(logger, ex); + } -internal sealed record WorkQueueOptions -( - // Required - IWork Processor, - // Optional with defaults - IWorkObserver? Observer = null, + private static void LogEventHandlerFailed(ILogger>? logger, Exception ex) + { + if (logger is not null) LogMessages.EventHandlerFailed(logger, ex); + } + + private static void LogReaderError(ILogger>? logger, Exception ex) + { + if (logger is not null) LogMessages.ReaderError(logger, ex); + } - TimeProvider? TimeProvider = null, + private static void LogShutdownError(ILogger>? logger, Exception ex) + { + if (logger is not null) LogMessages.ShutdownError(logger, ex); + } + + #endregion +} + +internal sealed record WorkQueueOptions( + IWork Processor, int? QueueCapacity = null, int? ConcurrencyLimit = null, int? MaxConcurrency = null, bool? SingleWriter = false, - Action? StatusChanged = null, - ILogger? Logger = null, - BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait -) + Action? StatusChanged = null, + BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait) { - public WorkQueueOptions WithTimeProvider(TimeProvider timeProvider) => - this with { TimeProvider = timeProvider }; - public WorkQueueOptions WithQueueCapacity(int capacity) { ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); @@ -594,60 +482,32 @@ public WorkQueueOptions WithConcurrencyLimit(int limit) } public WorkQueueOptions WithMaxConcurrency(int limit) - { - if (limit < 1) - { - limit = Environment.ProcessorCount; - } - return this with { MaxConcurrency = limit }; - } - - public WorkQueueOptions WithSingleWriter(bool singleWriter) => - this with { SingleWriter = singleWriter }; - - public WorkQueueOptions WithStatusCallback(Action callback) => - this with { StatusChanged = callback }; - - public WorkQueueOptions WithLogger(ILogger> logger) => - this with { Logger = logger }; + => this with { MaxConcurrency = limit < 1 ? Environment.ProcessorCount : limit }; + public WorkQueueOptions WithSingleWriter(bool sw) + => this with { SingleWriter = sw }; - public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) => - this with { FullMode = mode }; + public WorkQueueOptions WithStatusCallback(Action cb) + => this with { StatusChanged = cb }; - public WorkQueueOptions WithObserver(Func process) => - this with { Observer = new WorkObserverProcessor(process) }; + public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) + => this with { FullMode = mode }; - public WorkQueueOptions WithObserver(IWorkObserver observer) => - this with { Observer = observer }; + public static WorkQueueOptions Create(IWork processor) => new(processor); - // -- ct-or - public static WorkQueueOptions Create(IWork processor) => - new(processor); + public static WorkQueueOptions Create(Func process) + => new(new DelegatingWork(process)); - public static WorkQueueOptions Create(Func process) => - new(new WorkProcessor(process)); - - - - // -- wrap classes - sealed class WorkProcessor(Func process) : IWork + private sealed class DelegatingWork(Func process) : IWork { public Task Execute(TInput input, CancellationToken cancellationToken) => process(input, cancellationToken); } - sealed class WorkObserverProcessor(Func process) : IWorkObserver - { - public Task HandleChanges(TInput input, WorkInfo work, CancellationToken cancellationToken) - => process(input, work, cancellationToken); - } } - - -public sealed class ShutdownAsyncException : Exception +internal static class ThrowHelper { - public static ShutdownAsyncException Default => new(); + public static void QueueStopped() => throw new InvalidOperationException("Queue has been stopped."); } diff --git a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs index 7bc37507..efe8afc7 100644 --- a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs @@ -5,11 +5,9 @@ namespace SaTests.Classes; internal sealed class TrackingWork( - Func? executeFunc = null, - Action? onStatusChanged = null) : IWork, IWorkObserver + Func? executeFunc = null) : IWork { private readonly Func? _executeFunc = executeFunc; - private readonly Action? _onStatusChanged = onStatusChanged; // Thread-safe private int _maxParallelObserved; @@ -49,14 +47,13 @@ public async Task Execute(BlockingTaskModel model, CancellationToken cancellatio } } - public Task HandleChanges( + public void HandleChanges( BlockingTaskModel model, - WorkInfo work, - CancellationToken cancellationToken) + WorkStatus status, + Exception? exception) { - _onStatusChanged?.Invoke(work); - switch (work.Status) + switch (status) { case WorkStatus.Completed: Interlocked.Increment(ref _completedCount); @@ -68,8 +65,6 @@ public Task HandleChanges( Interlocked.Increment(ref _cancelledCount); break; } - - return Task.CompletedTask; } public void Reset() @@ -127,7 +122,8 @@ private WorkQueue CreateQueue( Processor: work, ConcurrencyLimit: concurrencyLimit, MaxConcurrency: maxConcurrency ?? concurrencyLimit * 2, - QueueCapacity: queueCapacity ?? 100 + QueueCapacity: queueCapacity ?? 100, + StatusChanged: work.HandleChanges ); var queue = new WorkQueue(options); diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/SaTests/Classes/WorkQueueTests.cs index 7de6d35c..2f7db73f 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/SaTests/Classes/WorkQueueTests.cs @@ -38,20 +38,6 @@ public Task Execute(TestModel model, CancellationToken cancellationToken) } } - private sealed class TestObserver : IWorkObserver - { - public readonly List Changes = []; - - public Task HandleChanges(TestModel model, WorkInfo work, CancellationToken cancellationToken) - { - lock (Changes) - { - Changes.Add(work); - } - return Task.CompletedTask; - } - } - static CancellationToken TestToken => TestContext.Current.CancellationToken; [Fact] @@ -149,12 +135,14 @@ public async Task ConcurrencyLimit_Respected() [Fact] public async Task FaultedTask_StatusIsFaulted() { + List<(WorkStatus Status, Exception? LastError)> changes = []; + // Arrange var processor = new TestWorkThatThrows(); - var observer = new TestObserver(); + using var queue = new WorkQueue(WorkQueueOptions .Create(processor) - .WithObserver(observer)); + .WithStatusCallback((_, s, e) => changes.Add((s, e)))); var model = new TestModel(); @@ -163,23 +151,26 @@ public async Task FaultedTask_StatusIsFaulted() await Task.Delay(100, TestToken); // Assert - var completed = observer.Changes.Find(x => x.Status == WorkStatus.Faulted); - Assert.NotNull(completed); - Assert.IsType(completed.LastError); - Assert.Contains("Test exception", completed.LastError?.Message); + Assert.Contains(changes, x => x.Status == WorkStatus.Faulted); + + Assert.Contains(changes, x => x.LastError is InvalidOperationException); + Assert.Contains(changes, x => x.LastError?.Message == "Test exception"); } [Fact] public async Task CancelledTask_StatusIsCancelled() { + List statuses = []; + + // Arrange using var cts = new CancellationTokenSource(); var processor = new TestWorkWithDelay(TimeSpan.FromSeconds(5)); - var observer = new TestObserver(); + using var queue = new WorkQueue(WorkQueueOptions .Create(processor) - .WithObserver(observer)); + .WithStatusCallback((_, s, _) => statuses.Add(s))); var model = new TestModel(); @@ -190,21 +181,21 @@ public async Task CancelledTask_StatusIsCancelled() await queue.WaitForIdleAsync(TestToken); // Assert - var cancelled = observer.Changes.Find(x => x.Status == WorkStatus.Cancelled); - Assert.NotNull(cancelled); - Assert.NotEqual(0, cancelled.Id); + Assert.Contains(WorkStatus.Cancelled, statuses); } [Fact] public async Task ShutdownAsync_StopsProcessing() { + List errors = []; + // Arrange var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(500)); - var observer = new TestObserver(); + using var queue = new WorkQueue(WorkQueueOptions .Create(processor) - .WithObserver(observer)); + .WithStatusCallback((m, s, e) => errors.Add(e))); for (int i = 0; i < 5; i++) await queue.Enqueue(new TestModel(), cancellationToken: TestToken); @@ -224,49 +215,25 @@ await Assert.ThrowsAsync(async () => await queue.WaitForIdleAsync(TestToken); Assert.Equal(0, queue.QueueTasks); - var canceled = observer.Changes.Count(c => c.LastError is OperationCanceledException); + var canceled = errors.Count(c => c is OperationCanceledException); Assert.Equal(5, canceled); } - [Fact] - public async Task StatusChanged_Event_FiresOnStateChange() - { - // Arrange - var processor = new TestWork(); - WorkInfo? lastInfo = null; - var eventTcs = new TaskCompletionSource(); - - using var queue = new WorkQueue( - WorkQueueOptions - .Create(processor) - .WithStatusCallback( - (info) => - { - lastInfo = info; - if (info.Status == WorkStatus.Completed) - eventTcs.TrySetResult(info); - })); - var model = new TestModel(); - - // Act - await queue.Enqueue(model, cancellationToken: TestToken); - var result = await eventTcs.Task; - - // Assert - Assert.Equal(WorkStatus.Completed, result.Status); - } [Fact] public async Task Observer_InvokedOnStateChange() { + + List statuses = []; + + // Arrange - var observer = new TestObserver(); var processor = new TestWork(); using var queue = new WorkQueue( WorkQueueOptions .Create(processor) - .WithObserver(observer)); + .WithStatusCallback((_, s, _) => statuses.Add(s))); var model = new TestModel(); @@ -275,10 +242,9 @@ public async Task Observer_InvokedOnStateChange() await queue.WaitForIdleAsync(cancellationToken: TestToken); // Assert - Assert.True(observer.Changes.Count >= 3); // Queued → Running → Completed - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Queued); - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Running); - Assert.Contains(observer.Changes, x => x.Status == WorkStatus.Completed); + Assert.True(statuses.Count >= 2); // Running → Completed + Assert.Contains(statuses, x => x == WorkStatus.Running); + Assert.Contains(statuses, x => x == WorkStatus.Completed); } [Fact] @@ -315,7 +281,7 @@ public async Task Enqueue_Disposed_Throws() { var queue = new WorkQueue( WorkQueueOptions.Create(new TestWork())); - await queue.DisposeAsync(); + queue.Dispose(); await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); From 206b1f0461e39bbdfaf4222f738a3b49e5b3b488 Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 24 Apr 2026 15:45:17 +0300 Subject: [PATCH 26/28] ~ WorkStatus --- src/Sa/Classes/WorkQueue.cs | 45 +++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index 6cc1e2c2..d9898b56 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -42,6 +42,14 @@ internal record WorkItem(TInput Input, CancellationToken CancellationTok internal sealed partial class WorkQueue : IWorkQueue { + private enum QueueState + { + Active = 0, + ShuttingDown = 1, + Shutdown = 2, + Disposed = 3 + } + private readonly Channel> _queue; private readonly Lock _rootSync = new(); @@ -62,7 +70,7 @@ internal sealed partial class WorkQueue : IWorkQueue /// /// 0: Active, 1: ShuttingDown, 2: Shutdown, 3: Disposed /// - private int _state; + private volatile QueueState _state; // Lock-free счётчик активных + ожидающих задач private volatile int _pendingAndActiveCount; @@ -119,7 +127,9 @@ public int ConcurrencyLimit get => _concurrency; set { - ObjectDisposedException.ThrowIf(_state == 2, this); + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); + if (!_isEnabled) ThrowHelper.QueueStopped(); + var newLimit = Math.Clamp(value, 0, _maxConcurrency); var oldLimit = Interlocked.Exchange(ref _concurrency, newLimit); if (newLimit != oldLimit) AdjustReaders(newLimit, oldLimit); @@ -128,8 +138,7 @@ public int ConcurrencyLimit public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_state == 3, this); - + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); if (!_isEnabled) ThrowHelper.QueueStopped(); MarkActive(); @@ -147,7 +156,7 @@ public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) { - ObjectDisposedException.ThrowIf(_state == 3, this); + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); using var cts = CancellationTokenSource.CreateLinkedTokenSource( _shutdownCts.Token, @@ -273,7 +282,7 @@ private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) } finally { - if (_state < 2) + if (_state < QueueState.Shutdown) { MarkInactive(); } @@ -322,8 +331,10 @@ private void OnStatusChanged(TInput item, WorkStatus status, Exception? error = public async Task ShutdownAsync() { - if (Interlocked.Exchange(ref _state, 1) != 0) return; // Idempotent + if (Interlocked.Exchange(ref _state, QueueState.ShuttingDown) != QueueState.Active) return; // Idempotent + _isEnabled = false; + try { await _shutdownCts.CancelAsync().ConfigureAwait(false); @@ -343,7 +354,7 @@ public async Task ShutdownAsync() } finally { - _state = 2; // Mark as Disposed + _state = QueueState.ShuttingDown; lock (_rootSync) { @@ -365,27 +376,33 @@ private async Task ClearRemainingItems() } } + private void CompleteDispose() + { + Interlocked.Exchange(ref _state, QueueState.Disposed); + _shutdownCts.Dispose(); + lock (_rootSync) _idleTcs.TrySetResult(); + } + public void Dispose() { - if (Interlocked.Exchange(ref _state, 2) != 2) + if (IsEnabled) { _isEnabled = false; _shutdownCts.Cancel(); _queue.Writer.TryComplete(); } - Interlocked.Exchange(ref _state, 3); - _shutdownCts.Dispose(); + CompleteDispose(); } public async ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _state, 2) != 2) + if (IsEnabled) { await ShutdownAsync().ConfigureAwait(false); } - Interlocked.Exchange(ref _state, 3); - _shutdownCts.Dispose(); + + CompleteDispose(); } #endregion From 48cf2de3f5182ccfc008cc340299faa9db8bf24c Mon Sep 17 00:00:00 2001 From: dundich Date: Fri, 24 Apr 2026 17:17:34 +0300 Subject: [PATCH 27/28] WorkQueueExtensions --- src/Sa/Classes/WorkQueue.cs | 142 ++++++++++++++++++++++++++++++++++-- 1 file changed, 135 insertions(+), 7 deletions(-) diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs index d9898b56..7e3a89a5 100644 --- a/src/Sa/Classes/WorkQueue.cs +++ b/src/Sa/Classes/WorkQueue.cs @@ -11,6 +11,31 @@ internal interface IWork Task Execute(TInput input, CancellationToken cancellationToken); } +internal enum ReaderScalingStrategy +{ + /// + /// Отменять новейших читателей (LIFO). + /// + /// Старые читатели живут дольше — полезно для кэшей, пулов соединений. + Lifo = 0, + + /// + /// Отменять старейших читателей (FIFO). + /// + /// Равномерное распределение времени жизни — полезно для ротации ресурсов. + Fifo = 1, + + /// + /// RoundRobin: отменять по очереди, циклически + /// + RoundRobin = 2, + + /// + /// Random: отменять случайных читателей (равномерное распределение) + /// + Random = 3 +} + internal enum WorkStatus { Running, @@ -36,7 +61,6 @@ internal interface IWorkQueue : IDisposable, IAsyncDisposable Task ShutdownAsync(); } - internal record WorkItem(TInput Input, CancellationToken CancellationToken); @@ -79,6 +103,10 @@ private enum QueueState private readonly List _activeReaders = []; + private readonly ReaderScalingStrategy _scalingStrategy; + private int _lastRemovedIndex = -1; // Для RoundRobin + + public WorkQueue(WorkQueueOptions options, ILogger>? logger = null) { ArgumentNullException.ThrowIfNull(options.Processor); @@ -93,6 +121,8 @@ public WorkQueue(WorkQueueOptions options, ILogger>? l _concurrency = Math.Clamp(options.ConcurrencyLimit ?? Environment.ProcessorCount, 0, _maxConcurrency); _queueCapacity = options.QueueCapacity ?? _maxConcurrency; + _scalingStrategy = options.ReaderScalingStrategy; + _queue = Channel.CreateBounded>(new BoundedChannelOptions(_queueCapacity) { @@ -171,6 +201,24 @@ public async Task WaitForIdleAsync(CancellationToken cancellationToken = default } } + + /// + /// Принудительно отменяет всех читателей без ожидания. + /// Используйте только в экстренных случаях (например, при зависании). + /// + public void ForceCancelReaders() + { + lock (_readersSync) + { + foreach (var reader in _activeReaders) + { + try { reader.StopTokenSource.Cancel(); } + catch (ObjectDisposedException) { /* уже отменён */ } + } + _activeReaders.Clear(); + } + } + #region Reader Management private void SpawnReaders(int count) @@ -201,17 +249,68 @@ private void AdjustReaders(int target, int current) } else if (delta < 0) { - // Отменяем лишних читателей с конца списка var toCancel = Math.Min(-delta, _activeReaders.Count); - for (var i = 0; i < toCancel; i++) + + + var indices = _scalingStrategy switch { - _activeReaders[^1].StopTokenSource.Cancel(); - _activeReaders.RemoveAt(_activeReaders.Count - 1); + ReaderScalingStrategy.Lifo => + Enumerable.Range(_activeReaders.Count - toCancel, toCancel), + + ReaderScalingStrategy.Fifo => + Enumerable.Range(0, toCancel), + + ReaderScalingStrategy.RoundRobin => + GetRoundRobinIndices(_activeReaders.Count, toCancel), + + ReaderScalingStrategy.Random => + GetRandomIndices(_activeReaders.Count, toCancel), + + _ => Enumerable.Range(_activeReaders.Count - toCancel, toCancel) + }; + + + foreach (var idx in indices.OrderByDescending(i => i)) + { + _activeReaders[idx].StopTokenSource.Cancel(); + _activeReaders.RemoveAt(idx); } } } } + + private IEnumerable GetRoundRobinIndices(int totalCount, int toCancel) + { + if (totalCount == 0) yield break; + + var start = (_lastRemovedIndex + 1) % totalCount; + for (var i = 0; i < toCancel; i++) + { + var idx = (start + i) % totalCount; + yield return idx; + } + _lastRemovedIndex = (start + toCancel - 1) % totalCount; + } + + private static IEnumerable GetRandomIndices(int totalCount, int toCancel) + { + if (toCancel >= totalCount) + { + for (var i = 0; i < totalCount; i++) yield return i; + yield break; + } + + var selected = new HashSet(toCancel); + while (selected.Count < toCancel) + { + // Random.Shared.Next(min, max) потокобезопасен и без блокировок + selected.Add(Random.Shared.Next(0, totalCount)); + } + + foreach (var idx in selected) yield return idx; + } + private async Task ReaderLoopAsync(CancellationToken stopToken) { var shutdownToken = _shutdownCts.Token; @@ -477,6 +576,7 @@ private static void LogShutdownError(ILogger>? logger, Excepti #endregion } + internal sealed record WorkQueueOptions( IWork Processor, int? QueueCapacity = null, @@ -484,7 +584,8 @@ internal sealed record WorkQueueOptions( int? MaxConcurrency = null, bool? SingleWriter = false, Action? StatusChanged = null, - BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait) + BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait, + ReaderScalingStrategy ReaderScalingStrategy = ReaderScalingStrategy.Lifo) { public WorkQueueOptions WithQueueCapacity(int capacity) { @@ -510,6 +611,9 @@ public WorkQueueOptions WithStatusCallback(Action WithFullMode(BoundedChannelFullMode mode) => this with { FullMode = mode }; + public WorkQueueOptions WithReaderScalingStrategy(ReaderScalingStrategy strategy) + => this with { ReaderScalingStrategy = strategy }; + public static WorkQueueOptions Create(IWork processor) => new(processor); @@ -521,10 +625,34 @@ private sealed class DelegatingWork(Func proces public Task Execute(TInput input, CancellationToken cancellationToken) => process(input, cancellationToken); } - } internal static class ThrowHelper { public static void QueueStopped() => throw new InvalidOperationException("Queue has been stopped."); } + +internal static class WorkQueueExtensions +{ + public static IWorkQueue CreateSimple( + this IWork processor, + int concurrency = -1, + ILogger? logger = null) + { + var options = WorkQueueOptions.Create(processor) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new WorkQueue(options, logger as ILogger>); + } + + public static IWorkQueue CreateSimple( + Func process, + int concurrency = -1, + ILogger? logger = null) + { + var options = WorkQueueOptions.Create(process) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new WorkQueue(options, logger as ILogger>); + } +} From d09790df4c09703f4c91f4e789aaa148d83aa3f8 Mon Sep 17 00:00:00 2001 From: dundich Date: Tue, 28 Apr 2026 13:07:24 +0300 Subject: [PATCH 28/28] + SaWorkQueue --- build/tasks.ps1 | 2 + src/Sa.Schedule/Engine/IJobController.cs | 16 +- src/Sa.Schedule/Engine/JobContext.cs | 3 +- src/Sa.Schedule/Engine/JobController.cs | 58 +- src/Sa.Schedule/Engine/JobRunner.cs | 5 +- src/Sa.Schedule/Engine/JobScheduler.cs | 11 +- src/Sa.Schedule/Engine/Scheduler.cs | 12 +- src/Sa.Schedule/Sa.Schedule.csproj | 5 +- src/Sa.Utils.WorkQueue/ISaWork.cs | 10 + src/Sa.Utils.WorkQueue/ISaWorkQueue.cs | 21 + src/Sa.Utils.WorkQueue/Readme.md | 127 ++++ .../Sa.Utils.WorkQueue.csproj | 22 + .../SaExecutionErrorStrategy.cs | 13 + .../SaReaderScalingStrategy.cs | 26 + src/Sa.Utils.WorkQueue/SaWorkQueue.cs | 617 ++++++++++++++++ src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs | 78 +++ src/Sa.Utils.WorkQueue/SaWorkStatus.cs | 27 + src/Sa.Utils.WorkQueue/Setup.cs | 71 ++ src/Sa.slnx | 6 +- src/Sa/Classes/WorkQueue.cs | 658 ------------------ src/Tests/Host.Test.Properties.xml | 4 + .../Sa.ScheduleTests/JobSchedulerTests.cs | 4 +- .../Sa.Utils.WorkQueue.Tests.csproj | 13 + .../WorkQueueConcurrencyTests.cs | 231 +++--- .../WorkQueueDiTests.cs | 63 ++ .../WorkQueueRecoveryTests.cs | 149 ++++ .../WorkQueueTests.cs | 90 ++- 27 files changed, 1494 insertions(+), 848 deletions(-) create mode 100644 src/Sa.Utils.WorkQueue/ISaWork.cs create mode 100644 src/Sa.Utils.WorkQueue/ISaWorkQueue.cs create mode 100644 src/Sa.Utils.WorkQueue/Readme.md create mode 100644 src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj create mode 100644 src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs create mode 100644 src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs create mode 100644 src/Sa.Utils.WorkQueue/SaWorkQueue.cs create mode 100644 src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs create mode 100644 src/Sa.Utils.WorkQueue/SaWorkStatus.cs create mode 100644 src/Sa.Utils.WorkQueue/Setup.cs delete mode 100644 src/Sa/Classes/WorkQueue.cs create mode 100644 src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj rename src/Tests/{SaTests/Classes => Sa.Utils.WorkQueue.Tests}/WorkQueueConcurrencyTests.cs (72%) create mode 100644 src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs create mode 100644 src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs rename src/Tests/{SaTests/Classes => Sa.Utils.WorkQueue.Tests}/WorkQueueTests.cs (71%) diff --git a/build/tasks.ps1 b/build/tasks.ps1 index 46491728..f76b844a 100644 --- a/build/tasks.ps1 +++ b/build/tasks.ps1 @@ -8,6 +8,8 @@ $dist_folder = "$root\dist" $msbuild_verbosity = "n" $projects = @( + "Sa.Utils.WorkQueue", + "Sa.Media", "Sa.Media.FFmpeg", diff --git a/src/Sa.Schedule/Engine/IJobController.cs b/src/Sa.Schedule/Engine/IJobController.cs index 64b86721..f958661e 100644 --- a/src/Sa.Schedule/Engine/IJobController.cs +++ b/src/Sa.Schedule/Engine/IJobController.cs @@ -15,27 +15,19 @@ internal interface IJobController { int Index { get; } - ValueTask WaitToRun(CancellationToken cancellationToken); - - void Init(); - - void Finish(); + void Start(); + void Shutdown(); - - // Методы управления паузой bool IsPaused { get; } void Pause(); void Resume(); - ValueTask WaitIfPaused(CancellationToken cancellationToken); - - - - // iteration events + ValueTask WaitToRun(CancellationToken cancellationToken); + ValueTask WaitIfPaused(CancellationToken cancellationToken); ValueTask CanExecute(CancellationToken cancellationToken); Task Execute(CancellationToken cancellationToken); diff --git a/src/Sa.Schedule/Engine/JobContext.cs b/src/Sa.Schedule/Engine/JobContext.cs index e7b8d520..b6e65717 100644 --- a/src/Sa.Schedule/Engine/JobContext.cs +++ b/src/Sa.Schedule/Engine/JobContext.cs @@ -7,8 +7,7 @@ namespace Sa.Schedule.Engine; internal sealed class JobContext(IJobSettings settings) : IJobContext { - public string JobName => settings.Properties.JobName - ?? $"{settings.JobId}"; + public string JobName => settings.Properties.JobName ?? $"{settings.JobId}"; public IJobSettings Settings => settings; diff --git a/src/Sa.Schedule/Engine/JobController.cs b/src/Sa.Schedule/Engine/JobController.cs index 25c0a92d..97b31e6f 100644 --- a/src/Sa.Schedule/Engine/JobController.cs +++ b/src/Sa.Schedule/Engine/JobController.cs @@ -13,17 +13,16 @@ internal sealed partial class JobController( IJobSettings settings, IInterceptorSettings interceptorSettings, IServiceScopeFactory scopeFactory, - TimeProvider timeProvider) : IJobController + TimeProvider timeProvider) : IJobController, IDisposable { private readonly JobContext _context = new(settings); - private readonly SemaphoreSlim _pauseSemaphore = new(1, 1); - + private volatile bool _disposed; private volatile JobExecutor? _job; - private volatile bool _isPaused; + private readonly CancellationTokenSource _shutdownCts = new(); public int Index => index; @@ -32,6 +31,8 @@ internal sealed partial class JobController( public async ValueTask WaitToRun(CancellationToken cancellationToken) { + if (_disposed) return; + if (_context.NumRuns == 0 && settings.Properties.InitialDelay.HasValue && settings.Properties.InitialDelay.Value != TimeSpan.Zero) @@ -43,22 +44,38 @@ public async ValueTask WaitToRun(CancellationToken cancellationToken) public void Pause() { - if (_isPaused) return; + if (_disposed) return; + + if (Interlocked.CompareExchange(ref _isPaused, true, false)) + { + return; + } - _isPaused = true; _pauseSemaphore.Wait(); // Блокируем семафор } public void Resume() { - if (!_isPaused) return; + if (_disposed) return; - _isPaused = false; + if (!Interlocked.CompareExchange(ref _isPaused, false, true)) + { + return; + } + ReleaseSemaphore(); + } + + private void ReleaseSemaphore() + { try { _pauseSemaphore.Release(); } + catch (ObjectDisposedException) + { + // ignore + } catch (SemaphoreFullException) { // Уже разблокирован @@ -67,23 +84,28 @@ public void Resume() public async ValueTask WaitIfPaused(CancellationToken cancellationToken) { - if (!_isPaused) return; + if (_disposed || !_isPaused) return; + + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, _shutdownCts.Token); try { - await _pauseSemaphore.WaitAsync(cancellationToken); + await _pauseSemaphore.WaitAsync(cts.Token); } finally { - if (!_isPaused) + if (!_disposed && !_isPaused) { - _pauseSemaphore.Release(); + ReleaseSemaphore(); } } } - public void Init() + public void Start() { + if (_disposed) return; + _job = new JobExecutor(settings, interceptorSettings, scopeFactory); _context.ServiceProvider = _job.ServiceProvider; @@ -92,11 +114,17 @@ public void Init() _context.NumRuns++; } - public void Finish() + public void Shutdown() { + if (_disposed) return; + + _disposed = true; + _shutdownCts.Cancel(); + _context.ServiceProvider = NullJobServices.Instance; _job?.Dispose(); _pauseSemaphore.Dispose(); + _shutdownCts.Dispose(); } public async ValueTask CanExecute(CancellationToken cancellationToken) @@ -200,4 +228,6 @@ public void ExecutionFailed(Exception exception) Message = "[{JobName}] {FailedRetryAttempts} out of {RetryCount} reps when the job failed due to an error: {Type} “{Error}”")] static partial void LogFailedRetryAttempts( ILogger logger, string jobName, int failedRetryAttempts, int retryCount, string type, string error); + + public void Dispose() => Shutdown(); } diff --git a/src/Sa.Schedule/Engine/JobRunner.cs b/src/Sa.Schedule/Engine/JobRunner.cs index 6f9d7a55..729824e2 100644 --- a/src/Sa.Schedule/Engine/JobRunner.cs +++ b/src/Sa.Schedule/Engine/JobRunner.cs @@ -2,14 +2,13 @@ namespace Sa.Schedule.Engine; - internal sealed class JobRunner() : IJobRunner { public async Task Run(IJobController controller, CancellationToken cancellationToken) { await controller.WaitToRun(cancellationToken); - controller.Init(); + controller.Start(); try { @@ -17,7 +16,7 @@ public async Task Run(IJobController controller, CancellationToken cancellationT } finally { - controller.Finish(); + controller.Shutdown(); } } diff --git a/src/Sa.Schedule/Engine/JobScheduler.cs b/src/Sa.Schedule/Engine/JobScheduler.cs index 1df784d5..47e96629 100644 --- a/src/Sa.Schedule/Engine/JobScheduler.cs +++ b/src/Sa.Schedule/Engine/JobScheduler.cs @@ -1,5 +1,5 @@ using Microsoft.Extensions.Primitives; -using Sa.Classes; +using Sa.Utils.WorkQueue; namespace Sa.Schedule.Engine; @@ -17,7 +17,7 @@ private readonly static IChangeToken NoneChangeToken private bool _disposed; - private readonly WorkQueue _jobs; + private readonly SaWorkQueue _jobs; private readonly Func _createController; @@ -42,7 +42,7 @@ public JobScheduler( concurrency = Math.Clamp(concurrency, 0, maxConcurrency); - _jobs = new(WorkQueueOptions.Create(CreateJob) + _jobs = new SaWorkQueue(SaWorkQueueOptions.Create(CreateJob) .WithQueueCapacity(maxConcurrency) .WithMaxConcurrency(maxConcurrency) .WithConcurrencyLimit(concurrency) @@ -128,7 +128,6 @@ public async Task Start(CancellationToken cancellationToken) controllers.Add(controller); await _jobs.Enqueue(controller, stoppingToken); } - } finally { @@ -170,6 +169,7 @@ private void RefreshConcurrency() public async Task Stop() { + if (_disposed) return; lock (_lock) { if (_disposed || !_started.GetValueOrDefault()) return; @@ -183,7 +183,7 @@ public async Task Stop() public void Dispose() { - + if (_disposed) return; CancellationTokenSource ctsStopping; lock (_lock) @@ -209,6 +209,7 @@ public void Dispose() public async ValueTask DisposeAsync() { + if (_disposed) return; lock (_lock) { if (_disposed) return; diff --git a/src/Sa.Schedule/Engine/Scheduler.cs b/src/Sa.Schedule/Engine/Scheduler.cs index 9a753818..42fb5eb2 100644 --- a/src/Sa.Schedule/Engine/Scheduler.cs +++ b/src/Sa.Schedule/Engine/Scheduler.cs @@ -3,7 +3,7 @@ internal sealed class Scheduler(IScheduleSettings settings, IJobFactory factory) : IScheduler, IDisposable, IAsyncDisposable { - private bool _disposed; + private volatile bool _disposed; public IScheduleSettings Settings => settings; @@ -55,10 +55,14 @@ public void Dispose() public async ValueTask DisposeAsync() { - await Stop(); - foreach (var job in Schedules) + if (!_disposed) { - await job.DisposeAsync(); + _disposed = true; + await Stop(); + foreach (var job in Schedules) + { + await job.DisposeAsync(); + } } } } diff --git a/src/Sa.Schedule/Sa.Schedule.csproj b/src/Sa.Schedule/Sa.Schedule.csproj index 5b56126c..4b1f3d11 100644 --- a/src/Sa.Schedule/Sa.Schedule.csproj +++ b/src/Sa.Schedule/Sa.Schedule.csproj @@ -17,7 +17,6 @@ - @@ -29,4 +28,8 @@ + + + + diff --git a/src/Sa.Utils.WorkQueue/ISaWork.cs b/src/Sa.Utils.WorkQueue/ISaWork.cs new file mode 100644 index 00000000..4400270b --- /dev/null +++ b/src/Sa.Utils.WorkQueue/ISaWork.cs @@ -0,0 +1,10 @@ +namespace Sa.Utils.WorkQueue; + + +/// +/// BLL interface. +/// +public interface ISaWork +{ + Task Execute(TInput input, CancellationToken cancellationToken); +} diff --git a/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs b/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs new file mode 100644 index 00000000..08f20212 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/ISaWorkQueue.cs @@ -0,0 +1,21 @@ +namespace Sa.Utils.WorkQueue; + +/// +/// Asynchronous task queue with limited parallelism. +/// +public interface ISaWorkQueue : IDisposable, IAsyncDisposable +{ + bool IsEnabled { get; } + int QueueTasks { get; } + bool IsIdle(); + int ConcurrencyLimit { get; set; } + int MaxConcurrency { get; } + int QueueCapacity { get; } + + Exception? ShutdownError { get; } + + ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default); + Task WaitForIdleAsync(CancellationToken cancellationToken = default); + Task ShutdownAsync(); + void ForceCancelReaders(); +} diff --git a/src/Sa.Utils.WorkQueue/Readme.md b/src/Sa.Utils.WorkQueue/Readme.md new file mode 100644 index 00000000..3d1e4aff --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Readme.md @@ -0,0 +1,127 @@ +# SaWorkQueue — Async Queue with Concurrency Limiting + +> High-performance task queue for .NET 10 with dynamic scaling, DI integration, and a type-safe API. + +--- + +## Features + +| Feature | Description | +|---------|-------------| +| **Concurrency limiting** | Control the number of simultaneously executing tasks via `ConcurrencyLimit` | +| **Dynamic scaling** | Change the limit at runtime: `queue.ConcurrencyLimit = newLimit` | +| **Scaling strategies** | `Lifo` • `Fifo` • `RoundRobin` • `Random` — choose the one that fits your scenario | +| **DI integration** | Registration via `AddWorkQueue` with configuration support | +| **Zero-allocation logging** | `[LoggerMessage]` generation for `ILogger` | +| **Safe shutdown** | `DisposeAsync`, `ShutdownAsync`, `WaitForIdleAsync` — idempotent and thread‑safe | + +--- + +## 🚀 Quick Start + +### 1️⃣ Implement your task processor + +```csharp +public sealed class OrderWork : ISaWork +{ + private readonly ILogger _logger; + public OrderWork(ILogger logger) => _logger = logger; + + public async Task Execute(OrderInput input, CancellationToken ct) + { + _logger.LogInformation("Processing order {OrderId}", input.OrderId); + await ProcessAsync(input, ct); // Your business logic + } +} +``` + +### 2️⃣ Register with DI + +```csharp +builder.Services.AddSaWorkQueue((sp, opts) => + opts + .WithConcurrencyLimit(4) + .WithQueueCapacity(100) + .WithReaderScalingStrategy(SaReaderScalingStrategy.RoundRobin) + .WithStatusChanged((input, status, ex) => + { + // logger.LogDebug("Order {Id} → {Status}", input.OrderId, status); + })); +``` + +### 3️⃣ Use via injection + +```csharp +public class OrderService(ISaWorkQueue queue) +{ + public async Task SubmitAsync(OrderInput order, CancellationToken ct) + { + await queue.Enqueue(order, ct); // Does not block the caller + } + + public bool IsIdle() => queue.IsIdle(); + public int Pending => queue.QueueTasks; +} +``` + +--- + +## ⚙️ `WorkQueueOptions` Configuration + +```csharp +SaWorkQueueOptions.Create(processor) + .WithConcurrencyLimit(int) // Concurrency limit (default: CPU count) + .WithQueueCapacity(int) // Queue capacity (default: equals the limit) + .WithMaxConcurrency(int) // Absolute maximum number of readers + .WithReaderScalingStrategy(enum) // Lifo | Fifo | RoundRobin | Random + .WithSingleWriter(bool) // Optimisation for a single write source + .WithFullMode(enum) // Wait | DropWrite | DropOldest + .WithStatusCallback(Action<...>) // Callback on task status change + .WithHandleItemFaulted(Func<...>) // Continue | StopReader | ShutdownQueue + .WithItemDisplayName(Func<...>) // Display name for each work item (e.g., for logging) +``` + +--- + +## Reader Scaling Strategies + +| Strategy | Behaviour | Best for | +|----------|-----------|----------| +| `Lifo` (default) | Cancels the most recently created | CPU‑bound tasks, cache locality | +| `Fifo` | Cancels the earliest created | Connection pools, databases, rotation | +| `RoundRobin` 🔄 | Cycles through the queue | Load balancing, stable workers | +| `Random` 🎲 | Random selection | Testing, avoiding patterns | + +--- + +## 🔑 Key Methods of `ISaWorkQueue` + +```csharp +// Enqueue a task (does not block if there is room in the queue) +await queue.Enqueue(input, cancellationToken); + +// Wait until all tasks have been fully processed +await queue.WaitForIdleAsync(ct); + +// Graceful shutdown: finish active tasks + clear the queue +await queue.ShutdownAsync(); + +// Emergency stop of readers (without waiting for completion) +queue.ForceCancelReaders(); + +// Monitoring +bool idle = queue.IsIdle(); // true if no active or pending tasks +int pending = queue.QueueTasks; // total tasks in progress + in the queue +int limit = queue.ConcurrencyLimit; // current concurrency limit +``` + +--- + +## ⚠️ Important Notes + +1. **Lifecycle**: the queue is registered as `Singleton`. Do not use `Scoped`/`Transient`. +2. **`StatusChanged`**: invoked on a thread‑pool thread. Avoid long synchronous operations inside. +3. **Cancellation**: tasks receive a `CancellationToken`. Handle `OperationCanceledException` properly. +4. **`ForceCancelReaders`**: use only in emergencies — may leave the queue in an inconsistent state. +5. **Reusability**: all shutdown/cleanup methods (`Shutdown`, `Dispose`) are idempotent — safe to call multiple times. + diff --git a/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj new file mode 100644 index 00000000..de70e49e --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Sa.Utils.WorkQueue.csproj @@ -0,0 +1,22 @@ + + + + + + 0.9.0 + WorkQueue wrapper for Channels + + + + 1701;1702;CS8602; + + + + 1701;1702;CS8602; + + + + + + + diff --git a/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs b/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs new file mode 100644 index 00000000..64f5cb45 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaExecutionErrorStrategy.cs @@ -0,0 +1,13 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaExecutionErrorStrategy +{ + /// Mark the item as Faulted and continue processing (default). + Continue, + + /// Mark the item as Faulted and stop the current reader (it will be replaced). + StopReader, + + /// Mark the item as Faulted and initiate a shutdown of the entire queue. + ShutdownQueue +} diff --git a/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs b/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs new file mode 100644 index 00000000..d6a8eb27 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaReaderScalingStrategy.cs @@ -0,0 +1,26 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaReaderScalingStrategy +{ + /// + /// Cancel the most recent readers (LIFO). + /// + /// Older readers live longer — useful for caches, connection pools. + Lifo = 0, + + /// + /// Cancel the oldest readers (FIFO). + /// + /// Even lifetime distribution — useful for resource rotation. + Fifo = 1, + + /// + /// Round‑robin: cancel readers in cyclic order. + /// + RoundRobin = 2, + + /// + /// Random: cancel random readers (uniform distribution). + /// + Random = 3 +} diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueue.cs b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs new file mode 100644 index 00000000..e4f460c9 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkQueue.cs @@ -0,0 +1,617 @@ +namespace Sa.Utils.WorkQueue; + +using Microsoft.Extensions.Logging; +using System.Threading; +using System.Threading.Channels; + +public sealed partial class SaWorkQueue : ISaWorkQueue +{ + private sealed record WorkItem(TInput Input, CancellationToken CancellationToken); + + private enum QueueState + { + Active = 0, + Shutdown = 1, + Disposed = 2 + } + + private readonly Channel _queue; + + private readonly Lock _wiSync = new(); + private readonly Lock _readersSync = new(); + private readonly Lock _cbSync = new(); + + private readonly ILogger? _logger; + private readonly ISaWork _processor; + + private readonly Action? _statusChanged; + private readonly Func _handleItemFaulted; + private readonly Func _getItemDisplayName; + + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly int _maxConcurrency; + private readonly int _queueCapacity; + + private volatile int _concurrency; + /// + /// 0: Active, 1: Shutdown, 2: Disposed + /// + private volatile QueueState _state; + + private volatile Exception? _shutdownError = null; + + // список ожидающих задач + private volatile int _taskCount; + private TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private volatile int _readerCount; + + private readonly List _ctsReaders = []; + private readonly List _taskReaders = []; + + private readonly SaReaderScalingStrategy _scalingStrategy; + private int _lastRemovedIndex = -1; // Для RoundRobin + + + public SaWorkQueue(SaWorkQueueOptions options, ILogger>? logger = null) + { + ArgumentNullException.ThrowIfNull(options.Processor); + + _logger = logger; + _processor = options.Processor; + + _statusChanged = options.StatusChanged; + + _maxConcurrency = options.MaxConcurrency > 0 ? options.MaxConcurrency.Value : Environment.ProcessorCount; + _concurrency = Math.Clamp(options.ConcurrencyLimit ?? Environment.ProcessorCount, 0, _maxConcurrency); + _queueCapacity = options.QueueCapacity ?? _maxConcurrency; + + _scalingStrategy = options.ReaderScalingStrategy; + _getItemDisplayName = options.GetItemDisplayName ?? (item => $"{item}"); + _handleItemFaulted = options.HandleItemFaulted ?? ((_, ex) => SaExecutionErrorStrategy.ShutdownQueue); + + _queue = Channel.CreateBounded(new BoundedChannelOptions(_queueCapacity) + { + AllowSynchronousContinuations = false, + SingleReader = false, + SingleWriter = options.SingleWriter ?? false, + FullMode = options.FullMode + }); + + SpawnReaders(_concurrency); + } + + public bool IsEnabled => _state == QueueState.Active; + + public int QueueTasks => _taskCount; + + public bool IsIdle() => QueueTasks == 0 && _queue.Reader.Count == 0; + + public int MaxConcurrency => _maxConcurrency; + public int QueueCapacity => _queueCapacity; + public Exception? ShutdownError => _shutdownError; + + public int ConcurrencyLimit + { + get => _concurrency; + set + { + var newLimit = Math.Clamp(value, 0, _maxConcurrency); + var oldLimit = Interlocked.Exchange(ref _concurrency, newLimit); + if (newLimit != oldLimit && IsEnabled) + { + AdjustReaders(newLimit); + } + } + } + + public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); + if (!IsEnabled) ThrowHelper.QueueStopped(); + + var wi = new WorkItem(input, cancellationToken); + MarkActive(); + + try + { + await _queue.Writer.WriteAsync(wi, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + MarkInactive(); + + if (ex is ChannelClosedException) + { + ThrowHelper.QueueStopped(); + } + } + } + + public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource( + _shutdownCts.Token, + cancellationToken); + + while (!IsIdle()) + { + TaskCompletionSource tcs; + lock (_wiSync) { tcs = _idleTcs; } + try + { + await tcs.Task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + // ignore + } + } + } + + + /// + /// Forcefully cancels all readers without waiting. + /// Use only in emergency situations (e.g., when hanging). + /// + public void ForceCancelReaders() + { + if (!IsEnabled) return; + + CancellationTokenSource[] readers; + + lock (_readersSync) + { + readers = [.. _ctsReaders.Where(c => !c.IsCancellationRequested)]; + } + + foreach (var reader in readers) + { + try { reader.Cancel(); } catch (ObjectDisposedException) { /* ignored */ } + } + + lock (_wiSync) + { + _idleTcs.TrySetResult(); + } + } + + public async Task ForceCancelReadersAsync() + { + if (!IsEnabled) return; + + CancellationTokenSource[] readers; + + lock (_readersSync) + { + readers = [.. _ctsReaders.Where(c => !c.IsCancellationRequested)]; + } + + foreach (var reader in readers) + { + try { await reader.CancelAsync(); } catch (ObjectDisposedException) { /* ignored */ } + } + + lock (_wiSync) + { + _idleTcs.TrySetResult(); + } + } + + private void SpawnReaders(int count) + { + lock (_readersSync) + { + for (var i = 0; i < count; i++) + { + StartReaderUnderLock(); + } + } + } + + private void StartReaderUnderLock() + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(_shutdownCts.Token); + var task = ReaderLoopAsync(cts); + + _ctsReaders.Add(cts); + _taskReaders.Add(task); + _readerCount++; + } + + private void AdjustReaders(int target) + { + if (!IsEnabled) return; + + lock (_readersSync) + { + int current = _readerCount; + var delta = target - current; + if (delta > 0) + { + for (int i = 0; i < delta; i++) + { + StartReaderUnderLock(); + } + } + else if (delta < 0) + { + var total = _ctsReaders.Count; + if (total == 0) return; + var toCancel = Math.Min(-delta, total); + + var cancellationSources = SelectCancellationSources(toCancel, total); + foreach (var cts in cancellationSources) + { + cts.Cancel(); + } + } + } + } + + private List SelectCancellationSources(int toCancel, int total) + { + var list = new List(toCancel); + + IEnumerable indices = _scalingStrategy switch + { + SaReaderScalingStrategy.Lifo => Enumerable.Range(total - toCancel, toCancel), + SaReaderScalingStrategy.Fifo => Enumerable.Range(0, toCancel), + SaReaderScalingStrategy.RoundRobin => GetRoundRobinIndices(total, toCancel), + SaReaderScalingStrategy.Random => GetRandomIndices(total, toCancel), + _ => Enumerable.Range(total - toCancel, toCancel) + }; + + foreach (var idx in indices.Where(c => c < total)) + { + list.Add(_ctsReaders[idx]); + } + + return list; + } + + private IEnumerable GetRoundRobinIndices(int totalCount, int toCancel) + { + if (totalCount == 0) yield break; + var start = (_lastRemovedIndex + 1) % totalCount; + for (var i = 0; i < toCancel; i++) + { + yield return (start + i) % totalCount; + } + _lastRemovedIndex = (start + toCancel - 1) % totalCount; + } + + private static IEnumerable GetRandomIndices(int totalCount, int toCancel) + { + if (toCancel >= totalCount) + { + for (var i = 0; i < totalCount; i++) yield return i; + yield break; + } + + var selected = new HashSet(toCancel); + while (selected.Count < toCancel) + { + selected.Add(Random.Shared.Next(0, totalCount)); + } + + foreach (var idx in selected) yield return idx; + } + + private async Task ReaderLoopAsync(CancellationTokenSource cts) + { + try + { + while (!cts.IsCancellationRequested) + { + WorkItem item; + try + { + item = await _queue.Reader.ReadAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (ChannelClosedException) + { + break; + } + + bool isContinue = false; + + using var ctsExec = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, item.CancellationToken); + + + isContinue = await ExecuteItemAsync(item, ctsExec.Token).ConfigureAwait(false); + if (!isContinue) break; + + } + } + catch (Exception ex) + { + LogReaderError(_logger, ex); + await ShutdownAsync(); + } + finally + { + RemoveReader(cts); + } + } + + private void RemoveReader(CancellationTokenSource cts) + { + lock (_readersSync) + { + _ctsReaders.Remove(cts); + _readerCount--; + _concurrency = _readerCount; + } + } + + private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) + { + try + { + ct.ThrowIfCancellationRequested(); + OnStatusChanged(item.Input, SaWorkStatus.Running); + await _processor.Execute(item.Input, ct).ConfigureAwait(false); + OnStatusChanged(item.Input, SaWorkStatus.Completed); + return true; + } + catch (OperationCanceledException ex) when (item.CancellationToken.IsCancellationRequested) + { + OnStatusChanged(item.Input, SaWorkStatus.Aborted); + LogItemAborted(_logger, _getItemDisplayName(item.Input), ex); + return true; + } + catch (OperationCanceledException ex) when (ct.IsCancellationRequested) + { + OnStatusChanged(item.Input, SaWorkStatus.Cancelled); + LogItemCancelled(_logger, _getItemDisplayName(item.Input), ex); + return false; + } + catch (Exception ex) + { + var dislpayItem = _getItemDisplayName(item.Input); + + SaExecutionErrorStrategy errorStrategy = SaExecutionErrorStrategy.ShutdownQueue; + try + { + OnStatusChanged(item.Input, SaWorkStatus.Faulted, ex); + LogItemExecutionFailed(_logger, dislpayItem, ex); + + errorStrategy = _handleItemFaulted(item.Input, ex); + } + catch (Exception callbackEx) + { + LogItemHandlerFailed(_logger, dislpayItem, callbackEx); + } + + return errorStrategy switch + { + SaExecutionErrorStrategy.Continue => true, + SaExecutionErrorStrategy.StopReader => false, + SaExecutionErrorStrategy.ShutdownQueue => HandleShutdownOnError(ex), + _ => true + }; + } + finally + { + MarkInactive(); + } + } + + private bool HandleShutdownOnError(Exception ex) + { + _shutdownError = ex; + _ = ShutdownAsync(); + return false; + } + + private void MarkActive() + { + TaskCompletionSource? tcs = null; + lock (_wiSync) + { + if (_taskCount++ == 0) + { + tcs = _idleTcs; + _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + tcs?.TrySetResult(); + } + + private void MarkInactive() + { + TaskCompletionSource? tcs = null; + lock (_wiSync) + { + if (--_taskCount == 0) + { + tcs = _idleTcs; + } + } + tcs?.TrySetResult(); + } + + private void OnStatusChanged(TInput item, SaWorkStatus status, Exception? error = null) + { + if (_statusChanged is null) return; + + Exception? handlerEx = null; + lock (_cbSync) + { + try + { + _statusChanged(item, status, error); + } + catch (Exception ex) + { + handlerEx = ex; + } + } + + if (handlerEx is not null) + { + LogItemHandlerFailed(_logger, _getItemDisplayName(item), handlerEx); + } + } + + public async Task ShutdownAsync() + { + if (Interlocked.CompareExchange(ref _state, QueueState.Shutdown, QueueState.Active) != QueueState.Active) + return; + + try + { + await _shutdownCts.CancelAsync().ConfigureAwait(false); + _queue.Writer.TryComplete(); + await WaitForReadersToCompleteAsync(); + ClearRemainingItems(); + } + catch (Exception ex) + { + LogShutdownError(_logger, ex); + } + } + + private async Task WaitForReadersToCompleteAsync() + { + Task[] tasks; + lock (_readersSync) + { + tasks = [.. _taskReaders]; + } + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + public void Shutdown() + { + if (Interlocked.CompareExchange(ref _state, QueueState.Shutdown, QueueState.Active) != QueueState.Active) + return; + + try + { + _shutdownCts.Cancel(); + _queue.Writer.TryComplete(); + + Task[] tasks; + lock (_readersSync) + { + tasks = [.. _taskReaders]; + } + + Task.WhenAll(tasks).GetAwaiter().GetResult(); + + ClearRemainingItems(); + } + catch (Exception ex) + { + LogShutdownError(_logger, ex); + } + } + + private void ClearRemainingItems() + { + OperationCanceledException? err = null; + + while (_queue.Reader.TryRead(out var item)) + { + err ??= ThrowHelper.QueueShutdownException; + OnStatusChanged(item.Input, SaWorkStatus.Faulted, err); + MarkInactive(); + } + } + + private void CompleteDispose() + { + if (Interlocked.Exchange(ref _state, QueueState.Disposed) != QueueState.Disposed) + { + _shutdownCts.Dispose(); + } + } + + public void Dispose() + { + Shutdown(); + CompleteDispose(); + } + + public async ValueTask DisposeAsync() + { + if (IsEnabled) + { + await ShutdownAsync().ConfigureAwait(false); + } + CompleteDispose(); + } + + + #region Logging Definitions (Source Generator) + + private static partial class LogMessages + { + [LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = "[{Item}] processing was cancelled")] + public static partial void ItemCancelled(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = "[{Item}] processing was aborted")] + public static partial void ItemAborted(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 3, Level = LogLevel.Error, Message = "[{Item}] execution failed")] + public static partial void ItemExecutionFailed(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 4, Level = LogLevel.Error, Message = "[{Item}] event handler failed for work item")] + public static partial void ItemHandlerFailed(ILogger logger, string item, Exception exception); + + [LoggerMessage(EventId = 5, Level = LogLevel.Error, Message = "ReaderTask error")] + public static partial void ReaderError(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 6, Level = LogLevel.Error, Message = "Error during shutdown")] + public static partial void ShutdownError(ILogger logger, Exception exception); + } + + private static void LogItemCancelled(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemCancelled(logger, item, ex); + } + + private static void LogItemAborted(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemAborted(logger, item, ex); + } + + private static void LogItemExecutionFailed(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemExecutionFailed(logger, item, ex); + } + + private static void LogItemHandlerFailed(ILogger? logger, string item, Exception ex) + { + if (logger is not null) LogMessages.ItemHandlerFailed(logger, item, ex); + } + + private static void LogReaderError(ILogger? logger, Exception ex) + { + if (logger is not null) LogMessages.ReaderError(logger, ex); + } + + private static void LogShutdownError(ILogger? logger, Exception ex) + { + if (logger is not null) LogMessages.ShutdownError(logger, ex); + } + + #endregion +} + + +internal static class ThrowHelper +{ + public static void QueueStopped() => throw new InvalidOperationException("Queue has been stopped."); + public static OperationCanceledException QueueShutdownException { get; } + = new OperationCanceledException("Queue shutdown"); +} diff --git a/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs b/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs new file mode 100644 index 00000000..6576df25 --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkQueueOptions.cs @@ -0,0 +1,78 @@ +namespace Sa.Utils.WorkQueue; + +using System.Threading.Channels; + +public sealed record SaWorkQueueOptions( + ISaWork Processor, + int? QueueCapacity = null, + int? ConcurrencyLimit = null, + int? MaxConcurrency = null, + bool? SingleWriter = false, + Func? HandleItemFaulted = null, + Action? StatusChanged = null, + BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait, + SaReaderScalingStrategy ReaderScalingStrategy = SaReaderScalingStrategy.Lifo, + Func? GetItemDisplayName = null) +{ + /// Creates a new options instance with the specified queue capacity. + /// Must be at least 1. + public SaWorkQueueOptions WithQueueCapacity(int capacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + return this with { QueueCapacity = capacity }; + } + + /// Sets the concurrency limit (number of parallel processors). + /// 0 means unlimited, otherwise must be positive. + public SaWorkQueueOptions WithConcurrencyLimit(int limit) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); + return this with { ConcurrencyLimit = limit }; + } + + /// Sets the absolute maximum number of reader tasks. + /// If less than 1, defaults to CPU count. + public SaWorkQueueOptions WithMaxConcurrency(int limit) + => this with { MaxConcurrency = limit < 1 ? Environment.ProcessorCount : limit }; + + /// Optimises for a single writer source. + public SaWorkQueueOptions WithSingleWriter(bool sw) + => this with { SingleWriter = sw }; + + /// Registers a callback for status changes of work items. + public SaWorkQueueOptions WithStatusCallback(Action cb) + => this with { StatusChanged = cb }; + + /// Sets a callback that decides the error handling strategy when an item fails. + public SaWorkQueueOptions WithHandleItemFaulted(Func cb) + => this with { HandleItemFaulted = cb }; + + /// Sets the behaviour when the queue is full. + public SaWorkQueueOptions WithFullMode(BoundedChannelFullMode mode) + => this with { FullMode = mode }; + + /// Sets the strategy for assigning items to readers. + public SaWorkQueueOptions WithReaderScalingStrategy(SaReaderScalingStrategy strategy) + => this with { ReaderScalingStrategy = strategy }; + + /// Sets a function to obtain a display name for each work item (e.g., for logging). + public SaWorkQueueOptions WithItemDisplayName(Func toString) + => this with { GetItemDisplayName = toString }; + + + /// Creates options from a delegate that processes a single item. + /// Async delegate that receives the item and a cancellation token. + public static SaWorkQueueOptions Create(Func process) + => new(new DelegatingWork(process)); + + /// Creates options from an processor. + public static SaWorkQueueOptions Create(ISaWork processor) => new(processor); + + // Helper adapter from delegate to ISaWork + private sealed class DelegatingWork(Func process) : ISaWork + { + public Task Execute(TInput input, CancellationToken cancellationToken) + => process(input, cancellationToken); + } +} + diff --git a/src/Sa.Utils.WorkQueue/SaWorkStatus.cs b/src/Sa.Utils.WorkQueue/SaWorkStatus.cs new file mode 100644 index 00000000..796f6bdd --- /dev/null +++ b/src/Sa.Utils.WorkQueue/SaWorkStatus.cs @@ -0,0 +1,27 @@ +namespace Sa.Utils.WorkQueue; + +public enum SaWorkStatus +{ + /// + /// in processing + /// + Running, + /// + /// it`s ok + /// + Completed, + /// + /// Unhandled technical error, may or may not be retried depending on policy. + /// + Faulted, + /// + /// Operation was cancelled by the system (timeout, shutdown, external trigger). + /// Automatic retry is permissible. + /// + Cancelled, + /// + /// Operation was intentionally aborted by user or orchestrator. + /// Automatic retry MUST NOT be performed. + /// + Aborted +} diff --git a/src/Sa.Utils.WorkQueue/Setup.cs b/src/Sa.Utils.WorkQueue/Setup.cs new file mode 100644 index 00000000..7f57585b --- /dev/null +++ b/src/Sa.Utils.WorkQueue/Setup.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using System.Diagnostics.CodeAnalysis; + +namespace Sa.Utils.WorkQueue; + +public static class Setup +{ + public static IServiceCollection AddSaWorkQueue( + this IServiceCollection services, + Func> configureOptions, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + + services.Add(new ServiceDescriptor( + typeof(ISaWorkQueue), + sp => + { + var options = configureOptions(sp); + var logger = sp.GetService>>(); + return new SaWorkQueue(options, logger); + }, + lifetime)); + + + return services; + } + + public static IServiceCollection AddSaWorkQueue<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TProcessor, TInput>( + this IServiceCollection services, + Func, SaWorkQueueOptions>? configureOptions = null) + where TProcessor : class, ISaWork + { + services.TryAddSingleton(); + + configureOptions ??= (_, opts) => opts; + + services.TryAddSingleton>( + sp => + { + var processor = sp.GetRequiredService(); + var options = configureOptions(sp, SaWorkQueueOptions.Create(processor)); + var logger = sp.GetService>>(); + return new SaWorkQueue(options, logger); + }); + + return services; + } + + + public static ISaWorkQueue CreateSimple( + this ISaWork processor, + int concurrency = -1) + { + var options = SaWorkQueueOptions.Create(processor) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new SaWorkQueue(options); + } + + public static ISaWorkQueue CreateSimple( + Func process, + int concurrency = -1) + { + var options = SaWorkQueueOptions.Create(process) + .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); + + return new SaWorkQueue(options); + } +} diff --git a/src/Sa.slnx b/src/Sa.slnx index 1866a29e..da46cab0 100644 --- a/src/Sa.slnx +++ b/src/Sa.slnx @@ -30,7 +30,7 @@ - + @@ -62,6 +62,7 @@ + @@ -69,5 +70,8 @@ + + + diff --git a/src/Sa/Classes/WorkQueue.cs b/src/Sa/Classes/WorkQueue.cs deleted file mode 100644 index 7e3a89a5..00000000 --- a/src/Sa/Classes/WorkQueue.cs +++ /dev/null @@ -1,658 +0,0 @@ -using Microsoft.Extensions.Logging; -using System.Threading.Channels; - -namespace Sa.Classes; - -/// -/// Интерфейс для выполнения бизнес-логики. -/// -internal interface IWork -{ - Task Execute(TInput input, CancellationToken cancellationToken); -} - -internal enum ReaderScalingStrategy -{ - /// - /// Отменять новейших читателей (LIFO). - /// - /// Старые читатели живут дольше — полезно для кэшей, пулов соединений. - Lifo = 0, - - /// - /// Отменять старейших читателей (FIFO). - /// - /// Равномерное распределение времени жизни — полезно для ротации ресурсов. - Fifo = 1, - - /// - /// RoundRobin: отменять по очереди, циклически - /// - RoundRobin = 2, - - /// - /// Random: отменять случайных читателей (равномерное распределение) - /// - Random = 3 -} - -internal enum WorkStatus -{ - Running, - Completed, - Faulted, - Cancelled -} - -/// -/// Асинхронная очередь задач с ограничением параллелизма. -/// -internal interface IWorkQueue : IDisposable, IAsyncDisposable -{ - bool IsEnabled { get; } - int QueueTasks { get; } - bool IsIdle(); - int ConcurrencyLimit { get; set; } - int MaxConcurrency { get; } - int QueueCapacity { get; } - - ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default); - Task WaitForIdleAsync(CancellationToken cancellationToken = default); - Task ShutdownAsync(); -} - -internal record WorkItem(TInput Input, CancellationToken CancellationToken); - - -internal sealed partial class WorkQueue : IWorkQueue -{ - private enum QueueState - { - Active = 0, - ShuttingDown = 1, - Shutdown = 2, - Disposed = 3 - } - - private readonly Channel> _queue; - - private readonly Lock _rootSync = new(); - private readonly Lock _readersSync = new(); - - private readonly ILogger>? _logger; - - private readonly IWork _processor; - - private readonly Action? _statusChanged; - - private readonly CancellationTokenSource _shutdownCts = new(); - private readonly int _maxConcurrency; - private readonly int _queueCapacity; - - private volatile int _concurrency; - private volatile bool _isEnabled = true; - /// - /// 0: Active, 1: ShuttingDown, 2: Shutdown, 3: Disposed - /// - private volatile QueueState _state; - - // Lock-free счётчик активных + ожидающих задач - private volatile int _pendingAndActiveCount; - private TaskCompletionSource _idleTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - - - private readonly List _activeReaders = []; - - private readonly ReaderScalingStrategy _scalingStrategy; - private int _lastRemovedIndex = -1; // Для RoundRobin - - - public WorkQueue(WorkQueueOptions options, ILogger>? logger = null) - { - ArgumentNullException.ThrowIfNull(options.Processor); - - _logger = logger; - - _processor = options.Processor; - - _statusChanged = options.StatusChanged; - - _maxConcurrency = options.MaxConcurrency > 0 ? options.MaxConcurrency.Value : Environment.ProcessorCount; - _concurrency = Math.Clamp(options.ConcurrencyLimit ?? Environment.ProcessorCount, 0, _maxConcurrency); - _queueCapacity = options.QueueCapacity ?? _maxConcurrency; - - _scalingStrategy = options.ReaderScalingStrategy; - - - _queue = Channel.CreateBounded>(new BoundedChannelOptions(_queueCapacity) - { - AllowSynchronousContinuations = false, - SingleReader = false, - SingleWriter = options.SingleWriter ?? false, - FullMode = options.FullMode - }); - - // Запускаем читателей сразу - SpawnReaders(_concurrency); - } - - public bool IsEnabled => _isEnabled; - - public int QueueTasks => _pendingAndActiveCount; - - public bool IsIdle() - { - lock (_rootSync) - { - return _pendingAndActiveCount <= 0 && _queue.Reader.Count == 0; - } - } - - public int MaxConcurrency => _maxConcurrency; - - public int QueueCapacity => _queueCapacity; - - public int ConcurrencyLimit - { - get => _concurrency; - set - { - ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); - if (!_isEnabled) ThrowHelper.QueueStopped(); - - var newLimit = Math.Clamp(value, 0, _maxConcurrency); - var oldLimit = Interlocked.Exchange(ref _concurrency, newLimit); - if (newLimit != oldLimit) AdjustReaders(newLimit, oldLimit); - } - } - - public async ValueTask Enqueue(TInput input, CancellationToken cancellationToken = default) - { - ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); - if (!_isEnabled) ThrowHelper.QueueStopped(); - - MarkActive(); - - try - { - await _queue.Writer.WriteAsync(new(input, cancellationToken), cancellationToken).ConfigureAwait(false); - } - catch - { - MarkInactive(); // Откат счётчика при ошибке записи - throw; - } - } - - public async Task WaitForIdleAsync(CancellationToken cancellationToken = default) - { - ObjectDisposedException.ThrowIf(_state == QueueState.Disposed, this); - - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - _shutdownCts.Token, - cancellationToken); - - while (!IsIdle()) - { - TaskCompletionSource tcs; - lock (_rootSync) { tcs = _idleTcs; } - - await tcs.Task.WaitAsync(cts.Token).ConfigureAwait(false); - } - } - - - /// - /// Принудительно отменяет всех читателей без ожидания. - /// Используйте только в экстренных случаях (например, при зависании). - /// - public void ForceCancelReaders() - { - lock (_readersSync) - { - foreach (var reader in _activeReaders) - { - try { reader.StopTokenSource.Cancel(); } - catch (ObjectDisposedException) { /* уже отменён */ } - } - _activeReaders.Clear(); - } - } - - #region Reader Management - - private void SpawnReaders(int count) - { - lock (_readersSync) - { - for (var i = 0; i < count; i++) - { - var cts = new CancellationTokenSource(); - var task = ReaderLoopAsync(cts.Token).ContinueWith(c => c.Dispose()); - var ctx = new ReaderContext(task, cts); - _activeReaders.Add(ctx); - } - } - } - - - private void AdjustReaders(int target, int current) - { - lock (_readersSync) - { - if (_state > 0) return; // Не масштабируем при завершении - - var delta = target - current; - if (delta > 0) - { - SpawnReaders(delta); - } - else if (delta < 0) - { - var toCancel = Math.Min(-delta, _activeReaders.Count); - - - var indices = _scalingStrategy switch - { - ReaderScalingStrategy.Lifo => - Enumerable.Range(_activeReaders.Count - toCancel, toCancel), - - ReaderScalingStrategy.Fifo => - Enumerable.Range(0, toCancel), - - ReaderScalingStrategy.RoundRobin => - GetRoundRobinIndices(_activeReaders.Count, toCancel), - - ReaderScalingStrategy.Random => - GetRandomIndices(_activeReaders.Count, toCancel), - - _ => Enumerable.Range(_activeReaders.Count - toCancel, toCancel) - }; - - - foreach (var idx in indices.OrderByDescending(i => i)) - { - _activeReaders[idx].StopTokenSource.Cancel(); - _activeReaders.RemoveAt(idx); - } - } - } - } - - - private IEnumerable GetRoundRobinIndices(int totalCount, int toCancel) - { - if (totalCount == 0) yield break; - - var start = (_lastRemovedIndex + 1) % totalCount; - for (var i = 0; i < toCancel; i++) - { - var idx = (start + i) % totalCount; - yield return idx; - } - _lastRemovedIndex = (start + toCancel - 1) % totalCount; - } - - private static IEnumerable GetRandomIndices(int totalCount, int toCancel) - { - if (toCancel >= totalCount) - { - for (var i = 0; i < totalCount; i++) yield return i; - yield break; - } - - var selected = new HashSet(toCancel); - while (selected.Count < toCancel) - { - // Random.Shared.Next(min, max) потокобезопасен и без блокировок - selected.Add(Random.Shared.Next(0, totalCount)); - } - - foreach (var idx in selected) yield return idx; - } - - private async Task ReaderLoopAsync(CancellationToken stopToken) - { - var shutdownToken = _shutdownCts.Token; - try - { - while (!stopToken.IsCancellationRequested - && !shutdownToken.IsCancellationRequested - && _isEnabled) - { - try - { - var item = await _queue.Reader.ReadAsync(shutdownToken).ConfigureAwait(false); - - using var cts = CancellationTokenSource.CreateLinkedTokenSource( - shutdownToken, - stopToken, - item.CancellationToken); - - await ExecuteItemAsync(item, cts.Token).ConfigureAwait(false); - } - catch (ChannelClosedException) { break; } - catch (OperationCanceledException) { break; } - catch (Exception ex) - { - LogReaderError(_logger, ex); - await Task.Delay(100, shutdownToken).ConfigureAwait(false); - } - } - } - finally - { - UnregisterReader(); - } - } - - private void UnregisterReader() - { - lock (_readersSync) - { - // Удаляем завершившегося читателя (если он ещё не был удалён при масштабировании) - _activeReaders.RemoveAll(c => c.Task.IsCompleted); - } - } - - #endregion - - #region Execution & State - - private async Task ExecuteItemAsync(WorkItem item, CancellationToken ct) - { - try - { - ct.ThrowIfCancellationRequested(); - OnStatusChanged(item.Input, WorkStatus.Running); - - await _processor.Execute(item.Input, ct).ConfigureAwait(false); - OnStatusChanged(item.Input, WorkStatus.Completed); - } - catch (OperationCanceledException ex) when (item.CancellationToken.IsCancellationRequested) - { - OnStatusChanged(item.Input, WorkStatus.Cancelled); - LogItemCancelled(_logger, ex); - } - catch (Exception ex) - { - OnStatusChanged(item.Input, WorkStatus.Faulted, ex); - LogItemExecutionFailed(_logger, ex); - } - finally - { - if (_state < QueueState.Shutdown) - { - MarkInactive(); - } - } - } - - private void MarkActive() - { - if (Interlocked.Increment(ref _pendingAndActiveCount) == 1) - { - TaskCompletionSource tcs; - - lock (_rootSync) - { - tcs = _idleTcs; - // Переход из Idle в Active: создаём новый TCS - _idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - } - - tcs.TrySetResult(); - } - } - - private void MarkInactive() - { - if (Interlocked.Decrement(ref _pendingAndActiveCount) == 0) - { - TaskCompletionSource tcs; - lock (_rootSync) { tcs = _idleTcs; } - tcs.TrySetResult(); - } - } - - private void OnStatusChanged(TInput item, WorkStatus status, Exception? error = null) - { - if (_statusChanged is not null) - { - try { _statusChanged(item, status, error); } - catch (Exception ex) { LogEventHandlerFailed(_logger, ex); } - } - } - - #endregion - - #region Shutdown & Dispose - - public async Task ShutdownAsync() - { - if (Interlocked.Exchange(ref _state, QueueState.ShuttingDown) != QueueState.Active) return; // Idempotent - - _isEnabled = false; - - try - { - await _shutdownCts.CancelAsync().ConfigureAwait(false); - _queue.Writer.TryComplete(); - - Task[] readers; - lock (_readersSync) { readers = [.. _activeReaders.Select(c => c.Task)]; } - await Task.WhenAll(readers).ConfigureAwait(false); - - await ClearRemainingItems().ConfigureAwait(false); - - } - catch (Exception ex) - { - // Логируем ошибку, если что-то пошло не так при остановке - LogShutdownError(_logger, ex); - } - finally - { - _state = QueueState.ShuttingDown; - - lock (_rootSync) - { - _idleTcs.TrySetResult(); - } - } - } - - private async Task ClearRemainingItems() - { - OperationCanceledException? err = null; - - while (_queue.Reader.TryRead(out var item)) - { - err ??= new OperationCanceledException("Queue shutdown"); - - OnStatusChanged(item.Input, WorkStatus.Faulted, err); - MarkInactive(); - } - } - - private void CompleteDispose() - { - Interlocked.Exchange(ref _state, QueueState.Disposed); - _shutdownCts.Dispose(); - lock (_rootSync) _idleTcs.TrySetResult(); - } - - public void Dispose() - { - if (IsEnabled) - { - _isEnabled = false; - _shutdownCts.Cancel(); - _queue.Writer.TryComplete(); - } - - CompleteDispose(); - } - - public async ValueTask DisposeAsync() - { - if (IsEnabled) - { - await ShutdownAsync().ConfigureAwait(false); - } - - CompleteDispose(); - } - - #endregion - - - private sealed record ReaderContext(Task Task, CancellationTokenSource StopTokenSource); - - - #region Logging Definitions (Source Generator) - - private static partial class LogMessages - { - [LoggerMessage( - EventId = 1, - Level = LogLevel.Warning, - Message = "Item processing was cancelled")] - public static partial void ItemCancelled(ILogger logger, Exception exception); - - [LoggerMessage( - EventId = 2, - Level = LogLevel.Error, - Message = "Item execution failed")] - public static partial void ItemExecutionFailed(ILogger logger, Exception exception); - - - [LoggerMessage( - EventId = 3, - Level = LogLevel.Error, - Message = "Event handler failed for work item")] - public static partial void EventHandlerFailed(ILogger logger, Exception exception); - - [LoggerMessage( - EventId = 4, - Level = LogLevel.Error, - Message = "ReaderTask error")] - public static partial void ReaderError(ILogger logger, Exception exception); - - - [LoggerMessage( - EventId = 5, - Level = LogLevel.Error, - Message = "Error during shutdown")] - public static partial void ShutdownError(ILogger logger, Exception exception); - } - - // Статические методы-обертки для удобного вызова из экземпляра класса - private static void LogItemCancelled(ILogger>? logger, Exception ex) - { - if (logger is not null) LogMessages.ItemCancelled(logger, ex); - } - - private static void LogItemExecutionFailed(ILogger>? logger, Exception ex) - { - if (logger is not null) LogMessages.ItemExecutionFailed(logger, ex); - } - - private static void LogEventHandlerFailed(ILogger>? logger, Exception ex) - { - if (logger is not null) LogMessages.EventHandlerFailed(logger, ex); - } - - private static void LogReaderError(ILogger>? logger, Exception ex) - { - if (logger is not null) LogMessages.ReaderError(logger, ex); - } - - private static void LogShutdownError(ILogger>? logger, Exception ex) - { - if (logger is not null) LogMessages.ShutdownError(logger, ex); - } - - #endregion -} - - -internal sealed record WorkQueueOptions( - IWork Processor, - int? QueueCapacity = null, - int? ConcurrencyLimit = null, - int? MaxConcurrency = null, - bool? SingleWriter = false, - Action? StatusChanged = null, - BoundedChannelFullMode FullMode = BoundedChannelFullMode.Wait, - ReaderScalingStrategy ReaderScalingStrategy = ReaderScalingStrategy.Lifo) -{ - public WorkQueueOptions WithQueueCapacity(int capacity) - { - ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); - return this with { QueueCapacity = capacity }; - } - - public WorkQueueOptions WithConcurrencyLimit(int limit) - { - ArgumentOutOfRangeException.ThrowIfLessThan(limit, 0); - return this with { ConcurrencyLimit = limit }; - } - - public WorkQueueOptions WithMaxConcurrency(int limit) - => this with { MaxConcurrency = limit < 1 ? Environment.ProcessorCount : limit }; - - public WorkQueueOptions WithSingleWriter(bool sw) - => this with { SingleWriter = sw }; - - public WorkQueueOptions WithStatusCallback(Action cb) - => this with { StatusChanged = cb }; - - public WorkQueueOptions WithFullMode(BoundedChannelFullMode mode) - => this with { FullMode = mode }; - - public WorkQueueOptions WithReaderScalingStrategy(ReaderScalingStrategy strategy) - => this with { ReaderScalingStrategy = strategy }; - - - public static WorkQueueOptions Create(IWork processor) => new(processor); - - public static WorkQueueOptions Create(Func process) - => new(new DelegatingWork(process)); - - private sealed class DelegatingWork(Func process) : IWork - { - public Task Execute(TInput input, CancellationToken cancellationToken) - => process(input, cancellationToken); - } -} - -internal static class ThrowHelper -{ - public static void QueueStopped() => throw new InvalidOperationException("Queue has been stopped."); -} - -internal static class WorkQueueExtensions -{ - public static IWorkQueue CreateSimple( - this IWork processor, - int concurrency = -1, - ILogger? logger = null) - { - var options = WorkQueueOptions.Create(processor) - .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); - - return new WorkQueue(options, logger as ILogger>); - } - - public static IWorkQueue CreateSimple( - Func process, - int concurrency = -1, - ILogger? logger = null) - { - var options = WorkQueueOptions.Create(process) - .WithConcurrencyLimit(concurrency > 0 ? concurrency : Environment.ProcessorCount); - - return new WorkQueue(options, logger as ILogger>); - } -} diff --git a/src/Tests/Host.Test.Properties.xml b/src/Tests/Host.Test.Properties.xml index 0a13bcdb..361d328c 100644 --- a/src/Tests/Host.Test.Properties.xml +++ b/src/Tests/Host.Test.Properties.xml @@ -7,6 +7,10 @@ false + + 1591;1701;1702;IDE1006;S3776 + + diff --git a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs index ce9d67c4..c512cc35 100644 --- a/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs +++ b/src/Tests/Sa.ScheduleTests/JobSchedulerTests.cs @@ -143,12 +143,12 @@ public void Resume() } - public void Init() + public void Start() { // } - public void Finish() + public void Shutdown() { // } diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj b/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj new file mode 100644 index 00000000..7ef1de06 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/Sa.Utils.WorkQueue.Tests.csproj @@ -0,0 +1,13 @@ + + + + + + true + + + + + + + diff --git a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs similarity index 72% rename from src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs rename to src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs index efe8afc7..01091d51 100644 --- a/src/Tests/SaTests/Classes/WorkQueueConcurrencyTests.cs +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueConcurrencyTests.cs @@ -1,101 +1,102 @@ -using Sa.Classes; -using System.Collections.Concurrent; +using System.Collections.Concurrent; -namespace SaTests.Classes; +namespace Sa.Utils.WorkQueue.Tests; -internal sealed class TrackingWork( - Func? executeFunc = null) : IWork +public sealed class WorkQueueConcurrencyTests : IAsyncLifetime { - private readonly Func? _executeFunc = executeFunc; - - // Thread-safe - private int _maxParallelObserved; - private int _currentParallel; - private long _completedCount; - private long _failedCount; - private long _cancelledCount; - - public int MaxParallelObserved => Volatile.Read(ref _maxParallelObserved); - public long CompletedCount => Interlocked.Read(ref _completedCount); - public long FailedCount => Interlocked.Read(ref _failedCount); - public long CancelledCount => Interlocked.Read(ref _cancelledCount); - public long TotalProcessed => CompletedCount + FailedCount + CancelledCount; - - public async Task Execute(BlockingTaskModel model, CancellationToken cancellationToken) + static CancellationToken TestToken => TestContext.Current.CancellationToken; + + internal sealed class TrackingWork( + Func? executeFunc = null) : ISaWork { - int current = Interlocked.Increment(ref _currentParallel); - try + private readonly Func? _executeFunc = executeFunc; + + // Thread-safe + private int _maxParallelObserved; + private int _currentParallel; + private long _completedCount; + private long _failedCount; + private long _cancelledCount; + + public int MaxParallelObserved => Volatile.Read(ref _maxParallelObserved); + public long CompletedCount => Interlocked.Read(ref _completedCount); + public long FailedCount => Interlocked.Read(ref _failedCount); + public long CancelledCount => Interlocked.Read(ref _cancelledCount); + public long TotalProcessed => CompletedCount + FailedCount + CancelledCount; + + public async Task Execute(BlockingTaskModel model, CancellationToken cancellationToken) { - int max = Volatile.Read(ref _maxParallelObserved); - if (current > max) + int current = Interlocked.Increment(ref _currentParallel); + try { - Interlocked.CompareExchange(ref _maxParallelObserved, current, max); - } + int max = Volatile.Read(ref _maxParallelObserved); + if (current > max) + { + Interlocked.CompareExchange(ref _maxParallelObserved, current, max); + } - await model.AllowCompletion.Task; + await model.AllowCompletion.Task; - if (_executeFunc != null) - await _executeFunc(model, cancellationToken); - else - await Task.Delay(50, cancellationToken); // Default delay + if (_executeFunc != null) + await _executeFunc(model, cancellationToken); + else + await Task.Delay(50, cancellationToken); // Default delay + } + finally + { + Interlocked.Decrement(ref _currentParallel); + } } - finally + + public void HandleChanges( + BlockingTaskModel _, + SaWorkStatus status, + Exception? __) { - Interlocked.Decrement(ref _currentParallel); - } - } - public void HandleChanges( - BlockingTaskModel model, - WorkStatus status, - Exception? exception) - { + switch (status) + { + case SaWorkStatus.Completed: + Interlocked.Increment(ref _completedCount); + break; + case SaWorkStatus.Faulted: + Interlocked.Increment(ref _failedCount); + break; + case SaWorkStatus.Cancelled: + Interlocked.Increment(ref _cancelledCount); + break; + } + } - switch (status) + public void Reset() { - case WorkStatus.Completed: - Interlocked.Increment(ref _completedCount); - break; - case WorkStatus.Faulted: - Interlocked.Increment(ref _failedCount); - break; - case WorkStatus.Cancelled: - Interlocked.Increment(ref _cancelledCount); - break; + Volatile.Write(ref _maxParallelObserved, 0); + Volatile.Write(ref _currentParallel, 0); + Interlocked.Exchange(ref _completedCount, 0); + Interlocked.Exchange(ref _failedCount, 0); + Interlocked.Exchange(ref _cancelledCount, 0); } } - public void Reset() + internal sealed record BlockingTaskModel( + TimeSpan? Delay = null, + bool ShouldFail = false, + string? Id = null + ) { - Volatile.Write(ref _maxParallelObserved, 0); - Volatile.Write(ref _currentParallel, 0); - Interlocked.Exchange(ref _completedCount, 0); - Interlocked.Exchange(ref _failedCount, 0); - Interlocked.Exchange(ref _cancelledCount, 0); - } -} + public TaskCompletionSource AllowCompletion { get; } + = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); -internal sealed record BlockingTaskModel( - TimeSpan? Delay = null, - bool ShouldFail = false, - string? Id = null -) -{ - public TaskCompletionSource AllowCompletion { get; } - = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - public BlockingTaskModel Unlock() - { - AllowCompletion.SetResult(); - return this; - } -}; + public BlockingTaskModel Unlock() + { + AllowCompletion.SetResult(); + return this; + } + }; -public sealed class WorkQueueConcurrencyTests : IAsyncLifetime -{ private readonly List _disposables = []; public ValueTask InitializeAsync() => ValueTask.CompletedTask; @@ -112,13 +113,13 @@ public async ValueTask DisposeAsync() _disposables.Clear(); } - private WorkQueue CreateQueue( + private SaWorkQueue CreateQueue( TrackingWork work, int concurrencyLimit, int? queueCapacity = null, int? maxConcurrency = null) { - var options = new WorkQueueOptions( + var options = new SaWorkQueueOptions( Processor: work, ConcurrencyLimit: concurrencyLimit, MaxConcurrency: maxConcurrency ?? concurrencyLimit * 2, @@ -126,7 +127,7 @@ private WorkQueue CreateQueue( StatusChanged: work.HandleChanges ); - var queue = new WorkQueue(options); + var queue = new SaWorkQueue(options); _disposables.Add(queue); return queue; } @@ -147,10 +148,10 @@ public async Task ConcurrencyLimit_Respected_AtStartup() { var model = new BlockingTaskModel(); blockers.Add(model.AllowCompletion); - await queue.Enqueue(model, TestContext.Current.CancellationToken); + await queue.Enqueue(model, TestToken); } - await Task.Delay(200, TestContext.Current.CancellationToken); + await Task.Delay(200, TestToken); Assert.True(work.MaxParallelObserved <= 3, @@ -162,7 +163,7 @@ public async Task ConcurrencyLimit_Respected_AtStartup() foreach (var tcs in blockers) tcs.SetResult(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.Equal(10, work.TotalProcessed); } @@ -178,20 +179,20 @@ public async Task ConcurrencyLimit_Zero_StopsProcessing() for (int i = 0; i < 5; i++) { var model = new BlockingTaskModel(Delay: TimeSpan.FromMilliseconds(10)); - await queue.Enqueue(model, TestContext.Current.CancellationToken); + await queue.Enqueue(model, TestToken); model.Unlock(); // Сразу разрешаем завершение } Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); - await Task.Delay(300, TestContext.Current.CancellationToken); + await Task.Delay(300, TestToken); Assert.Equal(5, queue.QueueTasks + work.TotalProcessed); Assert.Equal(0, work.TotalProcessed); // Восстанавливаем лимит и ждём завершения queue.ConcurrencyLimit = 2; - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.Equal(5, work.TotalProcessed); } @@ -210,9 +211,9 @@ public async Task ConcurrencyLimit_DynamicIncrease_Works() { var model = new BlockingTaskModel(Id: $"P1-{i}"); phase1Blockers.Add(model.AllowCompletion); - await queue.Enqueue(model, TestContext.Current.CancellationToken); + await queue.Enqueue(model, TestToken); } - await Task.Delay(150, TestContext.Current.CancellationToken); + await Task.Delay(150, TestToken); int maxAtPhase1 = work.MaxParallelObserved; Assert.True(maxAtPhase1 <= 2, $"Phase 1: expected <=2, got {maxAtPhase1}"); @@ -224,9 +225,9 @@ public async Task ConcurrencyLimit_DynamicIncrease_Works() { var model2 = new BlockingTaskModel(Id: $"P2-{i}"); phase2Blockers.Add(model2.AllowCompletion); - await queue.Enqueue(model2, TestContext.Current.CancellationToken); + await queue.Enqueue(model2, TestToken); } - await Task.Delay(150, TestContext.Current.CancellationToken); + await Task.Delay(150, TestToken); // Assert: после увеличения лимита должно быть возможно до 4 параллельных Assert.True(work.MaxParallelObserved <= 4, @@ -238,7 +239,7 @@ public async Task ConcurrencyLimit_DynamicIncrease_Works() foreach (var tcs in phase1Blockers.Concat(phase2Blockers)) tcs.SetResult(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.Equal(8, work.TotalProcessed); } @@ -261,14 +262,14 @@ public async Task ConcurrencyLimit_DynamicDecrease_ThrottlesReaders() blockers.Add(model.AllowCompletion); startedEvents.Add(startedTcs); - await queue.Enqueue(model, TestContext.Current.CancellationToken); + await queue.Enqueue(model, TestToken); } for (int i = 0; i < 4; i++) blockers[i].SetResult(); - await Task.Delay(200, TestContext.Current.CancellationToken); + await Task.Delay(200, TestToken); int beforeDecrease = work.MaxParallelObserved; Assert.True(beforeDecrease <= 4, $"Before decrease: expected <=4, got {beforeDecrease}"); @@ -276,7 +277,7 @@ public async Task ConcurrencyLimit_DynamicDecrease_ThrottlesReaders() // Act: decrease limit to 2 queue.ConcurrencyLimit = 2; - await Task.Delay(200, TestContext.Current.CancellationToken); + await Task.Delay(200, TestToken); Assert.True(work.MaxParallelObserved <= 4, $"After decrease: observed spike to {work.MaxParallelObserved}"); @@ -286,7 +287,7 @@ public async Task ConcurrencyLimit_DynamicDecrease_ThrottlesReaders() blockers[i].SetResult(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.Equal(8, work.TotalProcessed); Assert.Equal(0, queue.QueueTasks); @@ -316,14 +317,14 @@ public async Task ConcurrencyLimit_NeverExceeded_UnderLoad(int limit, int taskCo await Task.WhenAll(enqueueTasks); - await Task.Delay(300, TestContext.Current.CancellationToken); + await Task.Delay(300, TestToken); Assert.True(work.MaxParallelObserved <= limit); foreach (var tcs in blockers) tcs.SetResult(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.Equal(taskCount, work.TotalProcessed); } @@ -363,9 +364,24 @@ public async Task ConcurrencyLimitRapidChangesDoesNotBreak() } // cts.Cancel(); // Stop enqueuer }, cts.Token); + Task processTask = CreateProcess(totals, cts, blockers); + + await Task.WhenAll(enqueueTask, changeTask, processTask); + await queue.WaitForIdleAsync(TestToken); + + // Assert: no exceptions, all tasks processed, limit respected at peaks + Assert.True(work.MaxParallelObserved <= 6, + $"Max observed {work.MaxParallelObserved} exceeded max configured limit 6"); + + Assert.Equal(totals, work.TotalProcessed); // All accounted for + } + + private static Task CreateProcess( + int totals, CancellationTokenSource cts, ConcurrentQueue blockers) + { // Processor: complete tasks with small delay - var processTask = Task.Run(async () => + return Task.Run(async () => { int i = 0; @@ -396,15 +412,6 @@ public async Task ConcurrencyLimitRapidChangesDoesNotBreak() Console.WriteLine(e); } }, cts.Token); - - await Task.WhenAll(enqueueTask, changeTask, processTask); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); - - // Assert: no exceptions, all tasks processed, limit respected at peaks - Assert.True(work.MaxParallelObserved <= 6, - $"Max observed {work.MaxParallelObserved} exceeded max configured limit 6"); - - Assert.Equal(totals, work.TotalProcessed); // All accounted for } [Fact] @@ -425,7 +432,7 @@ public async Task ConcurrencyLimit_SetToMaxAllowed_ClampsCorrectly() Assert.Equal(3, queue.ConcurrencyLimit); } - + [Fact] public async Task ConcurrencyLimit_UpChanges_DontLoseTasks() { @@ -454,7 +461,7 @@ public async Task ConcurrencyLimit_UpChanges_DontLoseTasks() var model = new BlockingTaskModel(Id: $"T-{i:D3}"); models.Add(model); - var t = queue.Enqueue(model, TestContext.Current.CancellationToken).AsTask(); + var t = queue.Enqueue(model, TestToken).AsTask(); tasks.Add(t); } @@ -464,7 +471,7 @@ public async Task ConcurrencyLimit_UpChanges_DontLoseTasks() foreach (var model in models) model.Unlock(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); // Assert: all tasks completed, no duplicates, no losses Assert.Equal(20, completedIds.Count); @@ -492,7 +499,7 @@ public async Task ConcurrencyLimit_WithCancellation_HandlesGracefully() await Task.WhenAll(enqueueTasks); - await Task.Delay(100, TestContext.Current.CancellationToken); + await Task.Delay(100, TestToken); await cts.CancelAsync(); @@ -503,9 +510,9 @@ public async Task ConcurrencyLimit_WithCancellation_HandlesGracefully() // Shutdown should complete without hanging await queue.ShutdownAsync(); - await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestToken); Assert.True(work.TotalProcessed > 0); } -} +} diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs new file mode 100644 index 00000000..95b20529 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueDiTests.cs @@ -0,0 +1,63 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace Sa.Utils.WorkQueue.Tests; + + + +public sealed record TestInput(string Id, int Value); + + +public sealed class TestWorkProcessor : ISaWork +{ + public int ExecuteCount { get; private set; } + public TestInput? LastInput { get; private set; } + + public Task Execute(TestInput input, CancellationToken cancellationToken) + { + ExecuteCount++; + LastInput = input; + return Task.CompletedTask; + } +} + + +public sealed class WorkQueueTestFixture : IDisposable +{ + public ServiceProvider ServiceProvider { get; } + + public TestWorkProcessor Processor => ServiceProvider.GetRequiredService(); + + public WorkQueueTestFixture() + { + var services = new ServiceCollection(); + services.AddSaWorkQueue(); + ServiceProvider = services.BuildServiceProvider(); + } + + public void Dispose() => ServiceProvider.Dispose(); +} + + +public class WorkQueueDiTests(WorkQueueTestFixture fixture) : IClassFixture +{ + + + [Fact] + public async Task WorkQueue_With_Di() + { + var queue = fixture.ServiceProvider.GetRequiredService>(); + Assert.NotNull(queue); + + + var input = new TestInput("test-1", 42); + await queue.Enqueue(input, TestContext.Current.CancellationToken); + await queue.WaitForIdleAsync(TestContext.Current.CancellationToken); + + // Assert + + var processor = fixture.Processor; + + Assert.Equal(1, processor.ExecuteCount); + Assert.Equal(input, processor.LastInput); + } +} diff --git a/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs new file mode 100644 index 00000000..a0b5bfd5 --- /dev/null +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueRecoveryTests.cs @@ -0,0 +1,149 @@ +namespace Sa.Utils.WorkQueue.Tests; + +public class SaWorkQueueRecoveryTests +{ + static CancellationToken TestToken => TestContext.Current.CancellationToken; + + private class TestProcessor(int failOnValue = -1) : ISaWork + { + public int ProcessedCount; + + public Task Execute(int input, CancellationToken _) + { + if (input == failOnValue) throw new InvalidOperationException("Simulated processing failure"); + Interlocked.Increment(ref ProcessedCount); + return Task.CompletedTask; + } + } + + [Fact] + public async Task ForceCancelReaders_ThenResetConcurrency_ReadersRecoverAndProcessItems() + { + // Arrange + var options = new SaWorkQueueOptions(new TestProcessor()) + { + ConcurrencyLimit = 2, + MaxConcurrency = 4 + }; + + using var queue = new SaWorkQueue(options); + + // Добавим несколько задач + await queue.Enqueue(1, TestToken); + await queue.Enqueue(2, TestToken); + await queue.WaitForIdleAsync(TestToken); + + Assert.Equal(2, ((TestProcessor)options.Processor).ProcessedCount); + + await queue.ForceCancelReadersAsync(); + await Task.Delay(100, TestToken); + + Assert.True(queue.IsIdle()); + + + queue.ConcurrencyLimit = 2; + + await Task.Delay(100, TestToken); + + await queue.Enqueue(3, TestToken); + await queue.WaitForIdleAsync(TestToken); + Assert.Equal(3, ((TestProcessor)options.Processor).ProcessedCount); + } + + [Fact] + public async Task FaultedItem_DoesNotBreakReader_QueueContinuesProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 1, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.Continue + }; + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.Enqueue(2, TestToken); // OK после fail + + await queue.WaitForIdleAsync(TestToken); + + // Assert + Assert.Equal(2, processor.ProcessedCount); + Assert.Equal(new[] { SaWorkStatus.Running, SaWorkStatus.Completed, // item 1 + SaWorkStatus.Running, SaWorkStatus.Faulted, // item -1 + SaWorkStatus.Running, SaWorkStatus.Completed }, // item 2 + statuses); + + Assert.True(queue.IsIdle()); + } + + + [Fact] + public async Task FaultedItem_BreakReader_QueueContinuesProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 2, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.StopReader + }; + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.WaitForIdleAsync(TestToken); + + await queue.Enqueue(1, TestToken); // OK + + await queue.WaitForIdleAsync(TestToken); + + // Assert + Assert.Equal(2, processor.ProcessedCount); + + Assert.Equal(1, queue.ConcurrencyLimit); + } + + [Fact] + public async Task FaultedItem_ShutdownQueue_QueueFailedProcessing() + { + // Arrange + var statuses = new List(); + var processor = new TestProcessor(failOnValue: -1); + var options = new SaWorkQueueOptions(processor) + { + ConcurrencyLimit = 2, + StatusChanged = (item, status, ex) => statuses.Add(status), + HandleItemFaulted = (_, __) => SaExecutionErrorStrategy.ShutdownQueue + }; + + using var queue = new SaWorkQueue(options); + + // Act + await queue.Enqueue(1, TestToken); // OK + + await queue.Enqueue(-1, TestToken); // Fail + + await queue.WaitForIdleAsync(TestToken); + + + await Assert.ThrowsAsync(async () => + { + await queue.Enqueue(2, cancellationToken: CancellationToken.None); + }); + + Assert.False(queue.IsEnabled); + Assert.NotNull(queue.ShutdownError); + } +} diff --git a/src/Tests/SaTests/Classes/WorkQueueTests.cs b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs similarity index 71% rename from src/Tests/SaTests/Classes/WorkQueueTests.cs rename to src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs index 2f7db73f..40ad9b92 100644 --- a/src/Tests/SaTests/Classes/WorkQueueTests.cs +++ b/src/Tests/Sa.Utils.WorkQueue.Tests/WorkQueueTests.cs @@ -1,6 +1,4 @@ -using Sa.Classes; - -namespace SaTests.Classes; +namespace Sa.Utils.WorkQueue.Tests; public class WorkQueueTests { @@ -10,7 +8,7 @@ private sealed class TestModel public bool WasProcessed { get; set; } } - private sealed class TestWork : IWork + private sealed class TestWork : ISaWork { public Task Execute(TestModel model, CancellationToken cancellationToken) { @@ -19,18 +17,21 @@ public Task Execute(TestModel model, CancellationToken cancellationToken) } } - private sealed class TestWorkWithDelay(TimeSpan delay) : IWork + private sealed class TestWorkWithDelay(TimeSpan delay) : ISaWork { private readonly TimeSpan _delay = delay; public async Task Execute(TestModel model, CancellationToken cancellationToken) { await Task.Delay(_delay, cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + model.WasProcessed = true; } } - private sealed class TestWorkThatThrows : IWork + private sealed class TestWorkThatThrows : ISaWork { public Task Execute(TestModel model, CancellationToken cancellationToken) { @@ -46,7 +47,7 @@ public async Task Enqueue_SingleTask_ExecutesAndCompletes() // Arrange var model = new TestModel(); var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(50)); - using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); // Act await queue.Enqueue(model, cancellationToken: TestToken); @@ -65,7 +66,7 @@ public async Task Enqueue_WaitForIdle_Completed() // Arrange var model = new TestModel(); var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(10)); - using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); await queue.WaitForIdleAsync(cancellationToken: TestToken); // Act @@ -84,7 +85,7 @@ public async Task Enqueue_MultipleTasks_AllExecuted() { // Arrange var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(30)); - using var queue = new WorkQueue(WorkQueueOptions.Create(processor)); + using var queue = new SaWorkQueue(SaWorkQueueOptions.Create(processor)); var models = new List { new(), new(), new() @@ -109,7 +110,7 @@ public async Task ConcurrencyLimit_Respected() var processor = new TestWorkWithDelay(delay); var concurrencyLimit = 3; - using var queue = new WorkQueue(WorkQueueOptions + using var queue = new SaWorkQueue(SaWorkQueueOptions .Create(processor) .WithConcurrencyLimit(concurrencyLimit)); @@ -135,12 +136,12 @@ public async Task ConcurrencyLimit_Respected() [Fact] public async Task FaultedTask_StatusIsFaulted() { - List<(WorkStatus Status, Exception? LastError)> changes = []; + List<(SaWorkStatus Status, Exception? LastError)> changes = []; // Arrange var processor = new TestWorkThatThrows(); - using var queue = new WorkQueue(WorkQueueOptions + using var queue = new SaWorkQueue(SaWorkQueueOptions .Create(processor) .WithStatusCallback((_, s, e) => changes.Add((s, e)))); @@ -152,23 +153,23 @@ public async Task FaultedTask_StatusIsFaulted() // Assert - Assert.Contains(changes, x => x.Status == WorkStatus.Faulted); + Assert.Contains(changes, x => x.Status == SaWorkStatus.Faulted); Assert.Contains(changes, x => x.LastError is InvalidOperationException); Assert.Contains(changes, x => x.LastError?.Message == "Test exception"); } [Fact] - public async Task CancelledTask_StatusIsCancelled() + public async Task CancelledTask_StatusIsAborted() { - List statuses = []; + List statuses = []; // Arrange using var cts = new CancellationTokenSource(); var processor = new TestWorkWithDelay(TimeSpan.FromSeconds(5)); - using var queue = new WorkQueue(WorkQueueOptions + using var queue = new SaWorkQueue(SaWorkQueueOptions .Create(processor) .WithStatusCallback((_, s, _) => statuses.Add(s))); @@ -182,24 +183,31 @@ public async Task CancelledTask_StatusIsCancelled() // Assert - Assert.Contains(WorkStatus.Cancelled, statuses); + Assert.Contains(SaWorkStatus.Aborted, statuses); } [Fact] public async Task ShutdownAsync_StopsProcessing() { - List errors = []; + List errors = []; // Arrange - var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(500)); + var processor = new TestWorkWithDelay(TimeSpan.FromMilliseconds(300)); - using var queue = new WorkQueue(WorkQueueOptions + using var queue = new SaWorkQueue(SaWorkQueueOptions .Create(processor) - .WithStatusCallback((m, s, e) => errors.Add(e))); + .WithStatusCallback((m, s, e) => + { + if (s == SaWorkStatus.Cancelled) + errors.Add(s); + })); for (int i = 0; i < 5; i++) await queue.Enqueue(new TestModel(), cancellationToken: TestToken); + + await Task.Delay(50, TestToken); + // Act await queue.ShutdownAsync(); @@ -215,8 +223,7 @@ await Assert.ThrowsAsync(async () => await queue.WaitForIdleAsync(TestToken); Assert.Equal(0, queue.QueueTasks); - var canceled = errors.Count(c => c is OperationCanceledException); - Assert.Equal(5, canceled); + Assert.Equal(5, errors.Count); } @@ -225,13 +232,13 @@ await Assert.ThrowsAsync(async () => public async Task Observer_InvokedOnStateChange() { - List statuses = []; + List statuses = []; // Arrange var processor = new TestWork(); - using var queue = new WorkQueue( - WorkQueueOptions + using var queue = new SaWorkQueue( + SaWorkQueueOptions .Create(processor) .WithStatusCallback((_, s, _) => statuses.Add(s))); @@ -243,8 +250,8 @@ public async Task Observer_InvokedOnStateChange() // Assert Assert.True(statuses.Count >= 2); // Running → Completed - Assert.Contains(statuses, x => x == WorkStatus.Running); - Assert.Contains(statuses, x => x == WorkStatus.Completed); + Assert.Contains(statuses, x => x == SaWorkStatus.Running); + Assert.Contains(statuses, x => x == SaWorkStatus.Completed); } [Fact] @@ -252,8 +259,8 @@ public async Task DisposeAsync_CleansUpResources() { // Arrange var processor = new TestWork(); - var queue = new WorkQueue( - WorkQueueOptions.Create(processor)); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(processor)); await queue.Enqueue(new TestModel(), cancellationToken: TestToken); await queue.DisposeAsync(); @@ -265,8 +272,8 @@ await Assert.ThrowsAsync(() [Fact] public void ConcurrencyLimit_InvalidValue_Clamp() { - var queue = new WorkQueue( - WorkQueueOptions.Create(new TestWork())) + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(new TestWork())) { ConcurrencyLimit = -1 }; @@ -279,11 +286,26 @@ public void ConcurrencyLimit_InvalidValue_Clamp() [Fact] public async Task Enqueue_Disposed_Throws() { - var queue = new WorkQueue( - WorkQueueOptions.Create(new TestWork())); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(new TestWork())); queue.Dispose(); await Assert.ThrowsAsync(() => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); } + + [Fact] + public async Task ShutdownAsync_CleansUpResources() + { + // Arrange + var processor = new TestWork(); + var queue = new SaWorkQueue( + SaWorkQueueOptions.Create(processor)); + + await queue.Enqueue(new TestModel(), cancellationToken: TestToken); + await queue.ShutdownAsync(); + + await Assert.ThrowsAsync(() + => queue.Enqueue(new TestModel(), cancellationToken: TestToken).AsTask()); + } }