diff --git a/src/Jhoose.Security/Configuration/ReportingOptions.cs b/src/Jhoose.Security/Configuration/ReportingOptions.cs index fed581cb..16b4ad1e 100644 --- a/src/Jhoose.Security/Configuration/ReportingOptions.cs +++ b/src/Jhoose.Security/Configuration/ReportingOptions.cs @@ -6,6 +6,13 @@ public class ReportingOptions { public int RetainDays { get; set; } = 30; + /// + /// Maximum rows the purge job deletes per batch. Set to 0 to disable batching + /// (issue a single unbounded DELETE, matching pre-batching behavior). + /// Only the SQL provider honors this; ElasticSearch handles bulk deletes natively. + /// + public int PurgeBatchSize { get; set; } = 5000; + public string UseProvider { get; set; } = string.Empty; public string ConnectionString { get; set; } = string.Empty; diff --git a/src/Jhoose.Security/Features/Reporting/Database/SqlDatabaseReportingRepository.cs b/src/Jhoose.Security/Features/Reporting/Database/SqlDatabaseReportingRepository.cs index b1f71f13..1116eb1e 100644 --- a/src/Jhoose.Security/Features/Reporting/Database/SqlDatabaseReportingRepository.cs +++ b/src/Jhoose.Security/Features/Reporting/Database/SqlDatabaseReportingRepository.cs @@ -100,13 +100,27 @@ await sqlHelper.ExecuteStoredProcedure("GetSecurityReportSummary", parameters, ( } } - public async Task PurgeReporingData(DateTime beforeDate) + public async Task PurgeReporingData(DateTime beforeDate, int? batchSize = null) { try { - var sqlCommand = "DELETE FROM SecurityReportTo WHERE RecievedAt < @BeforeDate"; + // Unbatched path preserves prior behavior for callers that pass no batch size. + if (!batchSize.HasValue || batchSize.Value <= 0) + { + var sqlCommand = "DELETE FROM SecurityReportTo WHERE RecievedAt < @BeforeDate"; + + return await sqlHelper.ExecuteNonQuery( + sqlCommand, + sqlHelper.CreateParameter("BeforeDate", SqlDbType.DateTime, beforeDate)); + } + + // Single batch: caller is expected to loop until rows == 0. Keeps each + // statement's transaction small enough to commit before the SqlCommand + // timeout, instead of rolling back a multi-hour DELETE. + var batchedSql = "DELETE TOP (@BatchSize) FROM SecurityReportTo WHERE RecievedAt < @BeforeDate"; return await sqlHelper.ExecuteNonQuery( - sqlCommand, + batchedSql, + sqlHelper.CreateParameter("BatchSize", SqlDbType.Int, batchSize.Value), sqlHelper.CreateParameter("BeforeDate", SqlDbType.DateTime, beforeDate)); } catch (Exception ex) diff --git a/src/Jhoose.Security/Features/Reporting/ElasticSearch/ElasticSearchReportingRepository.cs b/src/Jhoose.Security/Features/Reporting/ElasticSearch/ElasticSearchReportingRepository.cs index 6221d7a7..6c788d72 100644 --- a/src/Jhoose.Security/Features/Reporting/ElasticSearch/ElasticSearchReportingRepository.cs +++ b/src/Jhoose.Security/Features/Reporting/ElasticSearch/ElasticSearchReportingRepository.cs @@ -119,8 +119,13 @@ public async Task GetDashboardSummary(DashboardSummary summary return summary; } - public async Task PurgeReporingData(DateTime beforeDate) + public async Task PurgeReporingData(DateTime beforeDate, int? batchSize = null) { + // batchSize is ignored: Elasticsearch DeleteByQuery handles large deletions + // server-side without the rollback risk SQL has, so the caller's batching + // loop is unnecessary here. The parameter exists only to satisfy the interface. + _ = batchSize; + var response = await this.client.Value.DeleteByQueryAsync>(d => d.Query(query => query.Bool(b => b.Must(m => m.Range(r => r.DateRange(dr => dr.Field(f => f.RecievedAt).Gte(beforeDate))))))); var deletedCount = response.Deleted ?? 0; diff --git a/src/Jhoose.Security/Features/Reporting/IReportingRepository.cs b/src/Jhoose.Security/Features/Reporting/IReportingRepository.cs index e6f3985c..cdf523df 100644 --- a/src/Jhoose.Security/Features/Reporting/IReportingRepository.cs +++ b/src/Jhoose.Security/Features/Reporting/IReportingRepository.cs @@ -18,7 +18,15 @@ public interface IReportingRepository Task GetDashboardSummary(DashboardSummary summary); - Task PurgeReporingData(DateTime beforeDate); + /// + /// Deletes reporting rows older than . + /// When is provided and greater than zero, providers + /// that support it should delete at most that many rows per call so callers can + /// loop and bound the work each query does (avoiding command-timeout rollbacks on + /// large tables). When null, the call behaves as before and deletes everything in + /// one statement. + /// + Task PurgeReporingData(DateTime beforeDate, int? batchSize = null); Task Search(CspSearchParams searchParams); } \ No newline at end of file diff --git a/src/Jhoose.Security/Features/Reporting/Jobs/PurgeReporintgDataJob.cs b/src/Jhoose.Security/Features/Reporting/Jobs/PurgeReporintgDataJob.cs index 87185028..e6e3e86a 100644 --- a/src/Jhoose.Security/Features/Reporting/Jobs/PurgeReporintgDataJob.cs +++ b/src/Jhoose.Security/Features/Reporting/Jobs/PurgeReporintgDataJob.cs @@ -16,6 +16,7 @@ public class PurgeReporintgDataJob : ScheduledJobBase private readonly IReportingRepositoryFactory reportingRepositoryFactory; private readonly IOptions options; private readonly ILogger logger; + private bool stopSignaled; public PurgeReporintgDataJob(IReportingRepositoryFactory reportingRepositoryFactory, IOptions options, @@ -24,8 +25,11 @@ public PurgeReporintgDataJob(IReportingRepositoryFactory reportingRepositoryFact this.reportingRepositoryFactory = reportingRepositoryFactory; this.options = options; this.logger = logger; + this.IsStoppable = true; } + public override void Stop() => stopSignaled = true; + public override string Execute() { var reportingRepository = reportingRepositoryFactory.GetReportingRepository(); @@ -42,9 +46,37 @@ public override string Execute() } var beforeDate = DateTime.UtcNow.AddDays(options.Value.RetainDays * -1); - var purged = reportingRepository.PurgeReporingData(beforeDate).Result; + var batchSize = options.Value.PurgeBatchSize; + + // PurgeBatchSize <= 0 disables batching and preserves the original + // single-DELETE behavior for anyone who relied on it. + if (batchSize <= 0) + { + var purgedOnce = reportingRepository.PurgeReporingData(beforeDate).Result; + return $"Purged {purgedOnce} records, from before {beforeDate}"; + } + + var totalPurged = 0; + var batches = 0; + + OnStatusChanged($"Purging reporting rows older than {beforeDate:u} in batches of {batchSize}..."); + + while (!stopSignaled) + { + var purgedInBatch = reportingRepository.PurgeReporingData(beforeDate, batchSize).Result; + if (purgedInBatch <= 0) + { + break; + } + + totalPurged += purgedInBatch; + batches++; + OnStatusChanged($"Batch {batches}: purged {purgedInBatch} (total {totalPurged})."); + } - return $"Purged {purged} records, from before {beforeDate}"; + return stopSignaled + ? $"Stopped. Purged {totalPurged} records, from before {beforeDate} across {batches} batches." + : $"Purged {totalPurged} records, from before {beforeDate} across {batches} batches."; } catch (Exception ex) {