From a68172a285aa3f64cf1772d6e7e3d28f792504f9 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 23 Jul 2026 16:24:04 +0100 Subject: [PATCH 1/4] Add additional AdjunctQueue endpoints --- .../Integration/CustomerAdjunctQueueTests.cs | 220 ++++++++++++++++++ .../Features/AdjunctQueues/AdjunctQueryX.cs | 9 + .../Converters/AdjunctQueueConverter.cs | 23 ++ .../CustomerAdjunctQueueController.cs | 80 +++++++ .../Requests/GetActiveAdjunctBatches.cs | 36 +++ .../Requests/GetAdjunctBatches.cs | 36 +++ .../Requests/GetCustomerAdjunctQueue.cs | 27 +++ .../Requests/GetRecentAdjunctBatches.cs | 34 +++ .../DLCS.HydraModel/CustomerAdjunctQueue.cs | 103 ++++++++ .../DLCS.Model/Processing/AdjunctQueue.cs | 9 + .../Processing/ICustomerQueueRepository.cs | 8 + .../Processing/CustomerQueueRepository.cs | 23 ++ .../Ingest/Handlers/IngestHandlerTests.cs | 28 +++ .../Engine/Ingest/IngestHandler.cs | 12 +- .../Integration/DlcsDatabaseFixture.cs | 2 + 15 files changed, 647 insertions(+), 3 deletions(-) create mode 100644 src/protagonist/API/Features/AdjunctQueues/Converters/AdjunctQueueConverter.cs create mode 100644 src/protagonist/API/Features/AdjunctQueues/Requests/GetActiveAdjunctBatches.cs create mode 100644 src/protagonist/API/Features/AdjunctQueues/Requests/GetAdjunctBatches.cs create mode 100644 src/protagonist/API/Features/AdjunctQueues/Requests/GetCustomerAdjunctQueue.cs create mode 100644 src/protagonist/API/Features/AdjunctQueues/Requests/GetRecentAdjunctBatches.cs create mode 100644 src/protagonist/DLCS.HydraModel/CustomerAdjunctQueue.cs create mode 100644 src/protagonist/DLCS.Model/Processing/AdjunctQueue.cs diff --git a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs index a153461fc..2d4072f6d 100644 --- a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs +++ b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs @@ -11,6 +11,7 @@ using DLCS.AWS.SNS.Messaging; using DLCS.Model.Assets; using DLCS.Model.Messaging; +using DLCS.Model.Processing; using DLCS.Repository; using DLCS.Web.Response; using FakeItEasy; @@ -23,6 +24,7 @@ using Test.Helpers.Integration; using Test.Helpers.Integration.Infrastructure; using AdjunctBatch = DLCS.HydraModel.AdjunctBatch; +using CustomerAdjunctQueue = DLCS.HydraModel.CustomerAdjunctQueue; namespace API.Tests.Integration; @@ -420,6 +422,224 @@ public async Task PostAdjunctBatch_Returns201_AdjunctHasBatchIdSet() adjunct.Batch.Should().Be(batchId, "adjunct should reference the created batch"); } + [Fact] + public async Task GetAdjunctQueue_Returns404_WhenQueueNotFound() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.NotFound); + } + + [Fact] + public async Task GetAdjunctQueue_Returns200_WithSizeFromQueueRow_WhenNoBatches() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 7 }); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var queue = await response.ReadAsHydraResponseAsync(); + queue.Size.Should().Be(7); + queue.BatchesWaiting.Should().Be(0); + queue.AdjunctsWaiting.Should().Be(0); + } + + [Fact] + public async Task GetAdjunctQueue_Returns200_WithBatchesWaitingAndAdjunctsWaiting_ExcludingInProgressAndFinishedBatches() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 10 }); + // finished - excluded + await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 5, completed: 5, + finished: DateTime.UtcNow); + // unfinished, nothing started yet - counts as waiting + await dbContext.AdjunctBatches.AddTestAdjunctBatch(2, assetId.Customer, count: 5, completed: 0); + // unfinished but already partially processed - excluded, still being worked on + await dbContext.AdjunctBatches.AddTestAdjunctBatch(3, assetId.Customer, count: 8, completed: 6); + // unfinished but has an error recorded - also excluded, platform has started working on it + await dbContext.AdjunctBatches.AddTestAdjunctBatch(4, assetId.Customer, count: 3, errors: 1); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var queue = await response.ReadAsHydraResponseAsync(); + queue.Size.Should().Be(10); + queue.BatchesWaiting.Should().Be(1, "only batch 2 has not started processing"); + queue.AdjunctsWaiting.Should().Be(5, "only batch 2's 5 adjuncts have not started processing"); + } + + [Fact] + public async Task GetAdjunctQueue_Links_ResolveToWorkingCollectionEndpoints() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 0 }); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 1, completed: 0); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(2, assetId.Customer, count: 1, completed: 1, + finished: DateTime.UtcNow); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue"); + response.StatusCode.Should().Be(HttpStatusCode.OK); + var queue = await response.ReadAsHydraResponseAsync(); + + // Assert + queue.Batches.Should().EndWith("/adjunctQueue/batches"); + queue.Active.Should().EndWith("/adjunctQueue/active"); + queue.Recent.Should().EndWith("/adjunctQueue/recent"); + + var batches = await (await httpClient.AsCustomer(assetId.Customer).GetAsync(queue.Batches)) + .ReadAsHydraResponseAsync>(); + batches.TotalItems.Should().Be(2); + + var active = await (await httpClient.AsCustomer(assetId.Customer).GetAsync(queue.Active)) + .ReadAsHydraResponseAsync>(); + active.TotalItems.Should().Be(1); + + var recent = await (await httpClient.AsCustomer(assetId.Customer).GetAsync(queue.Recent)) + .ReadAsHydraResponseAsync>(); + recent.TotalItems.Should().Be(1); + } + + [Fact] + public async Task GetAdjunctBatches_Returns200_Empty_WhenNoBatches() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/batches"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Should().BeEmpty(); + } + + [Fact] + public async Task GetAdjunctBatches_Returns200_MostRecentlySubmittedFirst_ByDefault() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + var earlier = DateTime.UtcNow.AddMinutes(-10); + var later = DateTime.UtcNow; + await dbContext.AdjunctBatches.AddTestAdjunctBatch(101, assetId.Customer, submitted: earlier, + finished: DateTime.UtcNow); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(102, assetId.Customer, submitted: later); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/batches"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(102, 101); + } + + [Fact] + public async Task GetAdjunctBatches_Returns200_OrdersAscending_WhenOrderByRequested() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + var earlier = DateTime.UtcNow.AddMinutes(-10); + var later = DateTime.UtcNow; + await dbContext.AdjunctBatches.AddTestAdjunctBatch(111, assetId.Customer, submitted: earlier); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(112, assetId.Customer, submitted: later); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/batches?orderBy=submitted"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(111, 112); + } + + [Fact] + public async Task GetActiveAdjunctBatches_Returns200_OnlyUnfinishedBatches() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(201, assetId.Customer); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(202, assetId.Customer, finished: DateTime.UtcNow); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/active"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Should().ContainSingle(b => b.GetLastPathElementAsInt() == 201); + } + + [Fact] + public async Task GetActiveAdjunctBatches_Returns200_OrdersDescending_WhenOrderByDescendingRequested() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + var earlier = DateTime.UtcNow.AddMinutes(-10); + var later = DateTime.UtcNow; + await dbContext.AdjunctBatches.AddTestAdjunctBatch(211, assetId.Customer, submitted: earlier); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(212, assetId.Customer, submitted: later); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/active?orderByDescending=submitted"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(212, 211); + } + + [Fact] + public async Task GetRecentAdjunctBatches_Returns200_OnlyFinishedBatches_OrderedByFinishedDescending() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(301, assetId.Customer); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(302, assetId.Customer, + finished: DateTime.UtcNow.AddDays(-7)); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(303, assetId.Customer, finished: DateTime.UtcNow); + await dbContext.SaveChangesAsync(); + + // Act + var response = await httpClient.AsCustomer(assetId.Customer) + .GetAsync($"/customers/{assetId.Customer}/adjunctQueue/recent"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var batches = await response.ReadAsHydraResponseAsync>(); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(303, 302); + } + [Fact] public async Task GetAdjunctBatch_Returns404_WhenBatchNotFound() { diff --git a/src/protagonist/API/Features/AdjunctQueues/AdjunctQueryX.cs b/src/protagonist/API/Features/AdjunctQueues/AdjunctQueryX.cs index 9badcefe4..0032a98d9 100644 --- a/src/protagonist/API/Features/AdjunctQueues/AdjunctQueryX.cs +++ b/src/protagonist/API/Features/AdjunctQueues/AdjunctQueryX.cs @@ -16,4 +16,13 @@ public static IQueryable AsOrderedAdjunctQuery(this IQueryable => orderable.Descending ? query.OrderByDescending(a => a.Created) : query.OrderBy(a => a.Created); + + /// + /// Order adjunct batches by Submitted date - currently the only supported orderBy field for adjunct batches. + /// Any other/unrecognised orderBy value is silently ignored and Submitted is used. + /// + public static IQueryable AsOrderedAdjunctBatchQuery(this IQueryable query, IOrderableRequest orderable) + => orderable.Descending + ? query.OrderByDescending(b => b.Submitted) + : query.OrderBy(b => b.Submitted); } diff --git a/src/protagonist/API/Features/AdjunctQueues/Converters/AdjunctQueueConverter.cs b/src/protagonist/API/Features/AdjunctQueues/Converters/AdjunctQueueConverter.cs new file mode 100644 index 000000000..553c06b68 --- /dev/null +++ b/src/protagonist/API/Features/AdjunctQueues/Converters/AdjunctQueueConverter.cs @@ -0,0 +1,23 @@ +using HydraAdjunctQueue = DLCS.HydraModel.CustomerAdjunctQueue; +using EntityAdjunctQueue = DLCS.Model.Processing.AdjunctQueue; + +namespace API.Features.AdjunctQueues.Converters; + +/// +/// Conversion between API and EF forms of AdjunctQueue resource +/// +public static class AdjunctQueueConverter +{ + /// + /// Convert AdjunctQueue entity to API resource + /// + public static HydraAdjunctQueue ToHydra(this EntityAdjunctQueue adjunctQueue, string baseUrl) + { + var hydra = new HydraAdjunctQueue(baseUrl, adjunctQueue.Customer); + hydra.Size = adjunctQueue.Size; + hydra.BatchesWaiting = adjunctQueue.BatchesWaiting; + hydra.AdjunctsWaiting = adjunctQueue.AdjunctsWaiting; + + return hydra; + } +} diff --git a/src/protagonist/API/Features/AdjunctQueues/CustomerAdjunctQueueController.cs b/src/protagonist/API/Features/AdjunctQueues/CustomerAdjunctQueueController.cs index a4c9c4ce9..e1170a2e9 100644 --- a/src/protagonist/API/Features/AdjunctQueues/CustomerAdjunctQueueController.cs +++ b/src/protagonist/API/Features/AdjunctQueues/CustomerAdjunctQueueController.cs @@ -24,6 +24,86 @@ public class CustomerAdjunctQueueController( IMediator mediator) : HydraController(options.Value, mediator) { + /// + /// Get details of default customer adjunct queue + /// + /// Id of customer to get adjunct queue details for + /// Current cancellation token + /// Hydra JSON-LD CustomerAdjunctQueue object + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(CustomerAdjunctQueue))] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task GetCustomerAdjunctQueue([FromRoute] int customerId, + CancellationToken cancellationToken) + { + return await HandleFetch( + new GetCustomerAdjunctQueue(customerId), + queue => queue.ToHydra(GetUrlRoots().BaseUrl), + errorTitle: "Get Customer Adjunct Queue failed", + cancellationToken: cancellationToken); + } + + /// + /// Get details of all customer adjunct batches. + /// + /// Supports ?page= and ?pageSize= query parameters for paging + /// + /// Id of customer + /// Current cancellation token + /// Hydra JSON-LD collection of AdjunctBatch objects + [HttpGet] + [Route("batches")] + [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(HydraCollection))] + public async Task GetAdjunctBatches([FromRoute] int customerId, CancellationToken cancellationToken) + { + return await HandlePagedFetch( + new GetAdjunctBatches(customerId), + batch => batch.ToHydra(GetUrlRoots().BaseUrl), + errorTitle: "Get adjunct batches failed", + cancellationToken: cancellationToken); + } + + /// + /// Get details of customer active adjunct batches. An "active" batch is one that is incomplete. + /// + /// Supports ?page= and ?pageSize= query parameters for paging + /// + /// Id of customer + /// Current cancellation token + /// Hydra JSON-LD collection of AdjunctBatch objects + [HttpGet] + [Route("active")] + [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(HydraCollection))] + public async Task GetActiveAdjunctBatches([FromRoute] int customerId, CancellationToken cancellationToken) + { + return await HandlePagedFetch( + new GetActiveAdjunctBatches(customerId), + batch => batch.ToHydra(GetUrlRoots().BaseUrl), + errorTitle: "Get active adjunct batches failed", + cancellationToken: cancellationToken); + } + + /// + /// Get details of customer recent adjunct batches. These are all batches that are finished, ordered by latest + /// finished. + /// + /// Supports ?page= and ?pageSize= query parameters for paging + /// + /// Id of customer + /// Current cancellation token + /// Hydra JSON-LD collection of AdjunctBatch objects + [HttpGet] + [Route("recent")] + [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(HydraCollection))] + public async Task GetRecentAdjunctBatches([FromRoute] int customerId, CancellationToken cancellationToken) + { + return await HandlePagedFetch( + new GetRecentAdjunctBatches(customerId), + batch => batch.ToHydra(GetUrlRoots().BaseUrl), + errorTitle: "Get recent adjunct batches failed", + cancellationToken: cancellationToken); + } + /// /// Get details of specified adjunct batch. /// diff --git a/src/protagonist/API/Features/AdjunctQueues/Requests/GetActiveAdjunctBatches.cs b/src/protagonist/API/Features/AdjunctQueues/Requests/GetActiveAdjunctBatches.cs new file mode 100644 index 000000000..596d7fd79 --- /dev/null +++ b/src/protagonist/API/Features/AdjunctQueues/Requests/GetActiveAdjunctBatches.cs @@ -0,0 +1,36 @@ +using API.Infrastructure.Page; +using API.Infrastructure.Requests; +using DLCS.Model.Assets; +using DLCS.Repository; +using MediatR; +using Microsoft.EntityFrameworkCore; + +namespace API.Features.AdjunctQueues.Requests; + +/// +/// Get a paged list of all Active (incomplete) adjunct batches for customer +/// +public class GetActiveAdjunctBatches(int customerId) : IRequest>>, IPagedRequest, + IOrderableRequest +{ + public int CustomerId { get; } = customerId; + public int Page { get; set; } + public int PageSize { get; set; } + public string? Field { get; set; } + public bool Descending { get; set; } +} + +public class GetActiveAdjunctBatchesHandler(DlcsContext dlcsContext) + : IRequestHandler>> +{ + public async Task>> Handle(GetActiveAdjunctBatches request, + CancellationToken cancellationToken) + { + var result = await dlcsContext.AdjunctBatches.AsNoTracking().CreatePagedResult(request, + q => q.Where(b => b.Customer == request.CustomerId && b.Finished == null), + batches => batches.AsOrderedAdjunctBatchQuery(request), + cancellationToken: cancellationToken); + + return FetchEntityResult>.Success(result); + } +} diff --git a/src/protagonist/API/Features/AdjunctQueues/Requests/GetAdjunctBatches.cs b/src/protagonist/API/Features/AdjunctQueues/Requests/GetAdjunctBatches.cs new file mode 100644 index 000000000..cfe18001b --- /dev/null +++ b/src/protagonist/API/Features/AdjunctQueues/Requests/GetAdjunctBatches.cs @@ -0,0 +1,36 @@ +using API.Infrastructure.Page; +using API.Infrastructure.Requests; +using DLCS.Model.Assets; +using DLCS.Repository; +using MediatR; +using Microsoft.EntityFrameworkCore; + +namespace API.Features.AdjunctQueues.Requests; + +/// +/// Get a paged list of all adjunct batches for customer, most recently submitted first by default +/// +public class GetAdjunctBatches(int customerId) : IRequest>>, IPagedRequest, + IOrderableRequest +{ + public int CustomerId { get; } = customerId; + public int Page { get; set; } + public int PageSize { get; set; } + public string? Field { get; set; } + public bool Descending { get; set; } = true; +} + +public class GetAdjunctBatchesHandler(DlcsContext dlcsContext) + : IRequestHandler>> +{ + public async Task>> Handle(GetAdjunctBatches request, + CancellationToken cancellationToken) + { + var result = await dlcsContext.AdjunctBatches.AsNoTracking().CreatePagedResult(request, + q => q.Where(b => b.Customer == request.CustomerId), + batches => batches.AsOrderedAdjunctBatchQuery(request), + cancellationToken: cancellationToken); + + return FetchEntityResult>.Success(result); + } +} diff --git a/src/protagonist/API/Features/AdjunctQueues/Requests/GetCustomerAdjunctQueue.cs b/src/protagonist/API/Features/AdjunctQueues/Requests/GetCustomerAdjunctQueue.cs new file mode 100644 index 000000000..06b357968 --- /dev/null +++ b/src/protagonist/API/Features/AdjunctQueues/Requests/GetCustomerAdjunctQueue.cs @@ -0,0 +1,27 @@ +using API.Infrastructure.Requests; +using DLCS.Model.Processing; +using MediatR; + +namespace API.Features.AdjunctQueues.Requests; + +/// +/// Get details of customer adjunct queue +/// +public class GetCustomerAdjunctQueue(int customerId) : IRequest> +{ + public int CustomerId { get; } = customerId; +} + +public class GetCustomerAdjunctQueueHandler(ICustomerQueueRepository customerQueueRepository) + : IRequestHandler> +{ + public async Task> Handle(GetCustomerAdjunctQueue request, + CancellationToken cancellationToken) + { + var adjunctQueue = await customerQueueRepository.GetAdjunctQueue(request.CustomerId, cancellationToken); + + return adjunctQueue == null + ? FetchEntityResult.NotFound() + : FetchEntityResult.Success(adjunctQueue); + } +} diff --git a/src/protagonist/API/Features/AdjunctQueues/Requests/GetRecentAdjunctBatches.cs b/src/protagonist/API/Features/AdjunctQueues/Requests/GetRecentAdjunctBatches.cs new file mode 100644 index 000000000..b71c9bd3e --- /dev/null +++ b/src/protagonist/API/Features/AdjunctQueues/Requests/GetRecentAdjunctBatches.cs @@ -0,0 +1,34 @@ +using API.Infrastructure.Page; +using API.Infrastructure.Requests; +using DLCS.Model.Assets; +using DLCS.Repository; +using MediatR; +using Microsoft.EntityFrameworkCore; + +namespace API.Features.AdjunctQueues.Requests; + +/// +/// Get a paged list of all Recent (finished, ordered by finished DESC) adjunct batches for customer +/// +public class GetRecentAdjunctBatches(int customerId) : IRequest>>, IPagedRequest +{ + public int CustomerId { get; } = customerId; + public int Page { get; set; } + public int PageSize { get; set; } +} + +public class GetRecentAdjunctBatchesHandler(DlcsContext dlcsContext) + : IRequestHandler>> +{ + public async Task>> Handle(GetRecentAdjunctBatches request, + CancellationToken cancellationToken) + { + var result = await dlcsContext.AdjunctBatches.AsNoTracking().CreatePagedResult( + request, + q => q.Where(b => b.Customer == request.CustomerId && b.Finished != null), + batches => batches.OrderByDescending(b => b.Finished), + cancellationToken: cancellationToken); + + return FetchEntityResult>.Success(result); + } +} diff --git a/src/protagonist/DLCS.HydraModel/CustomerAdjunctQueue.cs b/src/protagonist/DLCS.HydraModel/CustomerAdjunctQueue.cs new file mode 100644 index 000000000..88ca52703 --- /dev/null +++ b/src/protagonist/DLCS.HydraModel/CustomerAdjunctQueue.cs @@ -0,0 +1,103 @@ +using Hydra; +using Hydra.Model; +using Newtonsoft.Json; + +namespace DLCS.HydraModel; + +[HydraClass(typeof(CustomerAdjunctQueueClass), + Description = "The customer adjunct queue resource represents an overview of outstanding adjuncts and adjunct batches.", + UriTemplate = "/customers/{0}/adjunctQueue")] +public class CustomerAdjunctQueue : DlcsResource +{ + public CustomerAdjunctQueue() + { + } + + public CustomerAdjunctQueue(string baseUrl, int customerId) + { + CustomerId = customerId; + Init(baseUrl, true, CustomerId); + } + + [JsonIgnore] + public int CustomerId { get; set; } + + [RdfProperty(Description = "Number of total adjuncts in your queue, across all batches", + Range = Names.XmlSchema.NonNegativeInteger, ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 11, PropertyName = "size")] + public int Size { get; set; } + + [RdfProperty(Description = "Number of total adjunct batches that have not been started in your queue", + Range = Names.XmlSchema.NonNegativeInteger, ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 12, PropertyName = "batchesWaiting")] + public long BatchesWaiting { get; set; } + + [RdfProperty(Description = "Number of adjuncts waiting to be processed. These are adjuncts that have been " + + "submitted but the platform has not yet started working on them; excludes " + + "adjuncts currently being processed", + Range = Names.XmlSchema.NonNegativeInteger, ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 13, PropertyName = "adjunctsWaiting")] + public long AdjunctsWaiting { get; set; } + + // Hydra Link properties + [HydraLink(Description = "All adjunct batches for customer", Range = "vocab:AdjunctBatch", ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 20, PropertyName = "batches")] + public string? Batches { get; set; } + + [HydraLink(Description = "All active adjunct batches for customer", Range = "vocab:AdjunctQueue", ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 21, PropertyName = "active")] + public string? Active { get; set; } + + [HydraLink(Description = "All recent adjunct batches for customer", Range = "vocab:AdjunctQueue", ReadOnly = true, WriteOnly = false)] + [JsonProperty(Order = 22, PropertyName = "recent")] + public string? Recent { get; set; } +} + +public class CustomerAdjunctQueueClass : Class +{ + const string operationId = "_:customer_adjunct_queue_"; + + public CustomerAdjunctQueueClass() + { + BootstrapViaReflection(typeof(CustomerAdjunctQueue)); + } + + public override void DefineOperations() + { + SupportedOperations = CommonOperations.GetStandardResourceOperations( + operationId, "CustomerAdjunctQueue", Id, "GET", "POST"); + + GetHydraLinkProperty("batches").SupportedOperations = new[] + { + new Operation + { + Id = "_:customer_adjunct_queue_batch_collection_retrieve", + Method = "GET", + Label = "Retrieves all adjunct batches for customer", + Returns = Names.Hydra.Collection + } + }; + + GetHydraLinkProperty("active").SupportedOperations = new[] + { + new Operation + { + Id = "_:customer_adjunct_queue_active_collection_retrieve", + Method = "GET", + Label = "Retrieves the customer's currently running adjunct batches.", + Returns = Names.Hydra.Collection + } + }; + + GetHydraLinkProperty("recent").SupportedOperations = new[] + { + new Operation + { + Id = "_:customer_adjunct_queue_recent_collection_retrieve", + Method = "GET", + Label = "Retrieves the recent (finished) adjunct batches for customer.", + Returns = Names.Hydra.Collection + } + }; + } +} diff --git a/src/protagonist/DLCS.Model/Processing/AdjunctQueue.cs b/src/protagonist/DLCS.Model/Processing/AdjunctQueue.cs new file mode 100644 index 000000000..ef1b34b79 --- /dev/null +++ b/src/protagonist/DLCS.Model/Processing/AdjunctQueue.cs @@ -0,0 +1,9 @@ +namespace DLCS.Model.Processing; + +public class AdjunctQueue +{ + public int Customer { get; set; } + public int Size { get; set; } + public long BatchesWaiting { get; set; } + public long AdjunctsWaiting { get; set; } +} diff --git a/src/protagonist/DLCS.Model/Processing/ICustomerQueueRepository.cs b/src/protagonist/DLCS.Model/Processing/ICustomerQueueRepository.cs index 818b68e09..f91e6c0d9 100644 --- a/src/protagonist/DLCS.Model/Processing/ICustomerQueueRepository.cs +++ b/src/protagonist/DLCS.Model/Processing/ICustomerQueueRepository.cs @@ -11,6 +11,14 @@ public interface ICustomerQueueRepository /// Task Get(int customer, string name, CancellationToken cancellationToken); + /// + /// Get object for specified customer. + /// This consists of values from both the Queue and AdjunctBatch tables. BatchesWaiting/AdjunctsWaiting only + /// count batches that have not yet started processing - unlike Size, they exclude batches currently in + /// progress + /// + Task GetAdjunctQueue(int customer, CancellationToken cancellationToken); + /// /// Increment specified queue by specified amount. /// If queue doesn't exist it will be created with Size = incrementAmount diff --git a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs index d91d85a78..8772e0ca5 100644 --- a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs +++ b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs @@ -38,6 +38,29 @@ public CustomerQueueRepository(DlcsContext dlcsContext, ILogger GetAdjunctQueue(int customer, CancellationToken cancellationToken) + { + try + { + // NOTE - unlike Get() above, BatchesWaiting/AdjunctsWaiting only count batches that haven't started + // processing yet (nothing Completed or Errored) - this excludes batches currently being worked on, + // which still count towards Size + const string sql = @"SELECT +q.""Customer"", q.""Size"", b.""BatchesWaiting"", b.""AdjunctsWaiting"" FROM ""Queues"" q, + (SELECT COUNT(""Id"") AS ""BatchesWaiting"", COALESCE(SUM(""Count""), 0) AS ""AdjunctsWaiting"" + FROM ""AdjunctBatches"" + WHERE ""Customer"" = @customer AND ""Finished"" IS NULL AND ""Completed"" = 0 AND ""Errors"" = 0) b + WHERE q.""Customer"" = @customer AND q.""Name"" = 'adjunct' +"; + return await this.QueryFirstOrDefaultAsync(sql, new { customer }); + } + catch (Exception ex) + { + logger.LogError(ex, "Error getting adjunct queue counts for customer {Customer}", customer); + return null; + } + } + public async Task IncrementSize(int customer, string name, int incrementAmount = 1, CancellationToken cancellationToken = default) { diff --git a/src/protagonist/Engine.Tests/Ingest/Handlers/IngestHandlerTests.cs b/src/protagonist/Engine.Tests/Ingest/Handlers/IngestHandlerTests.cs index 8498bfcd5..631eebb67 100644 --- a/src/protagonist/Engine.Tests/Ingest/Handlers/IngestHandlerTests.cs +++ b/src/protagonist/Engine.Tests/Ingest/Handlers/IngestHandlerTests.cs @@ -89,4 +89,32 @@ public async Task HandleMessage_ReturnsFalse_IfSuccessOrQueued(IngestResultStatu .MustHaveHappened(); success.Should().BeTrue(); } + + [Fact] + public async Task HandleMessage_DecrementsAdjunctQueue_NotDefault_ForAdjunctIngestMessage() + { + // Arrange + var body = new JsonObject + { + ["created"] = "1985-10-26T09:00:00" + }; + var queueMessage = new QueueMessage + { + Body = body, QueueName = "protagonist-adjunct", + MessageAttributes = new() { { SqsQueueUtilities.Constants.MessageAttributeNames.IngestType, IngestAdjunctRequest.IngestType } } + }; + A.CallTo(() => adjunctIngester.Ingest(A._, A._)) + .Returns(new IngestResult(new AssetId(1, 2, "fake"), IngestResultStatus.Success)); + + // Act + var success = await sut.HandleMessage(queueMessage, CancellationToken.None); + + // Assert + A.CallTo(() => adjunctIngester.Ingest(A._, A._)).MustHaveHappened(); + A.CallTo(() => customerQueueRepository.DecrementSize(A._, QueueNames.Adjunct, A._, A._)) + .MustHaveHappened(); + A.CallTo(() => customerQueueRepository.DecrementSize(A._, QueueNames.Default, A._, A._)) + .MustNotHaveHappened(); + success.Should().BeTrue(); + } } diff --git a/src/protagonist/Engine/Ingest/IngestHandler.cs b/src/protagonist/Engine/Ingest/IngestHandler.cs index 4fbc85c06..b2c0f77e2 100644 --- a/src/protagonist/Engine/Ingest/IngestHandler.cs +++ b/src/protagonist/Engine/Ingest/IngestHandler.cs @@ -46,7 +46,7 @@ private async Task HandleIngest(QueueMessage message, logger.LogDebug("Message {MessageId} handled with result {IngestResult}", message.MessageId, ingestResult.Status); - await UpdateCustomerQueue(message, ingestResult, cancellationToken); + await UpdateCustomerQueue(message, ingestResult, GetQueueName(message), cancellationToken); } // return true so that the message is deleted from the queue in all instances. @@ -56,10 +56,16 @@ private async Task HandleIngest(QueueMessage message, return true; } + private static string GetQueueName(QueueMessage message) => message switch + { + _ when typeof(T) == typeof(IngestAdjunctRequest) => QueueNames.Adjunct, + _ when message.QueueName.Contains("priority", StringComparison.OrdinalIgnoreCase) => QueueNames.Priority, + _ => QueueNames.Default + }; + private async Task UpdateCustomerQueue(QueueMessage message, - IngestResult ingestResult, CancellationToken cancellationToken) + IngestResult ingestResult, string queue, CancellationToken cancellationToken) { - var queue = message.QueueName.Contains("priority", StringComparison.OrdinalIgnoreCase) ? QueueNames.Priority : QueueNames.Default; var customer = 0; try { diff --git a/src/protagonist/Test.Helpers/Integration/DlcsDatabaseFixture.cs b/src/protagonist/Test.Helpers/Integration/DlcsDatabaseFixture.cs index d3c077b04..a86cc7df0 100644 --- a/src/protagonist/Test.Helpers/Integration/DlcsDatabaseFixture.cs +++ b/src/protagonist/Test.Helpers/Integration/DlcsDatabaseFixture.cs @@ -41,6 +41,8 @@ public void CleanUp() DbContext.Database.ExecuteSqlRaw("DELETE FROM \"AuthTokens\""); DbContext.Database.ExecuteSqlRaw("DELETE FROM \"Batches\""); DbContext.Database.ExecuteSqlRaw("DELETE FROM \"Queues\""); + DbContext.Database.ExecuteSqlRaw("DELETE FROM \"AdjunctBatchAdjuncts\""); + DbContext.Database.ExecuteSqlRaw("DELETE FROM \"AdjunctBatches\""); DbContext.Database.ExecuteSqlRaw("DELETE FROM \"EntityCounters\" WHERE \"Type\" = 'space' AND \"Customer\" != 99"); DbContext.Database.ExecuteSqlRaw("DELETE FROM \"EntityCounters\" WHERE \"Type\" = 'space-images' AND \"Customer\" != 99"); DbContext.Database.ExecuteSqlRaw("DELETE FROM \"EntityCounters\" WHERE \"Type\" = 'customer-images' AND \"Scope\" != '99'"); From d67239f38ac71d309523e6c6589b341c0325e8bd Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 27 Jul 2026 10:10:52 +0100 Subject: [PATCH 2/4] Add migration fix for queue size --- .../BackfillAdjunctQueueSizeMigrationTests.cs | 66 + ...54327_BackfillAdjunctQueueSize.Designer.cs | 1379 +++++++++++++++++ ...20260723154327_BackfillAdjunctQueueSize.cs | 30 + .../Migrations/DlcsContextModelSnapshot.cs | 4 +- .../Processing/CustomerQueueRepository.cs | 34 +- .../Engine/Ingest/IngestHandler.cs | 4 +- 6 files changed, 1502 insertions(+), 15 deletions(-) create mode 100644 src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs create mode 100644 src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs create mode 100644 src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs diff --git a/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs b/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs new file mode 100644 index 000000000..0c9b3ee43 --- /dev/null +++ b/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs @@ -0,0 +1,66 @@ +using DLCS.Model.Processing; +using DLCS.Repository.Migrations; +using Microsoft.EntityFrameworkCore; +using Test.Helpers.Integration; + +namespace DLCS.Repository.Tests.Migrations; + +/// +/// Verifies the correction SQL applied by the 20260723154327_BackfillAdjunctQueueSize migration (exposed as +/// so this test can't drift from what actually ships). +/// +[Trait("Category", "Database")] +[Collection(DatabaseCollection.CollectionName)] +public class BackfillAdjunctQueueSizeMigrationTests +{ + private readonly DlcsContext dbContext; + + public BackfillAdjunctQueueSizeMigrationTests(DlcsDatabaseFixture dbFixture) + { + dbContext = dbFixture.DbContext; + dbFixture.CleanUp(); + } + + [Fact] + public async Task CorrectionSql_ResetsAdjunctQueueRows_LeavesOtherQueueNamesAlone() + { + // Arrange + await dbContext.Queues.AddAsync(new Queue { Customer = 501, Name = "adjunct", Size = 999 }); + await dbContext.Queues.AddAsync(new Queue { Customer = 502, Name = "adjunct", Size = 12 }); + await dbContext.Queues.AddAsync(new Queue { Customer = 503, Name = "default", Size = 42 }); + await dbContext.Queues.AddAsync(new Queue { Customer = 503, Name = "priority", Size = 7 }); + await dbContext.SaveChangesAsync(); + + // Act + await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); + + // Assert + var c501 = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 501 && q.Name == "adjunct"); + c501.Size.Should().Be(0); + + var c502 = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 502 && q.Name == "adjunct"); + c502.Size.Should().Be(0); + + var c503Default = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 503 && q.Name == "default"); + c503Default.Size.Should().Be(42, "non-adjunct queue rows must not be touched"); + + var c503Priority = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 503 && q.Name == "priority"); + c503Priority.Size.Should().Be(7, "non-adjunct queue rows must not be touched"); + } + + [Fact] + public async Task CorrectionSql_IsIdempotent_WhenRunTwice() + { + // Arrange + await dbContext.Queues.AddAsync(new Queue { Customer = 601, Name = "adjunct", Size = 999 }); + await dbContext.SaveChangesAsync(); + + // Act + await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); + await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); + + // Assert + var queue = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 601 && q.Name == "adjunct"); + queue.Size.Should().Be(0); + } +} diff --git a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs new file mode 100644 index 000000000..172f31d96 --- /dev/null +++ b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs @@ -0,0 +1,1379 @@ +// +using System; +using System.Collections.Generic; +using DLCS.Repository; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DLCS.Repository.Migrations +{ + [DbContext(typeof(DlcsContext))] + [Migration("20260723154327_BackfillAdjunctQueueSize")] + partial class BackfillAdjunctQueueSize + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .UseCollation("en_US.UTF-8") + .HasAnnotation("ProductVersion", "9.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "tablefunc"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.HasSequence("adjunct_batch_id_sequence") + .HasMin(1L); + + modelBuilder.HasSequence("batch_id_sequence") + .StartsAt(570185L) + .HasMin(1L) + .HasMax(9223372036854775807L); + + modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => + { + b.Property("Id") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("AssetId") + .HasColumnType("character varying(500)"); + + b.Property("Batch") + .HasColumnType("integer"); + + b.Property("Created") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("ExternalId") + .HasColumnType("text"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("IIIFLink") + .IsRequired() + .HasColumnType("text"); + + b.Property("Ingesting") + .HasColumnType("boolean"); + + b.Property("Label") + .HasColumnType("jsonb"); + + b.Property("Language") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("MediaType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Motivation") + .HasColumnType("text"); + + b.Property("Optimised") + .HasColumnType("boolean"); + + b.Property("Origin") + .HasColumnType("text"); + + b.Property("Profile") + .HasColumnType("text"); + + b.Property("Provides") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id", "AssetId"); + + b.HasIndex("AssetId"); + + b.HasIndex("Batch"); + + b.ToTable("Adjuncts"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValueSql("nextval('adjunct_batch_id_sequence'::regclass)"); + + b.Property("Completed") + .HasColumnType("integer"); + + b.Property("Count") + .HasColumnType("integer"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("integer"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex(new[] { "Customer", "Submitted" }, "IX_AdjunctBatchesByCustomerSubmitted"); + + b.ToTable("AdjunctBatches"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatchAdjunct", b => + { + b.Property("BatchId") + .HasColumnType("integer"); + + b.Property("AdjunctId") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("AssetId") + .HasColumnType("character varying(500)"); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("BatchId", "AdjunctId", "AssetId"); + + b.HasIndex("AdjunctId", "AssetId"); + + b.ToTable("AdjunctBatchAdjuncts"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Asset", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Batch") + .HasColumnType("integer"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("DeliveryChannels") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Duration") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasDefaultValueSql("0"); + + b.Property("Error") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)") + .HasDefaultValueSql("NULL::character varying"); + + b.Property("Family") + .ValueGeneratedOnAdd() + .HasColumnType("char(1)") + .HasDefaultValueSql("'I'::\"char\""); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("Height") + .HasColumnType("integer"); + + b.Property("ImageOptimisationPolicy") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasDefaultValueSql("'fast-lossy'::character varying"); + + b.Property("Ingesting") + .HasColumnType("boolean"); + + b.PrimitiveCollection>("Manifests") + .HasColumnType("text[]"); + + b.Property("MaxUnauthorised") + .HasColumnType("integer"); + + b.Property("MaxWidth") + .HasColumnType("integer"); + + b.Property("MediaType") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasDefaultValueSql("'image/jp2'::character varying"); + + b.Property("NotForDelivery") + .HasColumnType("boolean"); + + b.Property("NumberReference1") + .HasColumnType("integer"); + + b.Property("NumberReference2") + .HasColumnType("integer"); + + b.Property("NumberReference3") + .HasColumnType("integer"); + + b.Property("OpenFullMax") + .HasColumnType("integer"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("PreservedUri") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Reference1") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Reference2") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Reference3") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Roles") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Space") + .HasColumnType("integer"); + + b.Property("Tags") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("ThumbnailPolicy") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasDefaultValueSql("'original'::character varying"); + + b.Property("Width") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Manifests"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Manifests"), "gin"); + + b.HasIndex(new[] { "Batch" }, "IX_ImagesByBatch"); + + b.HasIndex(new[] { "Id", "Customer", "Space" }, "IX_ImagesByCustomerSpace"); + + b.HasIndex(new[] { "Id", "Customer", "Error", "Batch" }, "IX_ImagesByErrors") + .HasFilter("((\"Error\" IS NOT NULL) AND ((\"Error\")::text <> ''::text))"); + + b.HasIndex(new[] { "Reference1" }, "IX_ImagesByReference1"); + + b.HasIndex(new[] { "Reference2" }, "IX_ImagesByReference2"); + + b.HasIndex(new[] { "Reference3" }, "IX_ImagesByReference3"); + + b.HasIndex(new[] { "Customer", "Space" }, "IX_ImagesBySpace"); + + b.ToTable("Images", (string)null); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Batch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValueSql("nextval('batch_id_sequence'::regclass)"); + + b.Property("Completed") + .HasColumnType("integer"); + + b.Property("Count") + .HasColumnType("integer"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("integer"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("Submitted") + .HasColumnType("timestamp with time zone"); + + b.Property("Superseded") + .HasColumnType("boolean"); + + b.HasKey("Id"); + + b.HasIndex(new[] { "Customer", "Superseded", "Submitted" }, "IX_BatchTest"); + + b.ToTable("Batches"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.BatchAsset", b => + { + b.Property("BatchId") + .HasColumnType("integer"); + + b.Property("AssetId") + .HasColumnType("character varying(500)"); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("Finished") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("BatchId", "AssetId"); + + b.HasIndex("AssetId"); + + b.ToTable("BatchAssets"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.CustomHeaders.CustomHeader", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Role") + .ValueGeneratedOnAdd() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasDefaultValueSql("NULL::character varying"); + + b.Property("Space") + .HasColumnType("integer"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.HasIndex(new[] { "Customer", "Space" }, "IX_CustomHeaders_ByCustomerSpace"); + + b.ToTable("CustomHeaders"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.ImageDeliveryChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasMaxLength(100) + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Channel") + .IsRequired() + .HasColumnType("text"); + + b.Property("DeliveryChannelPolicyId") + .HasColumnType("integer"); + + b.Property("ImageId") + .IsRequired() + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.HasIndex("DeliveryChannelPolicyId"); + + b.HasIndex("ImageId"); + + b.ToTable("ImageDeliveryChannels"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.ImageLocation", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Nas") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("S3") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.ToTable("ImageLocation", (string)null); + }); + + modelBuilder.Entity("DLCS.Model.Assets.ImageStorage", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Space") + .HasColumnType("integer"); + + b.Property("AdjunctSize") + .HasColumnType("bigint"); + + b.Property("CheckingInProgress") + .HasColumnType("boolean"); + + b.Property("LastChecked") + .HasColumnType("timestamp with time zone"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("ThumbnailSize") + .HasColumnType("bigint"); + + b.HasKey("Id", "Customer", "Space"); + + b.HasIndex(new[] { "Customer", "Space", "Id" }, "IX_ImageStorageByCustomerSpace"); + + b.ToTable("ImageStorage", (string)null); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Metadata.AssetApplicationMetadata", b => + { + b.Property("AssetId") + .HasColumnType("character varying(500)"); + + b.Property("MetadataType") + .HasColumnType("text"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("MetadataValue") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone"); + + b.HasKey("AssetId", "MetadataType"); + + b.ToTable("AssetApplicationMetadata"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.NamedQueries.NamedQuery", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Global") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Template") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.HasKey("Id"); + + b.ToTable("NamedQueries"); + }); + + modelBuilder.Entity("DLCS.Model.Auth.Entities.AuthService", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("CallToAction") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("ChildAuthService") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Description") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("Label") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(250) + .HasColumnType("character varying(250)"); + + b.Property("PageDescription") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("PageLabel") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Profile") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("RoleProvider") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Ttl") + .HasColumnType("integer") + .HasColumnName("TTL"); + + b.HasKey("Id", "Customer"); + + b.ToTable("AuthServices"); + }); + + modelBuilder.Entity("DLCS.Model.Auth.Entities.Role", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Aliases") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("AuthService") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id", "Customer"); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("DLCS.Model.Auth.Entities.RoleProvider", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("AuthService") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Configuration") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("Credentials") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("RoleProviders"); + }); + + modelBuilder.Entity("DLCS.Model.Customers.Customer", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("AcceptedAgreement") + .HasColumnType("boolean"); + + b.Property("Administrator") + .HasColumnType("boolean"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Keys") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.HasIndex("DisplayName") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Customers"); + }); + + modelBuilder.Entity("DLCS.Model.Customers.CustomerOriginStrategy", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Credentials") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Optimised") + .HasColumnType("boolean"); + + b.Property("Order") + .HasColumnType("integer"); + + b.Property("Regex") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Strategy") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.ToTable("CustomerOriginStrategies"); + }); + + modelBuilder.Entity("DLCS.Model.Customers.SignupLink", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("CustomerId") + .HasColumnType("integer"); + + b.Property("Expires") + .HasColumnType("timestamp with time zone"); + + b.Property("Note") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SignupLinks"); + }); + + modelBuilder.Entity("DLCS.Model.Customers.User", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("EncryptedPassword") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Roles") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DLCS.Model.DeliveryChannels.DefaultDeliveryChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("DeliveryChannelPolicyId") + .HasColumnType("integer"); + + b.Property("MediaType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Space") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("DeliveryChannelPolicyId"); + + b.HasIndex("Customer", "Space", "MediaType", "DeliveryChannelPolicyId") + .IsUnique(); + + b.ToTable("DefaultDeliveryChannels"); + }); + + modelBuilder.Entity("DLCS.Model.Policies.DeliveryChannelPolicy", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Channel") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("DisplayName") + .HasColumnType("text"); + + b.Property("Modified") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("PolicyData") + .HasColumnType("text"); + + b.Property("System") + .HasColumnType("boolean"); + + b.HasKey("Id"); + + b.HasIndex("Customer", "Name", "Channel") + .IsUnique(); + + b.ToTable("DeliveryChannelPolicies"); + }); + + modelBuilder.Entity("DLCS.Model.Policies.ImageOptimisationPolicy", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Global") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("TechnicalDetails") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.HasKey("Id", "Customer"); + + b.ToTable("ImageOptimisationPolicies"); + + b.HasData( + new + { + Id = "none", + Customer = 1, + Global = true, + Name = "No optimisation/transcoding", + TechnicalDetails = "no-op" + }, + new + { + Id = "use-original", + Customer = 1, + Global = true, + Name = "Use original for image-server", + TechnicalDetails = "use-original" + }); + }); + + modelBuilder.Entity("DLCS.Model.Policies.OriginStrategy", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("RequiresCredentials") + .HasColumnType("boolean"); + + b.HasKey("Id"); + + b.ToTable("OriginStrategies"); + }); + + modelBuilder.Entity("DLCS.Model.Policies.ThumbnailPolicy", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Sizes") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.HasKey("Id"); + + b.ToTable("ThumbnailPolicies"); + }); + + modelBuilder.Entity("DLCS.Model.Processing.Queue", b => + { + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Name") + .ValueGeneratedOnAdd() + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasDefaultValueSql("'default'::character varying"); + + b.Property("Size") + .HasColumnType("integer"); + + b.HasKey("Customer", "Name"); + + b.ToTable("Queues"); + }); + + modelBuilder.Entity("DLCS.Model.Spaces.Space", b => + { + b.Property("Id") + .HasColumnType("integer"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("ImageBucket") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Keep") + .HasColumnType("boolean"); + + b.Property("MaxUnauthorised") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Roles") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Tags") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("Transform") + .HasColumnType("boolean"); + + b.HasKey("Id", "Customer") + .HasName("Spaces_pkey"); + + b.ToTable("Spaces"); + }); + + modelBuilder.Entity("DLCS.Model.Storage.CustomerStorage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("LastCalculated") + .HasColumnType("timestamp with time zone"); + + b.Property("NumberOfStoredAdjuncts") + .HasColumnType("bigint"); + + b.Property("NumberOfStoredImages") + .HasColumnType("bigint"); + + b.Property("Space") + .HasColumnType("integer"); + + b.Property("StoragePolicy") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("TotalSizeOfStoredAdjuncts") + .HasColumnType("bigint"); + + b.Property("TotalSizeOfStoredImages") + .HasColumnType("bigint"); + + b.Property("TotalSizeOfThumbnails") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("Customer") + .IsUnique() + .HasDatabaseName("IX_CustomerStorage_Customer_Aggregate") + .HasFilter("\"Space\" IS NULL"); + + b.HasIndex("Customer", "Space") + .IsUnique() + .HasDatabaseName("IX_CustomerStorage_Customer_Space") + .HasFilter("\"Space\" IS NOT NULL"); + + b.ToTable("CustomerStorage", (string)null); + }); + + modelBuilder.Entity("DLCS.Model.Storage.StoragePolicy", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("MaximumNumberOfStoredImages") + .HasColumnType("bigint"); + + b.Property("MaximumTotalSizeOfStoredImages") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("StoragePolicies"); + }); + + modelBuilder.Entity("DLCS.Repository.Auth.AuthToken", b => + { + b.Property("Id") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("BearerToken") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("CookieId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Expires") + .HasColumnType("timestamp with time zone"); + + b.Property("LastChecked") + .HasColumnType("timestamp with time zone"); + + b.Property("SessionUserId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Ttl") + .HasColumnType("integer") + .HasColumnName("TTL"); + + b.HasKey("Id"); + + b.HasIndex(new[] { "BearerToken" }, "IX_AuthTokens_BearerToken"); + + b.HasIndex(new[] { "CookieId" }, "IX_AuthTokens_CookieId"); + + b.ToTable("AuthTokens"); + }); + + modelBuilder.Entity("DLCS.Repository.Auth.SessionUser", b => + { + b.Property("Id") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Roles") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.HasKey("Id"); + + b.ToTable("SessionUsers"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.ActivityGroup", b => + { + b.Property("Group") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Inhabitant") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Since") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Group"); + + b.ToTable("ActivityGroups"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.CustomerImageServer", b => + { + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("ImageServer") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Customer"); + + b.ToTable("CustomerImageServers"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.EntityCounter", b => + { + b.Property("Type") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Scope") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Customer") + .HasColumnType("integer"); + + b.Property("Next") + .HasColumnType("bigint"); + + b.HasKey("Type", "Scope", "Customer"); + + b.ToTable("EntityCounters"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.ImageServer", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("InfoJsonTemplate") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.HasKey("Id"); + + b.ToTable("ImageServers"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.InfoJsonTemplate", b => + { + b.Property("Id") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Template") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.HasKey("Id"); + + b.ToTable("InfoJsonTemplates"); + }); + + modelBuilder.Entity("DLCS.Repository.Entities.MetricThreshold", b => + { + b.Property("Name") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Metric") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Lower") + .HasColumnType("bigint"); + + b.Property("Upper") + .HasColumnType("bigint"); + + b.HasKey("Name", "Metric"); + + b.ToTable("MetricThresholds"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => + { + b.HasOne("DLCS.Model.Assets.Asset", "Asset") + .WithMany("Adjuncts") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DLCS.Model.Assets.AdjunctBatch", null) + .WithMany() + .HasForeignKey("Batch") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatchAdjunct", b => + { + b.HasOne("DLCS.Model.Assets.AdjunctBatch", "Batch") + .WithMany("BatchAdjuncts") + .HasForeignKey("BatchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DLCS.Model.Assets.Adjunct", "Adjunct") + .WithMany("AdjunctBatchAdjuncts") + .HasForeignKey("AdjunctId", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Adjunct"); + + b.Navigation("Batch"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.BatchAsset", b => + { + b.HasOne("DLCS.Model.Assets.Asset", "Asset") + .WithMany("BatchAssets") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DLCS.Model.Assets.Batch", "Batch") + .WithMany("BatchAssets") + .HasForeignKey("BatchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + + b.Navigation("Batch"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.ImageDeliveryChannel", b => + { + b.HasOne("DLCS.Model.Policies.DeliveryChannelPolicy", "DeliveryChannelPolicy") + .WithMany() + .HasForeignKey("DeliveryChannelPolicyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DLCS.Model.Assets.Asset", null) + .WithMany("ImageDeliveryChannels") + .HasForeignKey("ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DeliveryChannelPolicy"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Metadata.AssetApplicationMetadata", b => + { + b.HasOne("DLCS.Model.Assets.Asset", "Asset") + .WithMany("AssetApplicationMetadata") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("DLCS.Model.DeliveryChannels.DefaultDeliveryChannel", b => + { + b.HasOne("DLCS.Model.Policies.DeliveryChannelPolicy", "DeliveryChannelPolicy") + .WithMany() + .HasForeignKey("DeliveryChannelPolicyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DeliveryChannelPolicy"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => + { + b.Navigation("AdjunctBatchAdjuncts"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatch", b => + { + b.Navigation("BatchAdjuncts"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Asset", b => + { + b.Navigation("Adjuncts"); + + b.Navigation("AssetApplicationMetadata"); + + b.Navigation("BatchAssets"); + + b.Navigation("ImageDeliveryChannels"); + }); + + modelBuilder.Entity("DLCS.Model.Assets.Batch", b => + { + b.Navigation("BatchAssets"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs new file mode 100644 index 000000000..7a029b82d --- /dev/null +++ b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs @@ -0,0 +1,30 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DLCS.Repository.Migrations +{ + /// + public partial class BackfillAdjunctQueueSize : Migration + { + /// + /// Resets every "adjunct" Queues row to 0. Engine was decrementing the wrong Queues row for completed + /// adjuncts (see the IngestHandler fix in the same PR), so any "adjunct" row created before that fix may + /// have drifted upwards and never come back down. Exposed as a constant so it can be exercised directly by + /// BackfillAdjunctQueueSizeMigrationTests without duplicating the SQL. + /// + public const string CorrectionSql = @"UPDATE ""Queues"" SET ""Size"" = 0 WHERE ""Name"" = 'adjunct';"; + + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(CorrectionSql); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // No-op - this is a one-off data correction, there is no prior value to restore to. + } + } +} diff --git a/src/protagonist/DLCS.Repository/Migrations/DlcsContextModelSnapshot.cs b/src/protagonist/DLCS.Repository/Migrations/DlcsContextModelSnapshot.cs index 86e07c964..feeccdef7 100644 --- a/src/protagonist/DLCS.Repository/Migrations/DlcsContextModelSnapshot.cs +++ b/src/protagonist/DLCS.Repository/Migrations/DlcsContextModelSnapshot.cs @@ -1276,12 +1276,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("DLCS.Model.Assets.Adjunct", null) + b.HasOne("DLCS.Model.Assets.Adjunct", "Adjunct") .WithMany("AdjunctBatchAdjuncts") .HasForeignKey("AdjunctId", "AssetId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); + b.Navigation("Adjunct"); + b.Navigation("Batch"); }); diff --git a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs index 8772e0ca5..daa635a47 100644 --- a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs +++ b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using DLCS.Model.Processing; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace DLCS.Repository.Processing; @@ -42,17 +43,26 @@ public CustomerQueueRepository(DlcsContext dlcsContext, ILogger(sql, new { customer }); + // BatchesWaiting/AdjunctsWaiting only count batches that haven't started processing yet (nothing + // Completed or Errored) - this excludes batches currently being worked on, which still count towards + // Size. Translated as correlated subqueries - Sum() over an empty set is coalesced to 0 by EF Core, + // matching the COALESCE the raw-SQL equivalent needed to write out by hand. + // LongCount()/the (long) cast on Sum() aren't decorative - they pick the bigint-returning overloads so + // EF sums/counts as bigint throughout, matching the long properties below, rather than accumulating as + // int32 (via Count()/int Sum()) and casting down then back up again. + var waitingBatches = DlcsContext.AdjunctBatches.Where(b => + b.Customer == customer && b.Finished == null && b.Completed == 0 && b.Errors == 0); + + return await DlcsContext.Queues + .Where(q => q.Customer == customer && q.Name == "adjunct") + .Select(q => new AdjunctQueue + { + Customer = q.Customer, + Size = q.Size, + BatchesWaiting = waitingBatches.LongCount(), + AdjunctsWaiting = waitingBatches.Sum(b => (long)b.Count) + }) + .FirstOrDefaultAsync(cancellationToken); } catch (Exception ex) { @@ -96,4 +106,4 @@ await DlcsContext.Queues.AddAsync( amount); } } -} \ No newline at end of file +} diff --git a/src/protagonist/Engine/Ingest/IngestHandler.cs b/src/protagonist/Engine/Ingest/IngestHandler.cs index b2c0f77e2..0ce74f646 100644 --- a/src/protagonist/Engine/Ingest/IngestHandler.cs +++ b/src/protagonist/Engine/Ingest/IngestHandler.cs @@ -46,7 +46,7 @@ private async Task HandleIngest(QueueMessage message, logger.LogDebug("Message {MessageId} handled with result {IngestResult}", message.MessageId, ingestResult.Status); - await UpdateCustomerQueue(message, ingestResult, GetQueueName(message), cancellationToken); + await UpdateCustomerQueue(ingestResult, GetQueueName(message), cancellationToken); } // return true so that the message is deleted from the queue in all instances. @@ -63,7 +63,7 @@ _ when message.QueueName.Contains("priority", StringComparison.OrdinalIgnoreCase _ => QueueNames.Default }; - private async Task UpdateCustomerQueue(QueueMessage message, + private async Task UpdateCustomerQueue( IngestResult ingestResult, string queue, CancellationToken cancellationToken) { var customer = 0; From 7be3b09d6a631950940f429c94295087c347c524 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 28 Jul 2026 15:03:08 +0100 Subject: [PATCH 3/4] Remove migration + move back to using SQL --- .../Integration/CustomerAdjunctQueueTests.cs | 6 +- .../BackfillAdjunctQueueSizeMigrationTests.cs | 66 - ...54327_BackfillAdjunctQueueSize.Designer.cs | 1379 ----------------- ...20260723154327_BackfillAdjunctQueueSize.cs | 30 - .../Processing/CustomerQueueRepository.cs | 29 +- 5 files changed, 14 insertions(+), 1496 deletions(-) delete mode 100644 src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs delete mode 100644 src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs delete mode 100644 src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs diff --git a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs index 2d4072f6d..853572463 100644 --- a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs +++ b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs @@ -441,7 +441,7 @@ public async Task GetAdjunctQueue_Returns200_WithSizeFromQueueRow_WhenNoBatches( { // Arrange var assetId = AssetIdGenerator.GetAssetId(); - await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 7 }); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = QueueNames.Adjunct, Size = 7 }); await dbContext.SaveChangesAsync(); // Act @@ -461,7 +461,7 @@ public async Task GetAdjunctQueue_Returns200_WithBatchesWaitingAndAdjunctsWaitin { // Arrange var assetId = AssetIdGenerator.GetAssetId(); - await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 10 }); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = QueueNames.Adjunct, Size = 10 }); // finished - excluded await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 5, completed: 5, finished: DateTime.UtcNow); @@ -490,7 +490,7 @@ public async Task GetAdjunctQueue_Links_ResolveToWorkingCollectionEndpoints() { // Arrange var assetId = AssetIdGenerator.GetAssetId(); - await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = "adjunct", Size = 0 }); + await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = QueueNames.Adjunct, Size = 0 }); await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 1, completed: 0); await dbContext.AdjunctBatches.AddTestAdjunctBatch(2, assetId.Customer, count: 1, completed: 1, finished: DateTime.UtcNow); diff --git a/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs b/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs deleted file mode 100644 index 0c9b3ee43..000000000 --- a/src/protagonist/DLCS.Repository.Tests/Migrations/BackfillAdjunctQueueSizeMigrationTests.cs +++ /dev/null @@ -1,66 +0,0 @@ -using DLCS.Model.Processing; -using DLCS.Repository.Migrations; -using Microsoft.EntityFrameworkCore; -using Test.Helpers.Integration; - -namespace DLCS.Repository.Tests.Migrations; - -/// -/// Verifies the correction SQL applied by the 20260723154327_BackfillAdjunctQueueSize migration (exposed as -/// so this test can't drift from what actually ships). -/// -[Trait("Category", "Database")] -[Collection(DatabaseCollection.CollectionName)] -public class BackfillAdjunctQueueSizeMigrationTests -{ - private readonly DlcsContext dbContext; - - public BackfillAdjunctQueueSizeMigrationTests(DlcsDatabaseFixture dbFixture) - { - dbContext = dbFixture.DbContext; - dbFixture.CleanUp(); - } - - [Fact] - public async Task CorrectionSql_ResetsAdjunctQueueRows_LeavesOtherQueueNamesAlone() - { - // Arrange - await dbContext.Queues.AddAsync(new Queue { Customer = 501, Name = "adjunct", Size = 999 }); - await dbContext.Queues.AddAsync(new Queue { Customer = 502, Name = "adjunct", Size = 12 }); - await dbContext.Queues.AddAsync(new Queue { Customer = 503, Name = "default", Size = 42 }); - await dbContext.Queues.AddAsync(new Queue { Customer = 503, Name = "priority", Size = 7 }); - await dbContext.SaveChangesAsync(); - - // Act - await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); - - // Assert - var c501 = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 501 && q.Name == "adjunct"); - c501.Size.Should().Be(0); - - var c502 = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 502 && q.Name == "adjunct"); - c502.Size.Should().Be(0); - - var c503Default = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 503 && q.Name == "default"); - c503Default.Size.Should().Be(42, "non-adjunct queue rows must not be touched"); - - var c503Priority = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 503 && q.Name == "priority"); - c503Priority.Size.Should().Be(7, "non-adjunct queue rows must not be touched"); - } - - [Fact] - public async Task CorrectionSql_IsIdempotent_WhenRunTwice() - { - // Arrange - await dbContext.Queues.AddAsync(new Queue { Customer = 601, Name = "adjunct", Size = 999 }); - await dbContext.SaveChangesAsync(); - - // Act - await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); - await dbContext.Database.ExecuteSqlRawAsync(BackfillAdjunctQueueSize.CorrectionSql); - - // Assert - var queue = await dbContext.Queues.AsNoTracking().SingleAsync(q => q.Customer == 601 && q.Name == "adjunct"); - queue.Size.Should().Be(0); - } -} diff --git a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs deleted file mode 100644 index 172f31d96..000000000 --- a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.Designer.cs +++ /dev/null @@ -1,1379 +0,0 @@ -// -using System; -using System.Collections.Generic; -using DLCS.Repository; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace DLCS.Repository.Migrations -{ - [DbContext(typeof(DlcsContext))] - [Migration("20260723154327_BackfillAdjunctQueueSize")] - partial class BackfillAdjunctQueueSize - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .UseCollation("en_US.UTF-8") - .HasAnnotation("ProductVersion", "9.0.3") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "tablefunc"); - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.HasSequence("adjunct_batch_id_sequence") - .HasMin(1L); - - modelBuilder.HasSequence("batch_id_sequence") - .StartsAt(570185L) - .HasMin(1L) - .HasMax(9223372036854775807L); - - modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => - { - b.Property("Id") - .HasMaxLength(200) - .HasColumnType("character varying(200)"); - - b.Property("AssetId") - .HasColumnType("character varying(500)"); - - b.Property("Batch") - .HasColumnType("integer"); - - b.Property("Created") - .ValueGeneratedOnAdd() - .HasColumnType("timestamp with time zone") - .HasDefaultValueSql("now()"); - - b.Property("Error") - .HasColumnType("text"); - - b.Property("ExternalId") - .HasColumnType("text"); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("IIIFLink") - .IsRequired() - .HasColumnType("text"); - - b.Property("Ingesting") - .HasColumnType("boolean"); - - b.Property("Label") - .HasColumnType("jsonb"); - - b.Property("Language") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("MediaType") - .IsRequired() - .HasColumnType("text"); - - b.Property("Motivation") - .HasColumnType("text"); - - b.Property("Optimised") - .HasColumnType("boolean"); - - b.Property("Origin") - .HasColumnType("text"); - - b.Property("Profile") - .HasColumnType("text"); - - b.Property("Provides") - .HasColumnType("text"); - - b.Property("Size") - .HasColumnType("bigint"); - - b.Property("Type") - .IsRequired() - .HasColumnType("text"); - - b.HasKey("Id", "AssetId"); - - b.HasIndex("AssetId"); - - b.HasIndex("Batch"); - - b.ToTable("Adjuncts"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatch", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("integer") - .HasDefaultValueSql("nextval('adjunct_batch_id_sequence'::regclass)"); - - b.Property("Completed") - .HasColumnType("integer"); - - b.Property("Count") - .HasColumnType("integer"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Errors") - .HasColumnType("integer"); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("Submitted") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Id"); - - b.HasIndex(new[] { "Customer", "Submitted" }, "IX_AdjunctBatchesByCustomerSubmitted"); - - b.ToTable("AdjunctBatches"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatchAdjunct", b => - { - b.Property("BatchId") - .HasColumnType("integer"); - - b.Property("AdjunctId") - .HasMaxLength(200) - .HasColumnType("character varying(200)"); - - b.Property("AssetId") - .HasColumnType("character varying(500)"); - - b.Property("Error") - .HasColumnType("text"); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("Status") - .HasColumnType("integer"); - - b.HasKey("BatchId", "AdjunctId", "AssetId"); - - b.HasIndex("AdjunctId", "AssetId"); - - b.ToTable("AdjunctBatchAdjuncts"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Asset", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Batch") - .HasColumnType("integer"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("DeliveryChannels") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Duration") - .ValueGeneratedOnAdd() - .HasColumnType("bigint") - .HasDefaultValueSql("0"); - - b.Property("Error") - .IsRequired() - .ValueGeneratedOnAdd() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)") - .HasDefaultValueSql("NULL::character varying"); - - b.Property("Family") - .ValueGeneratedOnAdd() - .HasColumnType("char(1)") - .HasDefaultValueSql("'I'::\"char\""); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("Height") - .HasColumnType("integer"); - - b.Property("ImageOptimisationPolicy") - .IsRequired() - .ValueGeneratedOnAdd() - .HasMaxLength(500) - .HasColumnType("character varying(500)") - .HasDefaultValueSql("'fast-lossy'::character varying"); - - b.Property("Ingesting") - .HasColumnType("boolean"); - - b.PrimitiveCollection>("Manifests") - .HasColumnType("text[]"); - - b.Property("MaxUnauthorised") - .HasColumnType("integer"); - - b.Property("MaxWidth") - .HasColumnType("integer"); - - b.Property("MediaType") - .IsRequired() - .ValueGeneratedOnAdd() - .HasMaxLength(100) - .HasColumnType("character varying(100)") - .HasDefaultValueSql("'image/jp2'::character varying"); - - b.Property("NotForDelivery") - .HasColumnType("boolean"); - - b.Property("NumberReference1") - .HasColumnType("integer"); - - b.Property("NumberReference2") - .HasColumnType("integer"); - - b.Property("NumberReference3") - .HasColumnType("integer"); - - b.Property("OpenFullMax") - .HasColumnType("integer"); - - b.Property("Origin") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("PreservedUri") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Reference1") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Reference2") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Reference3") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Roles") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Space") - .HasColumnType("integer"); - - b.Property("Tags") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("ThumbnailPolicy") - .IsRequired() - .ValueGeneratedOnAdd() - .HasMaxLength(500) - .HasColumnType("character varying(500)") - .HasDefaultValueSql("'original'::character varying"); - - b.Property("Width") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.HasIndex("Manifests"); - - NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("Manifests"), "gin"); - - b.HasIndex(new[] { "Batch" }, "IX_ImagesByBatch"); - - b.HasIndex(new[] { "Id", "Customer", "Space" }, "IX_ImagesByCustomerSpace"); - - b.HasIndex(new[] { "Id", "Customer", "Error", "Batch" }, "IX_ImagesByErrors") - .HasFilter("((\"Error\" IS NOT NULL) AND ((\"Error\")::text <> ''::text))"); - - b.HasIndex(new[] { "Reference1" }, "IX_ImagesByReference1"); - - b.HasIndex(new[] { "Reference2" }, "IX_ImagesByReference2"); - - b.HasIndex(new[] { "Reference3" }, "IX_ImagesByReference3"); - - b.HasIndex(new[] { "Customer", "Space" }, "IX_ImagesBySpace"); - - b.ToTable("Images", (string)null); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Batch", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("integer") - .HasDefaultValueSql("nextval('batch_id_sequence'::regclass)"); - - b.Property("Completed") - .HasColumnType("integer"); - - b.Property("Count") - .HasColumnType("integer"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Errors") - .HasColumnType("integer"); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("Submitted") - .HasColumnType("timestamp with time zone"); - - b.Property("Superseded") - .HasColumnType("boolean"); - - b.HasKey("Id"); - - b.HasIndex(new[] { "Customer", "Superseded", "Submitted" }, "IX_BatchTest"); - - b.ToTable("Batches"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.BatchAsset", b => - { - b.Property("BatchId") - .HasColumnType("integer"); - - b.Property("AssetId") - .HasColumnType("character varying(500)"); - - b.Property("Error") - .HasColumnType("text"); - - b.Property("Finished") - .HasColumnType("timestamp with time zone"); - - b.Property("Status") - .HasColumnType("integer"); - - b.HasKey("BatchId", "AssetId"); - - b.HasIndex("AssetId"); - - b.ToTable("BatchAssets"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.CustomHeaders.CustomHeader", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Key") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Role") - .ValueGeneratedOnAdd() - .HasMaxLength(500) - .HasColumnType("character varying(500)") - .HasDefaultValueSql("NULL::character varying"); - - b.Property("Space") - .HasColumnType("integer"); - - b.Property("Value") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.HasIndex(new[] { "Customer", "Space" }, "IX_CustomHeaders_ByCustomerSpace"); - - b.ToTable("CustomHeaders"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.ImageDeliveryChannel", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasMaxLength(100) - .HasColumnType("integer"); - - NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - - b.Property("Channel") - .IsRequired() - .HasColumnType("text"); - - b.Property("DeliveryChannelPolicyId") - .HasColumnType("integer"); - - b.Property("ImageId") - .IsRequired() - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.HasIndex("DeliveryChannelPolicyId"); - - b.HasIndex("ImageId"); - - b.ToTable("ImageDeliveryChannels"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.ImageLocation", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Nas") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("S3") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.ToTable("ImageLocation", (string)null); - }); - - modelBuilder.Entity("DLCS.Model.Assets.ImageStorage", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Space") - .HasColumnType("integer"); - - b.Property("AdjunctSize") - .HasColumnType("bigint"); - - b.Property("CheckingInProgress") - .HasColumnType("boolean"); - - b.Property("LastChecked") - .HasColumnType("timestamp with time zone"); - - b.Property("Size") - .HasColumnType("bigint"); - - b.Property("ThumbnailSize") - .HasColumnType("bigint"); - - b.HasKey("Id", "Customer", "Space"); - - b.HasIndex(new[] { "Customer", "Space", "Id" }, "IX_ImageStorageByCustomerSpace"); - - b.ToTable("ImageStorage", (string)null); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Metadata.AssetApplicationMetadata", b => - { - b.Property("AssetId") - .HasColumnType("character varying(500)"); - - b.Property("MetadataType") - .HasColumnType("text"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("MetadataValue") - .IsRequired() - .HasColumnType("jsonb"); - - b.Property("Modified") - .HasColumnType("timestamp with time zone"); - - b.HasKey("AssetId", "MetadataType"); - - b.ToTable("AssetApplicationMetadata"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.NamedQueries.NamedQuery", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Global") - .HasColumnType("boolean"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Template") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.HasKey("Id"); - - b.ToTable("NamedQueries"); - }); - - modelBuilder.Entity("DLCS.Model.Auth.Entities.AuthService", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("CallToAction") - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("ChildAuthService") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Description") - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.Property("Label") - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(250) - .HasColumnType("character varying(250)"); - - b.Property("PageDescription") - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.Property("PageLabel") - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Profile") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("RoleProvider") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Ttl") - .HasColumnType("integer") - .HasColumnName("TTL"); - - b.HasKey("Id", "Customer"); - - b.ToTable("AuthServices"); - }); - - modelBuilder.Entity("DLCS.Model.Auth.Entities.Role", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Aliases") - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("AuthService") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id", "Customer"); - - b.ToTable("Roles"); - }); - - modelBuilder.Entity("DLCS.Model.Auth.Entities.RoleProvider", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("AuthService") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Configuration") - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.Property("Credentials") - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("RoleProviders"); - }); - - modelBuilder.Entity("DLCS.Model.Customers.Customer", b => - { - b.Property("Id") - .HasColumnType("integer"); - - b.Property("AcceptedAgreement") - .HasColumnType("boolean"); - - b.Property("Administrator") - .HasColumnType("boolean"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("DisplayName") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Keys") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.HasIndex("DisplayName") - .IsUnique(); - - b.HasIndex("Name") - .IsUnique(); - - b.ToTable("Customers"); - }); - - modelBuilder.Entity("DLCS.Model.Customers.CustomerOriginStrategy", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Credentials") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Optimised") - .HasColumnType("boolean"); - - b.Property("Order") - .HasColumnType("integer"); - - b.Property("Regex") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Strategy") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.ToTable("CustomerOriginStrategies"); - }); - - modelBuilder.Entity("DLCS.Model.Customers.SignupLink", b => - { - b.Property("Id") - .HasColumnType("text"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("CustomerId") - .HasColumnType("integer"); - - b.Property("Expires") - .HasColumnType("timestamp with time zone"); - - b.Property("Note") - .HasColumnType("text"); - - b.HasKey("Id"); - - b.ToTable("SignupLinks"); - }); - - modelBuilder.Entity("DLCS.Model.Customers.User", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Email") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Enabled") - .HasColumnType("boolean"); - - b.Property("EncryptedPassword") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Roles") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.HasKey("Id"); - - b.ToTable("Users"); - }); - - modelBuilder.Entity("DLCS.Model.DeliveryChannels.DefaultDeliveryChannel", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("DeliveryChannelPolicyId") - .HasColumnType("integer"); - - b.Property("MediaType") - .IsRequired() - .HasMaxLength(255) - .HasColumnType("character varying(255)"); - - b.Property("Space") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.HasIndex("DeliveryChannelPolicyId"); - - b.HasIndex("Customer", "Space", "MediaType", "DeliveryChannelPolicyId") - .IsUnique(); - - b.ToTable("DefaultDeliveryChannels"); - }); - - modelBuilder.Entity("DLCS.Model.Policies.DeliveryChannelPolicy", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("integer"); - - NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - - b.Property("Channel") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("DisplayName") - .HasColumnType("text"); - - b.Property("Modified") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("PolicyData") - .HasColumnType("text"); - - b.Property("System") - .HasColumnType("boolean"); - - b.HasKey("Id"); - - b.HasIndex("Customer", "Name", "Channel") - .IsUnique(); - - b.ToTable("DeliveryChannelPolicies"); - }); - - modelBuilder.Entity("DLCS.Model.Policies.ImageOptimisationPolicy", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Global") - .HasColumnType("boolean"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("TechnicalDetails") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.HasKey("Id", "Customer"); - - b.ToTable("ImageOptimisationPolicies"); - - b.HasData( - new - { - Id = "none", - Customer = 1, - Global = true, - Name = "No optimisation/transcoding", - TechnicalDetails = "no-op" - }, - new - { - Id = "use-original", - Customer = 1, - Global = true, - Name = "Use original for image-server", - TechnicalDetails = "use-original" - }); - }); - - modelBuilder.Entity("DLCS.Model.Policies.OriginStrategy", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("RequiresCredentials") - .HasColumnType("boolean"); - - b.HasKey("Id"); - - b.ToTable("OriginStrategies"); - }); - - modelBuilder.Entity("DLCS.Model.Policies.ThumbnailPolicy", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Sizes") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.HasKey("Id"); - - b.ToTable("ThumbnailPolicies"); - }); - - modelBuilder.Entity("DLCS.Model.Processing.Queue", b => - { - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Name") - .ValueGeneratedOnAdd() - .HasMaxLength(500) - .HasColumnType("character varying(500)") - .HasDefaultValueSql("'default'::character varying"); - - b.Property("Size") - .HasColumnType("integer"); - - b.HasKey("Customer", "Name"); - - b.ToTable("Queues"); - }); - - modelBuilder.Entity("DLCS.Model.Spaces.Space", b => - { - b.Property("Id") - .HasColumnType("integer"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("ImageBucket") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Keep") - .HasColumnType("boolean"); - - b.Property("MaxUnauthorised") - .HasColumnType("integer"); - - b.Property("Name") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Roles") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Tags") - .IsRequired() - .HasMaxLength(1000) - .HasColumnType("character varying(1000)"); - - b.Property("Transform") - .HasColumnType("boolean"); - - b.HasKey("Id", "Customer") - .HasName("Spaces_pkey"); - - b.ToTable("Spaces"); - }); - - modelBuilder.Entity("DLCS.Model.Storage.CustomerStorage", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("integer"); - - NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("LastCalculated") - .HasColumnType("timestamp with time zone"); - - b.Property("NumberOfStoredAdjuncts") - .HasColumnType("bigint"); - - b.Property("NumberOfStoredImages") - .HasColumnType("bigint"); - - b.Property("Space") - .HasColumnType("integer"); - - b.Property("StoragePolicy") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("TotalSizeOfStoredAdjuncts") - .HasColumnType("bigint"); - - b.Property("TotalSizeOfStoredImages") - .HasColumnType("bigint"); - - b.Property("TotalSizeOfThumbnails") - .HasColumnType("bigint"); - - b.HasKey("Id"); - - b.HasIndex("Customer") - .IsUnique() - .HasDatabaseName("IX_CustomerStorage_Customer_Aggregate") - .HasFilter("\"Space\" IS NULL"); - - b.HasIndex("Customer", "Space") - .IsUnique() - .HasDatabaseName("IX_CustomerStorage_Customer_Space") - .HasFilter("\"Space\" IS NOT NULL"); - - b.ToTable("CustomerStorage", (string)null); - }); - - modelBuilder.Entity("DLCS.Model.Storage.StoragePolicy", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("MaximumNumberOfStoredImages") - .HasColumnType("bigint"); - - b.Property("MaximumTotalSizeOfStoredImages") - .HasColumnType("bigint"); - - b.HasKey("Id"); - - b.ToTable("StoragePolicies"); - }); - - modelBuilder.Entity("DLCS.Repository.Auth.AuthToken", b => - { - b.Property("Id") - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("BearerToken") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("CookieId") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Expires") - .HasColumnType("timestamp with time zone"); - - b.Property("LastChecked") - .HasColumnType("timestamp with time zone"); - - b.Property("SessionUserId") - .IsRequired() - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Ttl") - .HasColumnType("integer") - .HasColumnName("TTL"); - - b.HasKey("Id"); - - b.HasIndex(new[] { "BearerToken" }, "IX_AuthTokens_BearerToken"); - - b.HasIndex(new[] { "CookieId" }, "IX_AuthTokens_CookieId"); - - b.ToTable("AuthTokens"); - }); - - modelBuilder.Entity("DLCS.Repository.Auth.SessionUser", b => - { - b.Property("Id") - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Created") - .HasColumnType("timestamp with time zone"); - - b.Property("Roles") - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.HasKey("Id"); - - b.ToTable("SessionUsers"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.ActivityGroup", b => - { - b.Property("Group") - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Inhabitant") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Since") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Group"); - - b.ToTable("ActivityGroups"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.CustomerImageServer", b => - { - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("ImageServer") - .IsRequired() - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Customer"); - - b.ToTable("CustomerImageServers"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.EntityCounter", b => - { - b.Property("Type") - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Scope") - .HasMaxLength(100) - .HasColumnType("character varying(100)"); - - b.Property("Customer") - .HasColumnType("integer"); - - b.Property("Next") - .HasColumnType("bigint"); - - b.HasKey("Type", "Scope", "Customer"); - - b.ToTable("EntityCounters"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.ImageServer", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("InfoJsonTemplate") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.HasKey("Id"); - - b.ToTable("ImageServers"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.InfoJsonTemplate", b => - { - b.Property("Id") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Template") - .IsRequired() - .HasMaxLength(4000) - .HasColumnType("character varying(4000)"); - - b.HasKey("Id"); - - b.ToTable("InfoJsonTemplates"); - }); - - modelBuilder.Entity("DLCS.Repository.Entities.MetricThreshold", b => - { - b.Property("Name") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Metric") - .HasMaxLength(500) - .HasColumnType("character varying(500)"); - - b.Property("Lower") - .HasColumnType("bigint"); - - b.Property("Upper") - .HasColumnType("bigint"); - - b.HasKey("Name", "Metric"); - - b.ToTable("MetricThresholds"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => - { - b.HasOne("DLCS.Model.Assets.Asset", "Asset") - .WithMany("Adjuncts") - .HasForeignKey("AssetId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DLCS.Model.Assets.AdjunctBatch", null) - .WithMany() - .HasForeignKey("Batch") - .OnDelete(DeleteBehavior.SetNull); - - b.Navigation("Asset"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatchAdjunct", b => - { - b.HasOne("DLCS.Model.Assets.AdjunctBatch", "Batch") - .WithMany("BatchAdjuncts") - .HasForeignKey("BatchId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DLCS.Model.Assets.Adjunct", "Adjunct") - .WithMany("AdjunctBatchAdjuncts") - .HasForeignKey("AdjunctId", "AssetId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Adjunct"); - - b.Navigation("Batch"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.BatchAsset", b => - { - b.HasOne("DLCS.Model.Assets.Asset", "Asset") - .WithMany("BatchAssets") - .HasForeignKey("AssetId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DLCS.Model.Assets.Batch", "Batch") - .WithMany("BatchAssets") - .HasForeignKey("BatchId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Asset"); - - b.Navigation("Batch"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.ImageDeliveryChannel", b => - { - b.HasOne("DLCS.Model.Policies.DeliveryChannelPolicy", "DeliveryChannelPolicy") - .WithMany() - .HasForeignKey("DeliveryChannelPolicyId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.HasOne("DLCS.Model.Assets.Asset", null) - .WithMany("ImageDeliveryChannels") - .HasForeignKey("ImageId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("DeliveryChannelPolicy"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Metadata.AssetApplicationMetadata", b => - { - b.HasOne("DLCS.Model.Assets.Asset", "Asset") - .WithMany("AssetApplicationMetadata") - .HasForeignKey("AssetId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Asset"); - }); - - modelBuilder.Entity("DLCS.Model.DeliveryChannels.DefaultDeliveryChannel", b => - { - b.HasOne("DLCS.Model.Policies.DeliveryChannelPolicy", "DeliveryChannelPolicy") - .WithMany() - .HasForeignKey("DeliveryChannelPolicyId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("DeliveryChannelPolicy"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Adjunct", b => - { - b.Navigation("AdjunctBatchAdjuncts"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.AdjunctBatch", b => - { - b.Navigation("BatchAdjuncts"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Asset", b => - { - b.Navigation("Adjuncts"); - - b.Navigation("AssetApplicationMetadata"); - - b.Navigation("BatchAssets"); - - b.Navigation("ImageDeliveryChannels"); - }); - - modelBuilder.Entity("DLCS.Model.Assets.Batch", b => - { - b.Navigation("BatchAssets"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs b/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs deleted file mode 100644 index 7a029b82d..000000000 --- a/src/protagonist/DLCS.Repository/Migrations/20260723154327_BackfillAdjunctQueueSize.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DLCS.Repository.Migrations -{ - /// - public partial class BackfillAdjunctQueueSize : Migration - { - /// - /// Resets every "adjunct" Queues row to 0. Engine was decrementing the wrong Queues row for completed - /// adjuncts (see the IngestHandler fix in the same PR), so any "adjunct" row created before that fix may - /// have drifted upwards and never come back down. Exposed as a constant so it can be exercised directly by - /// BackfillAdjunctQueueSizeMigrationTests without duplicating the SQL. - /// - public const string CorrectionSql = @"UPDATE ""Queues"" SET ""Size"" = 0 WHERE ""Name"" = 'adjunct';"; - - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.Sql(CorrectionSql); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - // No-op - this is a one-off data correction, there is no prior value to restore to. - } - } -} diff --git a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs index daa635a47..75d89aa5a 100644 --- a/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs +++ b/src/protagonist/DLCS.Repository/Processing/CustomerQueueRepository.cs @@ -45,24 +45,17 @@ public CustomerQueueRepository(DlcsContext dlcsContext, ILogger - b.Customer == customer && b.Finished == null && b.Completed == 0 && b.Errors == 0); - - return await DlcsContext.Queues - .Where(q => q.Customer == customer && q.Name == "adjunct") - .Select(q => new AdjunctQueue - { - Customer = q.Customer, - Size = q.Size, - BatchesWaiting = waitingBatches.LongCount(), - AdjunctsWaiting = waitingBatches.Sum(b => (long)b.Count) - }) - .FirstOrDefaultAsync(cancellationToken); + // Size. Raw SQL rather than EF, as above - a single scan of AdjunctBatches via the subquery, rather + // than the two correlated-subquery scans EF would generate for BatchesWaiting/AdjunctsWaiting. + const string sql = @"SELECT +q.""Customer"", q.""Size"", b.""BatchesWaiting"", b.""AdjunctsWaiting"" FROM ""Queues"" q, + (SELECT COUNT(""Id"") AS ""BatchesWaiting"", COALESCE(SUM(""Count""), 0) AS ""AdjunctsWaiting"" + FROM ""AdjunctBatches"" WHERE ""Customer"" = @customer AND ""Finished"" IS NULL AND ""Completed"" = 0 + AND ""Errors"" = 0) b + WHERE q.""Customer"" = @customer AND ""Name"" = @name +"; + return await this.QueryFirstOrDefaultAsync(sql, + new { customer, name = QueueNames.Adjunct }); } catch (Exception ex) { From e377a4cc6b818fd7bcbaa15157dd19528c189e0d Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 28 Jul 2026 15:34:33 +0100 Subject: [PATCH 4/4] Generate batch id from postgres in tests --- .../Integration/CustomerAdjunctQueueTests.cs | 54 ++++++++++--------- .../Integration/DatabaseTestDataPopulation.cs | 13 +++-- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs index 853572463..acb253546 100644 --- a/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs +++ b/src/protagonist/API.Tests/Integration/CustomerAdjunctQueueTests.cs @@ -463,14 +463,14 @@ public async Task GetAdjunctQueue_Returns200_WithBatchesWaitingAndAdjunctsWaitin var assetId = AssetIdGenerator.GetAssetId(); await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = QueueNames.Adjunct, Size = 10 }); // finished - excluded - await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 5, completed: 5, + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 5, completed: 5, finished: DateTime.UtcNow); // unfinished, nothing started yet - counts as waiting - await dbContext.AdjunctBatches.AddTestAdjunctBatch(2, assetId.Customer, count: 5, completed: 0); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 5, completed: 0); // unfinished but already partially processed - excluded, still being worked on - await dbContext.AdjunctBatches.AddTestAdjunctBatch(3, assetId.Customer, count: 8, completed: 6); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 8, completed: 6); // unfinished but has an error recorded - also excluded, platform has started working on it - await dbContext.AdjunctBatches.AddTestAdjunctBatch(4, assetId.Customer, count: 3, errors: 1); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 3, errors: 1); await dbContext.SaveChangesAsync(); // Act @@ -491,8 +491,8 @@ public async Task GetAdjunctQueue_Links_ResolveToWorkingCollectionEndpoints() // Arrange var assetId = AssetIdGenerator.GetAssetId(); await dbContext.Queues.AddAsync(new Queue { Customer = assetId.Customer, Name = QueueNames.Adjunct, Size = 0 }); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(1, assetId.Customer, count: 1, completed: 0); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(2, assetId.Customer, count: 1, completed: 1, + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 1, completed: 0); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, count: 1, completed: 1, finished: DateTime.UtcNow); await dbContext.SaveChangesAsync(); @@ -543,9 +543,10 @@ public async Task GetAdjunctBatches_Returns200_MostRecentlySubmittedFirst_ByDefa var assetId = AssetIdGenerator.GetAssetId(); var earlier = DateTime.UtcNow.AddMinutes(-10); var later = DateTime.UtcNow; - await dbContext.AdjunctBatches.AddTestAdjunctBatch(101, assetId.Customer, submitted: earlier, - finished: DateTime.UtcNow); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(102, assetId.Customer, submitted: later); + var earlierBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: earlier, finished: DateTime.UtcNow)).Entity; + var laterBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: later)).Entity; await dbContext.SaveChangesAsync(); // Act @@ -555,7 +556,7 @@ await dbContext.AdjunctBatches.AddTestAdjunctBatch(101, assetId.Customer, submit // Assert response.StatusCode.Should().Be(HttpStatusCode.OK); var batches = await response.ReadAsHydraResponseAsync>(); - batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(102, 101); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(laterBatch.Id, earlierBatch.Id); } [Fact] @@ -565,8 +566,10 @@ public async Task GetAdjunctBatches_Returns200_OrdersAscending_WhenOrderByReques var assetId = AssetIdGenerator.GetAssetId(); var earlier = DateTime.UtcNow.AddMinutes(-10); var later = DateTime.UtcNow; - await dbContext.AdjunctBatches.AddTestAdjunctBatch(111, assetId.Customer, submitted: earlier); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(112, assetId.Customer, submitted: later); + var earlierBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: earlier)).Entity; + var laterBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: later)).Entity; await dbContext.SaveChangesAsync(); // Act @@ -576,7 +579,7 @@ public async Task GetAdjunctBatches_Returns200_OrdersAscending_WhenOrderByReques // Assert response.StatusCode.Should().Be(HttpStatusCode.OK); var batches = await response.ReadAsHydraResponseAsync>(); - batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(111, 112); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(earlierBatch.Id, laterBatch.Id); } [Fact] @@ -584,8 +587,8 @@ public async Task GetActiveAdjunctBatches_Returns200_OnlyUnfinishedBatches() { // Arrange var assetId = AssetIdGenerator.GetAssetId(); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(201, assetId.Customer); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(202, assetId.Customer, finished: DateTime.UtcNow); + var activeBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer)).Entity; + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, finished: DateTime.UtcNow); await dbContext.SaveChangesAsync(); // Act @@ -595,7 +598,7 @@ public async Task GetActiveAdjunctBatches_Returns200_OnlyUnfinishedBatches() // Assert response.StatusCode.Should().Be(HttpStatusCode.OK); var batches = await response.ReadAsHydraResponseAsync>(); - batches.Members.Should().ContainSingle(b => b.GetLastPathElementAsInt() == 201); + batches.Members.Should().ContainSingle(b => b.GetLastPathElementAsInt() == activeBatch.Id); } [Fact] @@ -605,8 +608,10 @@ public async Task GetActiveAdjunctBatches_Returns200_OrdersDescending_WhenOrderB var assetId = AssetIdGenerator.GetAssetId(); var earlier = DateTime.UtcNow.AddMinutes(-10); var later = DateTime.UtcNow; - await dbContext.AdjunctBatches.AddTestAdjunctBatch(211, assetId.Customer, submitted: earlier); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(212, assetId.Customer, submitted: later); + var earlierBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: earlier)).Entity; + var laterBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + submitted: later)).Entity; await dbContext.SaveChangesAsync(); // Act @@ -616,7 +621,7 @@ public async Task GetActiveAdjunctBatches_Returns200_OrdersDescending_WhenOrderB // Assert response.StatusCode.Should().Be(HttpStatusCode.OK); var batches = await response.ReadAsHydraResponseAsync>(); - batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(212, 211); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(laterBatch.Id, earlierBatch.Id); } [Fact] @@ -624,10 +629,11 @@ public async Task GetRecentAdjunctBatches_Returns200_OnlyFinishedBatches_Ordered { // Arrange var assetId = AssetIdGenerator.GetAssetId(); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(301, assetId.Customer); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(302, assetId.Customer, - finished: DateTime.UtcNow.AddDays(-7)); - await dbContext.AdjunctBatches.AddTestAdjunctBatch(303, assetId.Customer, finished: DateTime.UtcNow); + await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer); + var olderFinishedBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + finished: DateTime.UtcNow.AddDays(-7))).Entity; + var newerFinishedBatch = (await dbContext.AdjunctBatches.AddTestAdjunctBatch(customer: assetId.Customer, + finished: DateTime.UtcNow)).Entity; await dbContext.SaveChangesAsync(); // Act @@ -637,7 +643,7 @@ await dbContext.AdjunctBatches.AddTestAdjunctBatch(302, assetId.Customer, // Assert response.StatusCode.Should().Be(HttpStatusCode.OK); var batches = await response.ReadAsHydraResponseAsync>(); - batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(303, 302); + batches.Members.Select(b => b.GetLastPathElementAsInt()).Should().ContainInOrder(newerFinishedBatch.Id, olderFinishedBatch.Id); } [Fact] diff --git a/src/protagonist/Test.Helpers/Integration/DatabaseTestDataPopulation.cs b/src/protagonist/Test.Helpers/Integration/DatabaseTestDataPopulation.cs index 3f7a64ccf..0778e133a 100644 --- a/src/protagonist/Test.Helpers/Integration/DatabaseTestDataPopulation.cs +++ b/src/protagonist/Test.Helpers/Integration/DatabaseTestDataPopulation.cs @@ -307,11 +307,16 @@ public static ValueTask> AddTestBatchAsset(this DbSet batchAssets.AddAsync(new BatchAsset { AssetId = assetId, BatchId = batchId }); public static ValueTask> AddTestAdjunctBatch(this DbSet adjunctBatches, - int id, int customer = 99, int count = 1, int completed = 0, int errors = 0, + int? id = null, int customer = 99, int count = 1, int completed = 0, int errors = 0, DateTime? submitted = null, DateTime? finished = null) - => adjunctBatches.AddAsync(new AdjunctBatch + { + var batch = new AdjunctBatch { - Id = id, Customer = customer, Submitted = submitted ?? DateTime.UtcNow, + Customer = customer, Submitted = submitted ?? DateTime.UtcNow, Count = count, Completed = completed, Errors = errors, Finished = finished - }); + }; + if (id.HasValue) batch.Id = id.Value; + + return adjunctBatches.AddAsync(batch); + } }