diff --git a/data-egress-spike.md b/data-egress-spike.md deleted file mode 100644 index 0ea85c256..000000000 --- a/data-egress-spike.md +++ /dev/null @@ -1,115 +0,0 @@ -# Data Egress Spike - -## Aims - -To investigate the Zendesk API and find out how to get the ticket information out to generate Egress csv files to pass back to LDS - -## Background - -### Creating a ticket - -When a user makes a change request in the Portal, a record is created in the Portal Database in a table called ChangeRequests. This holds information related to the request such as (but not limited to): - -- Who made it -- The time it was made -- What type of request (Include, Remove etc) -- The request reference -- The Checking Window -- Request Status (Ready to submit, Submitted-Uncommitted, Submitted-Committed etc) -- The ID of the ticket created in Zendesk (initially blank) - -Example ChangeRequest record after submission (illustrative, not the real fields): - -| Reference | Window | Type | Status | TicketId | -|----------------|---------|---------|-----------------------|----------| -| KS4JUNE_ABC123 | KS4June | Remove | Submitted-Uncommitted | _null_ | -| KS4JUNE_ABC124 | KS4June | Include | Submitted-Uncommitted | _null_ | - -When the Checking Window is closed, all of the records for that Window that have a status of Submitted-Uncommitted are sent to Zendesk and the ChangeRequests table is updated with the Zendesk Ticket Ids and the status of the tickets are updated to Submitted-Committed. - -Example ChangeRequest record after the Window closed (illustrative, not the real fields): - -| Reference | Window | Type | Status | TicketId | -|----------------|---------|---------|-----------------------|----------| -| KS4JUNE_ABC123 | KS4June | Remove | Submitted-Committed | 88856 | -| KS4JUNE_ABC124 | KS4June | Include | Submitted-Committed | 89000 | - -At this point we have a complete list in the database of all of the Amendment Requests that have been sent through to Zendesk. - -**The Portal Database is the single source of truth for Amendment Requests.** - -## Egress - -Once tickets have been scrutinised in Zendesk and approved / rejected as appropriate, there is an Egress procedure that needs to happen to take the approved requests and send them to LDS. - -### Tactical (current) service -In the Tactical Service, egress involves running reports from Zendesk and then a LOT of post-processing of the Excel files before manually dropping a csv in the LDS storage account. - -### Proposed Strategic Service -For the Strategic Egress solution, the aim is to make the process as simple and automated as possible. Firstly, the API does not have an endpoint to run Zendesk reports, so these will no longer be used. Although even if we _could_ run the reports from the API, we wouldn't as we'd be back to all the post-processing. - -These are the steps we will go through: _Please note that this process as listed may be split up into separate stages as per the design in ADO. This is purely a technical proof of concept_ - -- Call the Zendesk API with all of the TicketIds that we have stored in the database for the required Window and the required Amendment Type. (for example, KS4June - Remove). This gives us back the Zendesk Ticket information for requested tickets. This is a snipped / truncated example of the data that the API gives us per ticket. - -``` - "tickets": [ - { - "id": 88856, - "custom_fields": [ - { - "id": 17207944800146, - "value": "8734603" - }, - { - "id": 17207993784978, - "value": "10000011" - }, - { - "id": 17208002901906, - "value": "2007-06-01" - }, - { - "id": 19056253670034, - "value": "auto_approved" - }, - { - "id": 19056595594898, - "value": "31_" - }, - { - "id": 19058058434322, - "value": "2026" - }, - { - "id": 19058091622546, - "value": "6" - }, - { - "id": 19058126549778, - "value": "ks4" - }, - { - "id": 19058409672594, - "value": "Bellingham" - }, - { - "id": 19058507283218, - "value": "Jude" - }, -``` -- Filter out any of the returned tickets that are not either "approved" or "auto_approved" (the example above is "auto_approved") -- Generate a csv in the appropriate LDS format with the filtered ticket data, converting the data as appropriate (in this example `8734603` into `873` for the LA, `4603` into the Establishment number, `31_` into `31` for the Correction_Type etc.) -- Upload the csv into the Storage account for LDS to be able to pick up. For example, for Removals: - -| Correction_ID | Correction_Type | Correction_Reason | Key_Stage | Establishment_Number | Surname | Forename | Sex | Date_of_Birth | Cycle_Year | Cycle_Month | Local_Authority | Learner_ID | -| ------------- | --------------- | ----------------- | --------- | -------------------- | ---------- | -------- | --- | ------------- | ---------- | ----------- | --------------- | ---------- | -| 88856 | 31 | 4 | KS4 | 4603 | Bellingham | Jude | M | 2007-06-01 | 2026 | 6 | 873 | 10000011 | - -- We will also at this point update the ChangeRequests table with the status of the Amendment Request from Zendesk (ie, from scrutiny to approve / reject as appropriate.) - -## Summary - -We can completely eliminate the manual processing and need for stored reports in Zendesk by using the API in this way. The only thing that should be noted is that: - - **The Strategic Service's database is the _single_ source of truth for Amendment Requests. It is from this database that the egress files are generated (using the Zendesk API). Therefore, any Amendment Requests added to Zendesk outside of the Portal process _will not be included in egress_.** \ No newline at end of file diff --git a/docker-compose.sandbox.yaml b/docker-compose.sandbox.yaml index fead29216..b80e27069 100644 --- a/docker-compose.sandbox.yaml +++ b/docker-compose.sandbox.yaml @@ -27,6 +27,10 @@ services: - Dev__ToolsEnabled=${DEV_TOOLS_ENABLED:-true} - ASPNETCORE_URLS=http://+:8080 - Notify__UseFake=${NOTIFY_USEFAKE:-true} + # This stack has no ZendeskSettings and no EgressStorage account; without opting into the + # dev outbox fake the app fails to start at all (AB#294553 review B1 made the real Zendesk + # client the default everywhere, including here). + - Zendesk__UseFake=true # Points at the sandbox db + azurite services on the sandbox network, # NOT the main stack's cypd_db / cypd_azurite. Kept literal so the # sandbox can never accidentally speak to the main-stack DB. diff --git a/docker-compose.yaml b/docker-compose.yaml index 5d621bbb0..77a1bfcc1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -22,8 +22,12 @@ services: # Local dev has no GOV.UK Notify secret, so use the console fake (logs emails # instead of sending). Set NOTIFY_USEFAKE=false + supply Notify__ApiKey to send real emails. - Notify__UseFake=${NOTIFY_USEFAKE:-true} + # AB#294553 review B1: the code/appsettings default is now the real Zendesk client (matching + # the worker), so the local/E2E stack must opt in to the dev outbox fake explicitly here. + - Zendesk__UseFake=${ZENDESK_USE_FAKE:-true} - ConnectionStrings__Postgres=${POSTGRES_CONNECTION_STRING:-Host=db;Database=cypd;Username=postgres;Password=postgres} - ConnectionStrings__AzureStorage=${AZURE_STORAGE_CONNECTION_STRING:-DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;} + - ConnectionStrings__EgressStorage=${EGRESS_STORAGE_CONNECTION_STRING:-DefaultEndpointsProtocol=http;AccountName=egressaccount;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/egressaccount;QueueEndpoint=http://azurite:10001/egressaccount;} - DfeSignIn__BaseUrl=${DFESIGNIN_BASEURL:-https://test-api.signin.education.gov.uk/} - DfeSignIn__ClientId=${DFESIGNIN_CLIENTID} - DfeSignIn__ApiClientSecret=${DFESIGNIN_APICLIENTSECRET} @@ -196,7 +200,7 @@ services: - "10000:10000" # Blob - "10001:10001" # Queue environment: - - AZURITE_ACCOUNTS=devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;ingressaccount:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + - AZURITE_ACCOUNTS=devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;ingressaccount:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;egressaccount:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== volumes: - azurite_data:/data command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --location /data --debug /data/debug.log --skipApiVersionCheck diff --git a/docs/add-pupil-journey.md b/docs/add-pupil-journey.md index d346b8309..a88c518c9 100644 --- a/docs/add-pupil-journey.md +++ b/docs/add-pupil-journey.md @@ -105,8 +105,8 @@ defensible to keep either way. ## LDS bound values -These are a hard contract against the `LDS_CYPMD_Data specification v2.4` the future egress story -reads from. +These are a hard contract against the `LDS_CYPMD_Data specification v2.4` the egress story reads +from. Consumed by `Application/Egress/EgressRecordBuilder.Build` since AB#294553. **Where the egress story will find them.** Not in a `RequestDocument`: `BuildAnswerRecord` and its `RawValue`/`Value` split only exist inside `RequestService.BuildRequestDocument`, which an Add diff --git a/docs/architecture/c4-container.md b/docs/architecture/c4-container.md index d71d014e4..cda909b45 100644 --- a/docs/architecture/c4-container.md +++ b/docs/architecture/c4-container.md @@ -119,7 +119,8 @@ design for as-built: | Gov Notify integration | **Implemented** — `INotifyService` / `NotifyEmailClient`; notifications are dispatched from the Web app via `NotificationBackgroundService`. | | **Admin Portal** | **Not a separate app** — admin functionality currently lives **inside the Web Portal** (`Web/Admin/`, `AdminController`, `WindowAdmin`, `QueueAdmin`, `StorageAdmin`, `ShareAdmin`, `AdminRequests`, `PageTreeAdmin`). | | **Request Status Updater** | **Planned** — no time-triggered Zendesk-polling worker exists yet (the worker currently runs only `DlqRetentionJob` / `MetricsRetentionJob`). | -| **Data Ingress / Egress Processing**, **LDS Interface Data Storage**, LDS exchange | **Planned** — the LDS pipeline is not implemented; pupil/school JSON in Data Blob Storage is currently produced by dev seeding (`SeedPupilData`). | +| **Data Egress Processing** | **Implemented (AB#294553)** as an admin-driven run inside the web app; the worker container in the diagram is not used. Pull → Preprocess → Transfer at `/admin/egress` (`DfE.CheckPerformanceData.Web/Controllers/Egress/EgressController.cs`), see `docs/data-egress.md`. | +| **Data Ingress Processing**, **LDS Interface Data Storage**, LDS exchange | **Planned** — the LDS ingress pipeline is not implemented; pupil/school JSON in Data Blob Storage is currently produced by dev seeding (`SeedPupilData`). | > Update this section as the LDS pipeline, a dedicated Admin Portal, and the > Request Status Updater are built out, or split the workers into separate diff --git a/docs/data-egress.md b/docs/data-egress.md new file mode 100644 index 000000000..3a4b5cf17 --- /dev/null +++ b/docs/data-egress.md @@ -0,0 +1,327 @@ +# Data egress to LDS (AB#294553 / spec AB#292610) + +A back-office admin section where a CYPMD ops user pulls the scrutiny decisions for one checking +window's Add and Remove amendment requests, preprocesses the approved ones into LDS-spec CSV +files, persists the processed records in the database, and transfers the files to the LDS storage +account. Everything lives inside the web app — no separate worker or scheduled job. + +## 1. What it does + +Five screens under `/admin/egress`, gated by `[RequireAdminSection(AdminNavKeys.Egress)]`: + +- **Pull** (`GET`/`POST /admin/egress`) — choose a checking window and one or more output types + (New learners, Remove learners), then pull. Also lists saved (in-progress) and completed runs. +- **Results** (`GET /admin/egress/runs/{id}/results`) — every pulled request, one tab per output + type, with its raw Zendesk decision and CYPMD values, before any filtering happens. +- **Preprocessing** (`GET /admin/egress/runs/{id}/preprocessing`, plus + `GET .../preprocessing/stream` for the SSE progress feed and `POST .../preprocessing` for the + no-JS run-to-completion fallback) — runs the eight-step pipeline and shows live progress. +- **Failed** (`GET /admin/egress/runs/{id}/failed`) — reached only when preprocessing found a + problem; lists every failure. +- **Summary** (`GET /admin/egress/runs/{id}/summary`, `POST .../transfer`) — confirms the target + container and file list, with a preview (`GET .../preview/{outputType}`) and download + (`GET .../download/{outputType}`) per file, then transfers. +- **Complete** (`GET /admin/egress/runs/{id}/complete`) — the transfer summary: files, record + counts, hashes, who did it and when. + +`GET /admin/egress/runs/{id}` (Resume) sends the browser to whichever of these a run's status +implies, so a saved run reopens without re-pulling. `POST /admin/egress/runs/{id}/abandon` ends a +run at any non-terminal point. + +## 2. Where the data comes from + +Zendesk supplies **only the decision**: the "Decision status" custom field on the ticket id already +stored on `ChangeRequests.CrmId`, read via `GET /api/v2/tickets/show_many.json` (100 ids per call). +Every other value — pupil details, dates, the school's establishment number, the journey's answers +— comes from CYPMD's own data: the `ChangeRequests` row and the persisted journey blob +(`requests/{reference}.json`, `IRequestStateBlobClient`). Production configures no Zendesk custom +fields at all, and an Add (new learner) request carries no establishment number, admission date, +year group or SEN status in Zendesk — so the database has to be the source of truth regardless. + +`ChangeRequests.OrganisationLaestab` is a new column, written at submit time from the DfE Sign-In +`organisation_laestab` claim (`RequestService.OrganisationLaestabOrNull`). It exists because a new +learner's pupil record is synthetic and carries no LAESTAB of its own — the school's LAESTAB from +the request row is the only source for that case. + +The pull's output, `EgressSourceRecord`, is the "as pulled" shape shown on Results and persisted as +the run's raw JSON (so a resume never re-pulls): identifiers (`ChangeRequestId`, `ReferenceNumber`, +`TicketId`), the `Decision` (a Zendesk value, or `EgressDecisions.NoTicket` / `NotFound`), the +output type and window type, submission metadata, the school's URN/LAESTAB, every pupil field +CYPMD holds, and the journey's answers flattened by question id (`EgressAnswers.Flatten`). + +## 3. Run lifecycle + +``` +Pulled → Preprocessing → (PreprocessingFailed | Preprocessed) → Transferring → (TransferFailed | Transferred) +``` + +`Abandoned` is reachable from any non-terminal status, including `Preprocessing` and +`Transferring` — the lock has no expiry, so a run stuck there after a pod restart or a crashed +tab must always be releasable, and every screen from Preprocessing onwards carries an Abandon +form. Abandoning a `Transferring` run also sweeps any blob it actually wrote (matched by the +`egressRunId` metadata stamped on upload) before releasing the pair, so a same-named retry never +collides with an orphaned file; the confirmation banner names what was removed, if anything, or +says nothing was transferred. A run is persisted the moment the pull succeeds +(`EgressRunService.StartAsync` → `IEgressRunRepository.CreateRunAsync`), so "Save and exit" on any +screen is just leaving the page — there is no separate save action. Resume +(`EgressController.Resume`) maps status to screen: `Pulled` → Results, `Preprocessing` → +Preprocessing, `PreprocessingFailed` → Failed, `Preprocessed`/`TransferFailed`/`Transferring` → +Summary, `Transferred` → Complete; `Abandoned` (or an unknown run) sends the user home with a +banner. A failed **transfer** can be re-run from the same screen; a `PreprocessingFailed` run +cannot — it releases its pair (its outputs go inactive) and the Failed screen's own copy directs +starting a new run instead, so two runs (the retried one and a colleague's fresh one) can never +both hold the same pair. + +## 4. Concurrency + +One active run per checking window **and** output type, enforced by a database constraint, not +just a service check: a partial unique index on `egress_run_outputs ("WindowId", "OutputType") +WHERE "IsActive"`. `IsActive` is true from `Pulled` all the way through `Transferred` — a +successful transfer leaves it active forever, so the same window and output type can never be sent +twice — and false for `PreprocessingFailed`, `TransferFailed` and `Abandoned`, so a failed or +abandoned run never blocks a new one. + +`EgressRunService.StartAsync` checks for a blocker first (a friendly refusal naming who holds it +and since when) and `EgressRunRepository.CreateRunAsync`/`TryReactivateAsync` also catch the +database's unique-violation as a race guard, in case two admins start at the same instant. The two +refusal sentences (`EgressController.Describe`, FLAGGED copy): + +- *"{Output type} for this checking window is already being processed by {name}, started {date} at + {time} UTC. Wait for that run to finish or be abandoned."* +- *"{Output type} for this checking window has already been transferred to LDS by {name} on {date} + at {time} UTC. It cannot be sent again."* + +## 5. Preprocessing + +Eight steps, reported one at a time over the same `IAsyncEnumerable` +(`EgressPreprocessor.RunAsync`) whether driven by the SSE stream or the no-JS POST: + +1. **Filter records** — keep `approved` and `auto_approved`; everything else is discarded here, + not shown as a failure. +2. **Derive correction codes** — Remove: the bare LDS code from the Zendesk "Correction reason + (31)" option for the journey's `reason` answer (plus two Post16 reasons mapped by hand); New: + the fixed code `10`. +3. **Split DfE establishment number** — the pupil's LAESTAB first, then the request row's + `OrganisationLaestab`; 7 digits only, split 3+4. +4. **Standardise dates** — date of birth (and, for New learners, admission date) to `yyyy-MM-dd`. +5. **Build LDS records** — the typed `NewLearnerRow`/`RemoveLearnerRow`. +6. **Trim values**. +7. **Validate against LDS spec** — required fields, permitted values, digit widths, date shapes; Surname and Forename (the only free-text cells, typed by the school) must not begin with `=`, `+`, `-` or `@`, which a spreadsheet would evaluate as a formula (OWASP CSV injection) — refused here with a named reason rather than rewritten with an apostrophe the LDS spec never asked for + (`LdsSpecValidator`). +8. **Save to database** — all or nothing: if **any** record failed at any step, nothing is written, + the run becomes `PreprocessingFailed`, and every failure (step, ticket id, reference, field, + reason) is listed on the Failed screen; otherwise rows go to `new_learners`/`remove_learners`, + file names are fixed for the run, and the run becomes `Preprocessed`. Every independent step + (2–4) runs regardless of an earlier one's outcome, so a record with two unrelated problems (say + an unmapped correction reason and an invalid LAESTAB) lists both in one pass rather than the + second only surfacing on a later re-run. + +Leaving the page mid-run (closing the SSE connection, or the no-JS POST being interrupted) puts the +run back to its status before preprocessing started — nothing durable happens until step 8 commits. +If the run was abandoned by someone else while this pipeline was still running, step 8's write is +guarded by the run's expected status and does nothing; the stream reports "this run was abandoned +while preprocessing" rather than claiming `Preprocessed` or `PreprocessingFailed` for a run that is +actually `Abandoned`. + +## 6. Files + +Both LDS files use the same writer (`EgressCsvWriter`): RFC 4180 quoting, `\r\n` between lines, no +line terminator after the final data row, every value trimmed, UTF-8 without a byte-order mark. + +Both files follow `LDS_CYPMD_Data specification_v2.4.xlsx` (sheets "New Learner" and "Remove +Learner", read top to bottom, keeping the rows marked X for the window's key stage). The column set +is therefore the *window's*: `EgressColumnSets.RemoveLearnersFor(windowType)` / +`NewLearnersFor(windowType)`. + +**Remove learners** — every key stage: + +``` +Correction_ID, Correction_Type, Correction_Reason, Key_Stage, Establishment_Number, Surname, +Forename, Sex, Date_of_Birth, Cycle_Year, Cycle_Month, Local_Authority, Learner_ID +``` + +then, KS4 only: `Year_Group` (populated for year-group-change removals from the journey's +`year-group-higher/lower-moved-to` answer, blank otherwise); 16-19 only: `Removal_Year_0`, +`Removal_Year_1`, `Removal_Year_2` (`TRUE`/`FALSE` from the `years-to-remove` checkbox — Year_0 is +the academic year ending in Cycle_Year; blank when the journey route did not ask). + +**New learners** — every key stage (the spec's `Middle_Name` is struck through in v2.4, "CYPMD will +not be sending this field from June 2026", so it is not emitted; there is no SEN attribute): + +``` +Correction_ID, Correction_Type, Key_Stage, Establishment_Number, Surname, Forename, Sex, +Date_of_Birth, Admission_Date, Post_Code, Cycle_Year, Cycle_Month, Local_Authority, URN, ULN, UPN, +Learner_ID, Year_Group +``` + +then, 16-19 only: `Attendance_Year_0`, `Attendance_Year_1`, `Attendance_Year_2`, `KS4_Year` — all +blank today because no Post16 Add journey exists. + +Values: `Key_Stage` is `KS2` / `KS4` / `16-19` (`EgressOutputTypes.KeyStageValue`; the *file name* +still uses AB#292610's `KS5` token). `Cycle_Year` / `Cycle_Month` are the checking window's start +year and month (the spec's "month in which the cycle takes place"), not each record's submission +date. `Sex` is `F` / `M` / `U` on both files. 16-19 `Correction_Reason` codes are the spec's +(`CorrectionCodes`): 4 deceased, 325 not at end of study, 326/328/331 not on roll (international / +external / apprentice), 329 other with evidence. + +File names are fixed when preprocessing completes: `CYPMD_LDS_{stage}_{type}_{yyyy_MM_dd}.csv`, +where `{stage}` is `KS2`/`KS4`/`KS5` (`EgressOutputTypes.StageToken`) and the date is the **London** +calendar date at the moment preprocessing finishes (`EgressOutputTypes.FileName`). + +## 7. Transfer and audit + +`EgressTransferService.TransferAsync` refuses before touching anything if every output's saved row +count is zero (every record was rejected, undecided, or lost to a misconfigured ticket source) — +Summary shows a plain message instead of Confirm in that case, so LDS is never sent a header-only +file for a pair that then locks forever. Otherwise it builds each file's bytes from the +**persisted rows**, never from the pulled payload, and uploads with `IfNoneMatch: *` (create-only — +an existing blob is never overwritten). + +Once the run's status has flipped to `Transferring`, the upload loop, the commit +(`MarkTransferredAsync`) and all compensation run with `CancellationToken.None`, not the request's +own token — a browser tab closing mid-upload must not abandon a run with files already in LDS. If +any file fails to upload, every file already written by that attempt is deleted +(`IEgressBlobClient.DeleteIfExistsAsync`), and the specific file that was mid-upload when the +failure happened is *also* checked and removed if it turns out to have landed server-side despite +the client seeing a failure (matched by the `egressRunId` metadata stamped on upload, so a blob +belonging to a different run is never touched); the run becomes `TransferFailed` with the reason +recorded. A post-upload failure in the commit itself (every file uploaded, but marking the run +`Transferred` throws, or loses a race because the run was abandoned in between) is caught the same +way and compensated identically — no partial state survives silently. A retry re-activates the +run's outputs first and is refused if another run has since claimed the pair. + +A create-only upload that hits an existing blob (`EgressBlobAlreadyExistsException`) is not +always a real collision. Before giving up, transfer reads the blob's `egressRunId` stamp +(`IEgressBlobClient.GetOwnerRunIdAsync`): a file stamped by a run that is now `Abandoned` (a sweep +the process died in the middle of) or by this very run (an earlier attempt whose compensation +delete failed) is reclaimed — deleted with the ownership-checked delete and the upload retried +exactly once. A file stamped by any live run, or carrying no stamp at all, is left untouched and +the run becomes `TransferFailed` with a reason that names who wrote it and the way out. Note that +the file name carries the *preprocessing* date, so retrying the same run on a later day reuses +the same name: the way out of a genuine same-stage/same-day collision is to abandon the run and +start a new one on a later day, or to have LDS remove the file. + +Every terminal write (`MarkPreprocessingFailedAsync`, `SavePreprocessedAsync`, +`MarkTransferredAsync`, `MarkTransferFailedAsync`) is guarded by the status the caller expects the +run to currently hold, and reports rows affected; a caller that gets zero back knows it lost a race +(most often to a concurrent Abandon) and never overwrites what actually happened with a stale +outcome. Success and failure each write an `AuditEntry` in the same transaction as the guarded +state change: `EntityType` `"EgressRun"`, `Action` `"Transfer"` or `"TransferFailed"`, `NewValues` a +JSON object with the outcome, window id, output types, file names, record counts, SHA-256 hashes, +target container and who/when. No audit row is ever written for a write that lost its race, and no +audit row ever claims success for a failed transfer. + +## 8. Configuration + +| Setting | Purpose | +|---|---| +| `ConnectionStrings:EgressStorage` | The LDS storage account. Absent → the Pull page and the Summary show a warning up front (`egress-storage-not-configured`), the app refuses to transfer with a clear "not configured" message rather than failing at startup, and the dev cleanup skips its blob sweep. Pull, preprocess, preview and download still work without it. **`appsettings.json` deliberately carries no default** (pinned by `AppSettingsEgressStorageTests`): a local-Azurite default there made every deployed environment without an account believe it had one at `127.0.0.1:10000` inside the pod, so transfers and cleanups failed after the SDK's retries instead of refusing. Local runs get it from `docker-compose.yaml` and the launch profiles. Review apps get the per-PR `lds` account (`terraform/application/application.tf`, `local.egress_storage_secrets`, gated on `var.config == "review"`) so the E2E transfer facts run end to end; the long-lived environments are still unconfigured until the real LDS account is known — see gaps below. | +| `EgressStorage:Container` (default `cypmd`) / `EgressStorage:Prefix` (default `extracts_input/`) | Where in the account files land. Bindable only so a test can point at a scratch container — never user-editable. | +| `Zendesk:UseFake` (default **`false`**) | Selects the ticket source: the real Zendesk client (`ZendeskEgressTicketSource`, via the same `AddZendeskApiClient` the worker uses) unless explicitly set to `true`, which selects the dev outbox (`DevOutboxEgressTicketSource`, no Zendesk settings needed). The default matches the worker's own configured default — a fresh environment that sets nothing reads real Zendesk decisions, not the dev outbox. `AddCpdEgress` refuses to start if `UseFake=true` is set in Production, regardless of configuration, so the dev outbox can never be reached there. Local/E2E stacks opt in explicitly via `Zendesk__UseFake=true` (`docker-compose.yaml`, `docker-compose.sandbox.yaml`), since neither has real Zendesk credentials. Review apps opt in too (`terraform/application/config/review.yml`): the E2E egress facts seed their decisions into the outbox via `/dev/egress/seed`, and against real esfa-preprod every seeded id read back as "Ticket not found", so the suite could not pass there. Web and worker share the ConfigMap, so review-app submissions land in the outbox rather than creating esfa-preprod tickets. | +| `ZendeskTicketFields:DecisionStatusId` | The real ticket source's required field id; `0` in production today, so it refuses to pull until configured. | +| Admin grant `egress` | `DefaultAdminAccessSeeder.AllSections` — without it a fresh database 404s on `/admin/egress` even for an admin. | + +## 9. Local development and E2E + +Locally the web container gets a third Azurite account (`docker-compose.yaml`, +`ConnectionStrings__EgressStorage`, alongside the app and ingress accounts), and opts in to the +dev outbox fake explicitly with `Zendesk__UseFake=true` — the code/config default is now the real +Zendesk client, which this stack has no credentials for. + +`DevEgressController` (dev-only, 404 unless `Dev:ToolsEnabled` and not Production, same rule as +`DevPipelineController`) stages fixture data with no worker and no real Zendesk: + +```bash +# Seed 3 approved Remove-learner requests for the KS4 June window +curl -X POST "http://localhost:8080/dev/egress/seed?windowId=F34D285B-8660-4D12-9C30-787328DEAA0A&outputType=RemoveLearners&decision=auto_approved&count=3&laestab=860/4070&urn=142313&reason=pupil-died" + +# Remove everything the harness created for that window (runs, requests, blobs) +curl -X POST "http://localhost:8080/dev/egress/cleanup?windowId=F34D285B-8660-4D12-9C30-787328DEAA0A" +``` + +`tests/DfE.CheckPerformanceData.E2ETests/Admin/DataEgressTests.cs` walks the whole journey over +plain HTTP (pull → results → preprocessing → summary → download → transfer → complete → refused), +a failing-record path, and — Linux-only — a real-browser fact for the streamed progress. Run just +this class: `docker compose --profile e2e run --rm e2e-tests sh -c 'dotnet test +tests/DfE.CheckPerformanceData.E2ETests/ --filter "FullyQualifiedName~DataEgressTests"'`. + +To inspect what actually landed in the local LDS account, the Storage browser under Danger zone +covers the **app** account only — the egress account needs a separate client (e.g. Azure Storage +Explorer, or the Azure CLI, pointed at the `EgressStorage` connection string from +`docker-compose.yaml` or `Properties/launchSettings.json`), container `cypmd`, prefix `extracts_input/`. + +Two rules keep a dev or review environment recoverable after a failed run: + +- `POST /dev/egress/cleanup` always deletes its database rows. The blob sweep is best effort — a + blob that could not be deleted is counted in the response's `blobErrors`, never thrown — because + a cleanup that answered 500 and left the runs behind is what broke the next deploy (below). +- The dev seeder (`SeedCheckingWindows`, run at start-up wherever `SeedDevelopmentData` is on) + deletes `egress_runs` before it wipes `CheckingWindows`. The foreign key from a run to its window + is RESTRICT on purpose — an egress is an audit record — so a leftover run used to make the wipe + throw before the host listened. On a review app that looked like a stalled rollout: the new pod + never became Ready, the old one kept serving, terraform reported "old replicas are pending + termination" and a re-run reported "No changes". PR #441's review app served a two-day-old image + that way; `CheckingWindowSeedWithEgressHistoryTests` pins the fix. + +## 10. Known gaps and follow-ups + +- **Merged learners** (and the KS4 June code `20`→`21` correction-code rule, AB#292610) is + deliberately out of scope — its own column set and rule, tracked as a follow-up ticket. +- **Runs history with filters and pagination** is AB#294590; the Pull page carries only a minimal + saved/completed list. +- **LDS spec v2.4 questions still open with LDS/BA** (see the PR notes §10): the struck + `Middle_Name` heading is omitted entirely; `Key_Stage` says `16-19` (v2.4 changed the Remove + sheet from 16-18, the New Learner sheet was not updated); year-group-change removals go out as + Correction_Type 31 / reason 17 (business-confirmed 2026-08-06) although the spec's hidden Addback + sheet has them as type 30; `Removal_Year_0..2` are blank unless the 16-19 journey took the + "other" route; 16-19 "other" is always 329 (evidence is always collected, so 330 never occurs); + Correction_Type 11 (Include learner) is in the spec but Include is not an egress output yet. +- **16-19 gaps**: no Add journey exists, so the New learners file for a 16-19 window can only ever + be header-only (and Transfer refuses a run whose outputs are all empty); 16-19 pupil records have + no MATCHREF, so `Learner_ID` fails validation for them until the 16-19 pupil file supplies one. +- **Trailing newline**: files end after the last data row with no trailing line terminator, to + honour "no additional rows below the final data row" — confirm this reading with LDS. +- **`ConnectionStrings__EgressStorage` is only in Terraform for review apps** (`terraform/ + application/application.tf`, `local.egress_storage_secrets`, pointing at the per-PR `lds` + account). Development, QA, preproduction and production stay unconfigured — add them once + the real LDS account details are known. +- **Production's `ZendeskTicketFields__DecisionStatusId` is `0`** — the real ticket source refuses + to pull until an environment configures it. +- The pre-existing gap that `Controllers/WindowAdmin/*` carries no `[RequireAdminSection]` is a + separate ticket and was not touched here; the new `EgressController` **is** gated. +- Every copy string introduced by this feature is FLAGGED for content sign-off — see the PR notes. +- **Same-stage same-day file-name collision**: two different checking windows of the same stage + (KS4 June and KS4 Autumn both map to the `KS4` file token) transferred on the same day produce + the same file name, so the second run's transfer fails with "already exists". The file name is + fixed at preprocessing, so retrying the *same* run on a later day collides again; the ways out + are to abandon the run and start a new one on a later day, or a manual delete in LDS — and the + recorded failure reason now says exactly that (§7). A stable fix means a different naming + scheme, which is LDS's to agree. Question for LDS: is one-window-per-stage-per-day a real + constraint? Tracked as a follow-up. +- **`PreprocessingStream` is a state-mutating GET** (the accepted `ValidateWindowController` + pattern) — the JS closes the `EventSource` on a terminal/error event, so the browser's automatic + reconnect never restarts the server-side pipeline. Left as-is; noted for the next contributor. +- **Abandon crash-window orphan**: R1 reordered `AbandonAsync` to write `Abandoned` before + sweeping the run's blobs (closing a data-loss race — see the R1 commit), which moved the crash + window rather than removing it: if the process dies after the write commits but before the + sweep loop finishes, the run ends up `Abandoned` with one or more files still in LDS storage. + The recovery is now automatic rather than a manual LDS delete: the next transfer whose file + name collides with such a file reclaims it (§7), because it is stamped with an `Abandoned` run's + id. Two residual caveats: LDS may already have picked the orphan up before it is reclaimed + (true of any orphan, reclaimed or not), and an orphan whose name no later transfer ever reuses + stays there until LDS or an operator removes it. +- Related: `EgressTransferService.FailAsync`'s compensation delete of its own just-uploaded files + (the `uploaded` list) calls `blobs.DeleteIfExistsAsync` unconditionally, unlike its + `possiblyOrphaned` check, which is ownership-checked via `DeleteIfOwnedByRunAsync`. In the + create-only-upload case this is safe today (a successful create-only PUT cannot belong to + another run), but it is a related blob-lifecycle-under-overlap gap worth tightening for + consistency. Follow-up, not an open production incident. +- An independent review of the transfer/lock state machine found one Blocker (B1: the web host + defaulted to the dev Zendesk fake with nothing but QA config overriding it, so Production would + have read the dev outbox table instead of real Zendesk decisions) and four Must-fixes (M1: + transfer was not atomic on cancellation or a post-upload database failure, leaving a run stuck + `Transferring` with files already in LDS; M2: re-running a `PreprocessingFailed` run bypassed + its released lock; M3: a run whose approved set was empty could still transfer a header-only + file and lock its pair forever; M4: every terminal status write was unconditional, so a + concurrent Abandon could be silently overwritten) plus several should-fixes, all addressed in + follow-up commits on this branch — see the PR description for the finding-by-finding record. diff --git a/docs/spikes/data-egress-spike.md b/docs/spikes/data-egress-spike.md index ce8605ddc..3f13af10f 100644 --- a/docs/spikes/data-egress-spike.md +++ b/docs/spikes/data-egress-spike.md @@ -1,3 +1,7 @@ +> **Implemented as of AB#294553** — see `docs/data-egress.md` for the shipped design; this page is +> kept for its background on Zendesk and the Remove learners column list, which the implementation +> follows verbatim. + # Data Egress Spike ## Aims diff --git a/src/DfE.CheckPerformanceData.Application/Admin/DefaultAdminAccessSeeder.cs b/src/DfE.CheckPerformanceData.Application/Admin/DefaultAdminAccessSeeder.cs index 8ac5bedfa..65489b3af 100644 --- a/src/DfE.CheckPerformanceData.Application/Admin/DefaultAdminAccessSeeder.cs +++ b/src/DfE.CheckPerformanceData.Application/Admin/DefaultAdminAccessSeeder.cs @@ -42,6 +42,8 @@ public sealed class DefaultAdminAccessSeeder(IAdminSectionAccessRepository repos "window-admin", "new-window", "manage-window", + // Data egress tile + the EgressController gate (AB#294553). The group container is omitted. + "egress", "reset-seed-data", "share-admin", "search-analytics", diff --git a/src/DfE.CheckPerformanceData.Application/Egress/CorrectionCodes.cs b/src/DfE.CheckPerformanceData.Application/Egress/CorrectionCodes.cs new file mode 100644 index 000000000..cd8886fe9 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/CorrectionCodes.cs @@ -0,0 +1,50 @@ +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The LDS correction reason (Correction_Reason) for a Remove request. +/// KS2/KS4: the numeric part of the Zendesk "Correction reason (31)" option the journey's reason +/// maps to ("4_31" → "4") — ZendeskTicketFieldOptions.CorrectionReason31 is the business-confirmed +/// source. 16-19: the codes listed in LDS_CYPMD_Data specification v2.4 "Remove Learner" K11, +/// keyed on the Post16 journey's reason and, for not-on-roll, its not-on-roll-reason. Anything +/// else is null — a missing code fails the record, it is never guessed. +/// +public static class CorrectionCodes +{ + public static string? RemoveReasonCode(CheckingWindowType windowType, string? reasonValue, string? notOnRollReason) + { + if (string.IsNullOrWhiteSpace(reasonValue)) + return null; + var reason = reasonValue.Trim(); + + if (windowType == CheckingWindowType.Post16) + return Post16Code(reason, notOnRollReason?.Trim()); + + var zendesk = ZendeskTicketFieldOptions.GetOptionValue(ZendeskTicketFieldConstants.CorrectionReason31Name, reason); + if (zendesk is null) return null; + var suffix = zendesk.LastIndexOf("_31", StringComparison.Ordinal); + return suffix > 0 ? zendesk[..suffix] : zendesk; + } + + // v2.4 K11: 4 Deceased; 325 Not at end of 16-18 study; 326 Not on Roll (International Student); + // 328 Not on Roll (External); 331 Not on Roll (Apprentice); 329 Other - with evidence / + // Not on Roll (Other - with evidence); 330 Other - evidence not required. CYPMD's Post16 "other" + // routes always collect evidence, so 330 is never produced here (FLAGGED in the PR notes). + private static string? Post16Code(string reason, string? notOnRollReason) => reason.ToLowerInvariant() switch + { + "student-died" => "4", + "not-at-end-of-16-19-study" => "325", + "other" => "329", + "not-on-roll" => notOnRollReason?.ToLowerInvariant() switch + { + "apprentice" => "331", + "external-candidate" => "328", + "international-student" => "326", + "other" => "329", + _ => null + }, + _ => null + }; +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressAnswers.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressAnswers.cs new file mode 100644 index 000000000..a3b00f1a3 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressAnswers.cs @@ -0,0 +1,35 @@ +using DfE.CheckPerformanceData.Application.Journey; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// Flattens a journey's answers to strings keyed by question id. Dates become yyyy-MM-dd (the +/// journey blob stores DateAnswer parts, never an ISO string — docs/add-pupil-journey.md), an +/// autocomplete's stable code wins over its label, checkbox values join with "|", and a blank +/// answer is omitted so callers can use "missing" and "empty" interchangeably. +/// +public static class EgressAnswers +{ + public static IReadOnlyDictionary Flatten(RequestState state) + { + var flat = new Dictionary(StringComparer.Ordinal); + foreach (var (questionId, answer) in state.QuestionAnswers) + { + var value = ValueOf(answer); + if (!string.IsNullOrWhiteSpace(value)) + flat[questionId] = value.Trim(); + } + return flat; + } + + private static string? ValueOf(QuestionAnswer answer) + { + if (answer.DateValue is { } date) + return date.ToDateOnly()?.ToString("yyyy-MM-dd"); + if (!string.IsNullOrWhiteSpace(answer.CodeValue)) + return answer.CodeValue; + if (answer.SelectedValues is { Count: > 0 } selected) + return string.Join("|", selected); + return answer.TextValue; + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressColumnSets.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressColumnSets.cs new file mode 100644 index 000000000..49cff392e --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressColumnSets.cs @@ -0,0 +1,98 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// One column of an LDS file: the exact heading and how a row yields its value. +public sealed record EgressColumn(string Header, Func Value); + +/// +/// THE definition of each LDS file: heading text, order and count (AB#292610 says headings must +/// match the spec exactly, with no extra columns). Both sets are LDS_CYPMD_Data specification v2.4 +/// read top to bottom ("New Learner" and "Remove Learner" sheets); the spec marks some attributes +/// N/A for a key stage, so the *For(windowType) methods are what callers use — the base lists are +/// the columns every key stage shares. Change a file's shape here and nowhere else. +/// +public static class EgressColumnSets +{ + public static readonly IReadOnlyList> RemoveLearners = + [ + new("Correction_ID", r => r.CorrectionId), + new("Correction_Type", r => r.CorrectionType), + new("Correction_Reason", r => r.CorrectionReason), + new("Key_Stage", r => r.KeyStage), + new("Establishment_Number", r => r.EstablishmentNumber), + new("Surname", r => r.Surname), + new("Forename", r => r.Forename), + new("Sex", r => r.Sex), + new("Date_of_Birth", r => r.DateOfBirth), + new("Cycle_Year", r => r.CycleYear), + new("Cycle_Month", r => r.CycleMonth), + new("Local_Authority", r => r.LocalAuthority), + new("Learner_ID", r => r.LearnerId) + ]; + + // v2.4 "New Learner": Middle_Name (row 15) is struck through — "CYPMD will not be sending this + // field from June 2026" — so it is not emitted at all. There is no SEN attribute in the spec. + public static readonly IReadOnlyList> NewLearners = + [ + new("Correction_ID", r => r.CorrectionId), + new("Correction_Type", r => r.CorrectionType), + new("Key_Stage", r => r.KeyStage), + new("Establishment_Number", r => r.EstablishmentNumber), + new("Surname", r => r.Surname), + new("Forename", r => r.Forename), + new("Sex", r => r.Sex), + new("Date_of_Birth", r => r.DateOfBirth), + new("Admission_Date", r => r.AdmissionDate), + new("Post_Code", r => r.Postcode), + new("Cycle_Year", r => r.CycleYear), + new("Cycle_Month", r => r.CycleMonth), + new("Local_Authority", r => r.LocalAuthority), + new("URN", r => r.SchoolUrn), + new("ULN", r => r.Uln), + new("UPN", r => r.Upn), + new("Learner_ID", r => r.LearnerId), + new("Year_Group", r => r.YearGroup) + ]; + + // v2.4 "New Learner" rows 29-32: 16-18 only (N/A for KS2/KS4). CYPMD has no Post16 Add journey, + // so nothing can populate these yet; the headings must still be present and the cells blank + // (all four are NULL-able). A future Post16 Add journey fills them from its own answers. + private static readonly IReadOnlyList> Post16NewLearnerColumns = + [ + new("Attendance_Year_0", _ => string.Empty), + new("Attendance_Year_1", _ => string.Empty), + new("Attendance_Year_2", _ => string.Empty), + new("KS4_Year", _ => string.Empty) + ]; + + public static IReadOnlyList> NewLearnersFor(CheckingWindowType windowType) => windowType switch + { + CheckingWindowType.KS2 or CheckingWindowType.KS4June or CheckingWindowType.KS4Autumn => NewLearners, + CheckingWindowType.Post16 => [.. NewLearners, .. Post16NewLearnerColumns], + _ => throw Unmapped(windowType) + }; + + // v2.4 "Remove Learner" row 22: Year_Group is X for KS4 only ("for year group change requests + // only"; "This field will be removed from the files for KS2 and 16-18"). + private static readonly EgressColumn RemoveYearGroup = new("Year_Group", r => r.YearGroup); + + // v2.4 "Remove Learner" rows 23-25: 16-18 only, TRUE/FALSE per academic year. + private static readonly IReadOnlyList> RemovalYears = + [ + new("Removal_Year_0", r => r.RemovalYear0), + new("Removal_Year_1", r => r.RemovalYear1), + new("Removal_Year_2", r => r.RemovalYear2) + ]; + + public static IReadOnlyList> RemoveLearnersFor(CheckingWindowType windowType) => windowType switch + { + CheckingWindowType.KS2 => RemoveLearners, + CheckingWindowType.KS4June or CheckingWindowType.KS4Autumn => [.. RemoveLearners, RemoveYearGroup], + CheckingWindowType.Post16 => [.. RemoveLearners, .. RemovalYears], + _ => throw Unmapped(windowType) + }; + + private static ArgumentOutOfRangeException Unmapped(CheckingWindowType windowType) => + new(nameof(windowType), windowType, "This window type has no LDS column set. Add it to EgressColumnSets before egressing it."); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressCsvWriter.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressCsvWriter.cs new file mode 100644 index 000000000..71b984b7d --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressCsvWriter.cs @@ -0,0 +1,32 @@ +using System.Text; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// Writes an LDS file: RFC 4180 quoting, CRLF between lines, no line terminator after the final +/// data row ("no additional rows below the final data row"), every value trimmed ("no trailing +/// blank characters"), UTF-8 without a byte-order mark. Deliberately not PupilCsvGenerator, whose +/// AppendLine emits the host platform's line ending and a trailing newline. +/// +public static class EgressCsvWriter +{ + private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false); + + public static byte[] Write(IReadOnlyList> columns, IReadOnlyList rows) + { + var lines = new List(rows.Count + 1) + { + string.Join(",", columns.Select(c => Escape(c.Header))) + }; + lines.AddRange(rows.Select(row => string.Join(",", columns.Select(c => Escape(c.Value(row)))))); + return Utf8NoBom.GetBytes(string.Join("\r\n", lines)); + } + + private static string Escape(string? value) + { + var trimmed = (value ?? string.Empty).Trim(); + return trimmed.IndexOfAny([',', '"', '\n', '\r']) >= 0 + ? $"\"{trimmed.Replace("\"", "\"\"")}\"" + : trimmed; + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressDates.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressDates.cs new file mode 100644 index 000000000..04c67b77b --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressDates.cs @@ -0,0 +1,18 @@ +using DfE.CheckPerformanceData.Application.CheckYourPupilData; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// Every date LDS receives is yyyy-MM-dd. Wraps PupilDateFormatter so the accepted input formats stay in one place. +public static class EgressDates +{ + public static bool TryToIso(string? raw, out string iso) + { + if (string.IsNullOrWhiteSpace(raw)) + { + iso = string.Empty; + return false; + } + + return PupilDateFormatter.TryToIsoDate(raw.Trim(), out iso); + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressDecisions.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressDecisions.cs new file mode 100644 index 000000000..9bd4297c0 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressDecisions.cs @@ -0,0 +1,34 @@ +using DfE.CheckPerformanceData.Application.ZendeskClient; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The decision a pulled record carries: one of the Zendesk "Decision status" option values, or one +/// of the two CYPMD-side markers for a request that has no readable ticket. The filter step keeps +/// approved and auto_approved only (AB#292610). +/// +public static class EgressDecisions +{ + /// The request row has no CrmId — it was never sent to Zendesk. + public const string NoTicket = "no-ticket"; + + /// Zendesk returned nothing for the stored ticket id. + public const string NotFound = "not-found"; + + public static bool IsApproved(string? decision) => + string.Equals(decision, ZendeskTicketFieldOptions.DecisionStatus.Approved, StringComparison.OrdinalIgnoreCase) + || string.Equals(decision, ZendeskTicketFieldOptions.DecisionStatus.AutoApproved, StringComparison.OrdinalIgnoreCase); + + public static string Label(string? decision) => decision?.ToLowerInvariant() switch + { + "approved" => "Approved", + "auto_approved" => "Auto approved", + "rejected" => "Rejected", + "auto_rejected" => "Auto rejected", + "scrutiny" => "Scrutiny", + NoTicket => "No Zendesk ticket", + NotFound => "Ticket not found", + null or "" => "Unknown", + var other => other + }; +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressOutputTypes.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressOutputTypes.cs new file mode 100644 index 000000000..4902231e6 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressOutputTypes.cs @@ -0,0 +1,75 @@ +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The one place that says what each output type is: which amendment journey feeds it, what it is +/// called, how its file is named and which LDS correction type it carries. Every switch throws on +/// an unmapped member so a new EgressOutputType cannot ship half-wired. +/// +public static class EgressOutputTypes +{ + public static readonly IReadOnlyList All = + [EgressOutputType.NewLearners, EgressOutputType.RemoveLearners]; + + public static WhatToChange WhatToChangeFor(EgressOutputType type) => type switch + { + EgressOutputType.NewLearners => WhatToChange.Add, + EgressOutputType.RemoveLearners => WhatToChange.Remove, + _ => throw Unmapped(type) + }; + + public static string Label(EgressOutputType type) => type switch + { + EgressOutputType.NewLearners => "New learners", + EgressOutputType.RemoveLearners => "Remove learners", + _ => throw Unmapped(type) + }; + + public static string FileToken(EgressOutputType type) => type switch + { + EgressOutputType.NewLearners => "NewLearners", + EgressOutputType.RemoveLearners => "RemoveLearners", + _ => throw Unmapped(type) + }; + + // LDS correction types (AB#292610): 10 = new learner, 31 = remove learner. The Zendesk field + // carries them as "31_" — the trailing underscore never reaches the output. + public static string CorrectionType(EgressOutputType type) => type switch + { + EgressOutputType.NewLearners => "10", + EgressOutputType.RemoveLearners => "31", + _ => throw Unmapped(type) + }; + + // The "stage" in the file name. 16-19 is KS5 to LDS even though nothing else in this codebase + // calls it that (CheckingWindowType.Post16, KeyStages.Post16). + public static string StageToken(CheckingWindowType windowType) => windowType switch + { + CheckingWindowType.KS4June => "KS4", + CheckingWindowType.KS4Autumn => "KS4", + CheckingWindowType.KS2 => "KS2", + CheckingWindowType.Post16 => "KS5", + _ => throw new ArgumentOutOfRangeException(nameof(windowType), windowType, + "This window type has no LDS stage token. Add it to EgressOutputTypes.StageToken before egressing it.") + }; + + // The Key_Stage CELL in both LDS files (LDS_CYPMD_Data specification v2.4: "KS2, KS4, 16-19"). + // Deliberately not StageToken: LDS names the 16-19 FILE KS5 but the column value is 16-19. + public static string KeyStageValue(CheckingWindowType windowType) => windowType switch + { + CheckingWindowType.KS4June => "KS4", + CheckingWindowType.KS4Autumn => "KS4", + CheckingWindowType.KS2 => "KS2", + CheckingWindowType.Post16 => "16-19", + _ => throw new ArgumentOutOfRangeException(nameof(windowType), windowType, + "This window type has no LDS Key_Stage value. Add it to EgressOutputTypes.KeyStageValue before egressing it.") + }; + + public static string FileName(CheckingWindowType windowType, EgressOutputType type, DateOnly exportDate) => + $"CYPMD_LDS_{StageToken(windowType)}_{FileToken(type)}_{exportDate:yyyy_MM_dd}.csv"; + + private static ArgumentOutOfRangeException Unmapped(EgressOutputType type) => + new(nameof(type), type, "This output type is not mapped. Add it to every switch in EgressOutputTypes before offering it on the Pull page."); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressPreprocessor.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressPreprocessor.cs new file mode 100644 index 000000000..0a92f0b66 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressPreprocessor.cs @@ -0,0 +1,168 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The Preprocessing step (AB#294553). Server-side, deterministic, reported one step at a time. +/// All-or-nothing: a single failing record leaves the run PreprocessingFailed with every failure +/// listed and NOTHING written, so no record is ever silently lost. Cancellation (the browser left +/// the stream) puts the run back where it was; the only durable write is the final step. +/// +public sealed class EgressPreprocessor(IEgressRunRepository repository, IWindowService windows, TimeProvider clock, ILogger logger) : IEgressPreprocessor +{ + public static readonly string[] StepNames = + [ + "Filter records", + EgressRecordBuilder.StepCodes, + EgressRecordBuilder.StepSplit, + EgressRecordBuilder.StepDates, + EgressRecordBuilder.StepBuild, + EgressRecordBuilder.StepTrim, + LdsSpecValidator.StepName, + "Save to database" + ]; + + private static readonly TimeZoneInfo London = TimeZoneInfo.FindSystemTimeZoneById("Europe/London"); + + public async IAsyncEnumerable RunAsync(Guid runId, [EnumeratorCancellation] CancellationToken ct) + { + var run = await repository.GetRunAsync(runId, ct); + if (run is null) + { + yield return Terminal(0, "Preprocessing", "This egress run no longer exists.", 0, 0, 0, isError: true, null); + yield break; + } + // M2: a PreprocessingFailed run releases its pair (IsActive=false) so that a fresh run for + // the same window/type is admitted — the Failed page's own copy says "start a new run". + // Re-running the failed run instead could reach Preprocessed while a colleague's fresh run + // also holds the pair, and both could then transfer it. A failed run is terminal here; it + // gets the same refusal shape as any other non-runnable status. + if (run.Status is not (EgressRunStatus.Pulled or EgressRunStatus.Preprocessed)) + { + // Second-pass nit: use the same human label Index.cshtml shows for this status, not + // the raw enum member name (e.g. "PreprocessingFailed" leaking straight to the user). + // Final-review nit: interpolating the label straight into "This run is {label} and + // cannot be preprocessed" reads as ungrammatical English for several labels (e.g. + // "This run is Transfer failed and cannot be preprocessed."). Phrase it as a stage + // statement instead, which reads correctly for every label in EgressRunStatuses.Label. + yield return Terminal(0, "Preprocessing", $"This run cannot be preprocessed because its stage is {EgressRunStatuses.Label(run.Status)}.", 0, 0, 0, isError: true, null); + yield break; + } + var previous = run.Status; + if (!await repository.TrySetStatusAsync(runId, previous, EgressRunStatus.Preprocessing, ct)) + { + yield return Terminal(0, "Preprocessing", "This run is being preprocessed by someone else.", 0, 0, 0, isError: true, null); + yield break; + } + + var completed = false; + try + { + await foreach (var progress in StepsAsync(run, ct)) + { + if (progress.IsComplete) completed = true; + yield return progress; + } + } + finally + { + if (!completed) + { + // Aborted mid-run (the stream was closed, or an exception): nothing durable happened + // after the status flip, so undo the flip. CancellationToken.None — the token that + // got us here is already cancelled. + await repository.TrySetStatusAsync(runId, EgressRunStatus.Preprocessing, previous, CancellationToken.None); + } + } + } + + private async IAsyncEnumerable StepsAsync(EgressRunDto run, [EnumeratorCancellation] CancellationToken ct) + { + // The window drives the Key_Stage cell, Cycle_Year/Cycle_Month (spec: the exercise's month, + // not each record's submission month) and the file-name stage — load it once, up front. + var window = await windows.GetByIdAsync(run.WindowId, ct) + ?? throw new InvalidOperationException($"Checking window {run.WindowId} for egress run {run.Id} no longer exists."); + var windowType = window.CheckingWindowType; + var cycleYear = window.StartDate.Year.ToString(CultureInfo.InvariantCulture); + var cycleMonth = window.StartDate.Month.ToString(CultureInfo.InvariantCulture); + var all = run.Outputs.SelectMany(o => o.Records).ToList(); + + // 1. Filter + yield return Running(1, all.Count); + ct.ThrowIfCancellationRequested(); + var items = all.Where(r => EgressDecisions.IsApproved(r.Decision)).Select(r => new EgressWorkItem(r, cycleYear, cycleMonth)).ToList(); + yield return Done(1, all.Count, items.Count, 0, $"{items.Count} of {all.Count} records are approved or auto-approved; {all.Count - items.Count} discarded."); + + // 2-6. Per-record transforms + Action[] transforms = [EgressRecordBuilder.DeriveCodes, EgressRecordBuilder.SplitEstablishment, EgressRecordBuilder.StandardiseDates, EgressRecordBuilder.Build, EgressRecordBuilder.Trim]; + for (var i = 0; i < transforms.Length; i++) + { + var step = i + 2; + yield return Running(step, items.Count); + ct.ThrowIfCancellationRequested(); + foreach (var item in items) transforms[i](item); + var failed = items.Count(x => x.HasFailed); + yield return Done(step, items.Count, items.Count - failed, failed, failed == 0 ? "Completed." : $"{failed} record(s) have problems so far."); + } + + // 7. Validate + yield return Running(7, items.Count); + ct.ThrowIfCancellationRequested(); + foreach (var item in items.Where(x => !x.HasFailed)) + { + var failures = item.NewRow is { } n ? LdsSpecValidator.Validate(n) + : item.RemoveRow is { } r ? LdsSpecValidator.Validate(r) : []; + item.Failures.AddRange(failures); + } + var allFailures = items.SelectMany(x => x.Failures).ToList(); + yield return Done(7, items.Count, items.Count(x => !x.HasFailed), allFailures.Count, + allFailures.Count == 0 ? "Every record conforms to the LDS spec." : $"{allFailures.Count} problem(s) across {items.Count(x => x.HasFailed)} record(s)."); + + // 8. Save (or fail the batch) + yield return Running(8, items.Count); + ct.ThrowIfCancellationRequested(); + if (allFailures.Count > 0) + { + // M4: 0 rows means the run was abandoned by someone else while this pipeline ran — do + // not report PreprocessingFailed for a run that is actually Abandoned. + if (await repository.MarkPreprocessingFailedAsync(run.Id, EgressRunStatus.Preprocessing, allFailures, ct) == 0) + { + logger.LogWarning("Egress run {RunId} was abandoned while preprocessing; its failures were not recorded", run.Id); + yield return Terminal(8, StepNames[7], "This run was abandoned while preprocessing.", items.Count, 0, allFailures.Count, isError: true, null); + yield break; + } + logger.LogWarning("Egress run {RunId} failed preprocessing with {Count} record failure(s)", run.Id, allFailures.Count); + yield return Terminal(8, StepNames[7], "Preprocessing stopped: no records were saved because some records failed. Correct the source data and start a new run.", + items.Count, 0, allFailures.Count, isError: true, EgressRunStatus.PreprocessingFailed); + yield break; + } + + var exportDate = DateOnly.FromDateTime(TimeZoneInfo.ConvertTime(clock.GetUtcNow(), London).DateTime); + var fileNames = run.Outputs.ToDictionary(o => o.OutputType, o => EgressOutputTypes.FileName(windowType, o.OutputType, exportDate)); + var newRows = items.Select(x => x.NewRow).OfType().ToList(); + var removeRows = items.Select(x => x.RemoveRow).OfType().ToList(); + // M4: 0 rows means the run was abandoned by someone else while this pipeline ran — do not + // report Preprocessed (with nothing actually saved) for a run that is actually Abandoned. + if (await repository.SavePreprocessedAsync(run.Id, EgressRunStatus.Preprocessing, newRows, removeRows, exportDate, fileNames, ct) == 0) + { + logger.LogWarning("Egress run {RunId} was abandoned while preprocessing; nothing was saved", run.Id); + yield return Terminal(8, StepNames[7], "This run was abandoned while preprocessing.", items.Count, 0, 0, isError: true, null); + yield break; + } + yield return Terminal(8, StepNames[7], $"Saved {newRows.Count + removeRows.Count} record(s) to the database.", items.Count, newRows.Count + removeRows.Count, 0, + isError: false, EgressRunStatus.Preprocessed); + } + + private static EgressProgress Running(int step, int recordsIn) => + new(step, StepNames.Length, StepNames[step - 1], "running", recordsIn, 0, 0, false, false, $"{StepNames[step - 1]}…", null); + + private static EgressProgress Done(int step, int recordsIn, int recordsOut, int failures, string message) => + new(step, StepNames.Length, StepNames[step - 1], "done", recordsIn, recordsOut, failures, false, false, message, null); + + private static EgressProgress Terminal(int step, string stepName, string message, int recordsIn, int recordsOut, int failures, bool isError, EgressRunStatus? final) => + new(step, StepNames.Length, stepName, isError ? "failed" : "done", recordsIn, recordsOut, failures, true, isError, message, final); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressRecordBuilder.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressRecordBuilder.cs new file mode 100644 index 000000000..abdda9336 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressRecordBuilder.cs @@ -0,0 +1,190 @@ +using System.Globalization; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// Mutable per-record state carried through steps 2-7. One instance per pulled record that survived the filter. +/// Cycle_Year/Cycle_Month are the WINDOW's (spec: the month the exercise takes place), so every row of a run agrees. +public sealed class EgressWorkItem(EgressSourceRecord source, string cycleYear, string cycleMonth) +{ + public EgressSourceRecord Source { get; } = source; + public string CycleYear { get; } = cycleYear; + public string CycleMonth { get; } = cycleMonth; + public string? CorrectionType { get; set; } + public string? CorrectionReason { get; set; } + public string LocalAuthority { get; set; } = string.Empty; + public string Establishment { get; set; } = string.Empty; + public string DateOfBirthIso { get; set; } = string.Empty; + public string AdmissionDateIso { get; set; } = string.Empty; + public List Failures { get; } = []; + public NewLearnerRow? NewRow { get; set; } + public RemoveLearnerRow? RemoveRow { get; set; } + + public bool HasFailed => Failures.Count > 0; + + public void Fail(string step, string field, string reason) => + Failures.Add(new EgressRecordFailure(step, Source.TicketId, Source.ReferenceNumber, field, reason)); +} + +/// +/// Steps 2-6 of the preprocessing pipeline, one pure-ish function each, so the pipeline can report +/// them one at a time. A step that finds a problem records it against the field LDS would see and +/// leaves the item failed; later steps skip a failed item rather than piling on secondary errors. +/// +public static class EgressRecordBuilder +{ + public const string StepCodes = "Derive correction codes"; + public const string StepSplit = "Split DfE establishment number"; + public const string StepDates = "Standardise dates"; + public const string StepBuild = "Build LDS records"; + public const string StepTrim = "Trim values"; + + public static void DeriveCodes(EgressWorkItem item) + { + item.CorrectionType = EgressOutputTypes.CorrectionType(item.Source.OutputType); + if (item.Source.OutputType != EgressOutputType.RemoveLearners) return; + + var reason = item.Source.Answer("reason"); + var notOnRollReason = item.Source.Answer("not-on-roll-reason"); + var code = CorrectionCodes.RemoveReasonCode(item.Source.WindowType, reason, notOnRollReason); + if (code is null) + { + var detail = notOnRollReason is null ? $"'{reason ?? "(none)"}'" : $"'{reason}' / '{notOnRollReason}'"; + item.Fail(StepCodes, "Correction_Reason", $"No LDS correction reason code is defined for removal reason {detail}"); + } + item.CorrectionReason = code; + } + + // S4: steps 2-4 are independent of one another (a bad correction reason says nothing about + // whether the LAESTAB is valid), so none of them gates on HasFailed — only Build/Trim/Validate + // do, once every independent check has had its turn. A record with two unrelated faults lists + // both instead of ops discovering the second one only on the next run. + // The pupil record's LAESTAB first (a real pupil always has one), then the school's from the + // request row (the only source for a new learner, whose synthetic pupil has none). + public static void SplitEstablishment(EgressWorkItem item) + { + var raw = !string.IsNullOrWhiteSpace(item.Source.PupilLaestab) ? item.Source.PupilLaestab : item.Source.OrganisationLaestab; + if (LaestabSplitter.TrySplit(raw, out var la, out var estab)) + { + item.LocalAuthority = la; + item.Establishment = estab; + return; + } + item.Fail(StepSplit, "Establishment_Number", + string.IsNullOrWhiteSpace(raw) + ? "No 7-digit DfE establishment number is held for this request or its school" + : $"DfE establishment number '{raw}' is not 7 digits"); + } + + public static void StandardiseDates(EgressWorkItem item) + { + var dobRaw = item.Source.OutputType == EgressOutputType.NewLearners + ? item.Source.Answer("date-of-birth") ?? item.Source.PupilDateOfBirth + : item.Source.PupilDateOfBirth; + if (EgressDates.TryToIso(dobRaw, out var dob)) item.DateOfBirthIso = dob; + else item.Fail(StepDates, "Date_of_Birth", $"Date of birth '{dobRaw ?? "(none)"}' is not a recognisable date"); + + if (item.Source.OutputType == EgressOutputType.NewLearners) + { + var admissionRaw = item.Source.Answer("admission-date"); + if (EgressDates.TryToIso(admissionRaw, out var admission)) item.AdmissionDateIso = admission; + else item.Fail(StepDates, "Admission_Date", $"Admission date '{admissionRaw ?? "(none)"}' is not a recognisable date"); + } + } + + public static void Build(EgressWorkItem item) + { + if (item.HasFailed) return; + var s = item.Source; + if (!s.JourneyFound) + { + item.Fail(StepBuild, "Correction_ID", "The request's journey record could not be read, so its values are unknown"); + return; + } + var stage = EgressOutputTypes.KeyStageValue(s.WindowType); + var ticket = s.TicketId?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + + if (s.OutputType == EgressOutputType.RemoveLearners) + { + var (year0, year1, year2) = RemovalYears(s, item.CycleYear); + item.RemoveRow = new RemoveLearnerRow( + CorrectionId: ticket, CorrectionType: item.CorrectionType ?? string.Empty, CorrectionReason: item.CorrectionReason ?? string.Empty, + KeyStage: stage, EstablishmentNumber: item.Establishment, Surname: s.PupilSurname ?? string.Empty, Forename: s.PupilFirstname ?? string.Empty, + Sex: (s.PupilSex ?? string.Empty).ToUpperInvariant(), DateOfBirth: item.DateOfBirthIso, CycleYear: item.CycleYear, CycleMonth: item.CycleMonth, + LocalAuthority: item.LocalAuthority, + LearnerId: s.PupilMatchRef > 0 ? s.PupilMatchRef.ToString(CultureInfo.InvariantCulture) : string.Empty, + ChangeRequestId: s.ChangeRequestId, TicketId: s.TicketId, ReferenceNumber: s.ReferenceNumber) + { + YearGroup = YearGroupMovedTo(s), RemovalYear0 = year0, RemovalYear1 = year1, RemovalYear2 = year2 + }; + return; + } + + // A new learner's identity comes from the typed journey answers (docs/add-pupil-journey.md); + // the synthetic pupil mirrors them but the answers are the record of what the school typed. + item.NewRow = new NewLearnerRow( + CorrectionId: ticket, CorrectionType: item.CorrectionType ?? string.Empty, KeyStage: stage, + LocalAuthority: item.LocalAuthority, EstablishmentNumber: item.Establishment, + Surname: s.Answer("last-name") ?? s.PupilSurname ?? string.Empty, + Forename: s.Answer("first-name") ?? s.PupilFirstname ?? string.Empty, + Sex: (s.Answer("sex") ?? s.PupilSex ?? string.Empty).ToUpperInvariant(), + DateOfBirth: item.DateOfBirthIso, AdmissionDate: item.AdmissionDateIso, Postcode: string.Empty, + CycleYear: item.CycleYear, CycleMonth: item.CycleMonth, SchoolUrn: s.OrganisationUrn.ToString(CultureInfo.InvariantCulture), + Uln: s.WindowType == CheckingWindowType.Post16 ? s.PupilIdentifier ?? string.Empty : string.Empty, + Upn: (s.Answer("upn") ?? string.Empty).ToUpperInvariant(), + LearnerId: s.PupilMatchRef > 0 ? s.PupilMatchRef.ToString(CultureInfo.InvariantCulture) : string.Empty, + YearGroup: s.Answer("year-group") ?? string.Empty, + ChangeRequestId: s.ChangeRequestId, TicketId: s.TicketId, ReferenceNumber: s.ReferenceNumber); + } + + public static void Trim(EgressWorkItem item) + { + if (item.HasFailed) return; + if (item.RemoveRow is { } r) + item.RemoveRow = r with + { + CorrectionId = r.CorrectionId.Trim(), CorrectionType = r.CorrectionType.Trim(), CorrectionReason = r.CorrectionReason.Trim(), + KeyStage = r.KeyStage.Trim(), EstablishmentNumber = r.EstablishmentNumber.Trim(), Surname = r.Surname.Trim(), Forename = r.Forename.Trim(), + Sex = r.Sex.Trim(), DateOfBirth = r.DateOfBirth.Trim(), CycleYear = r.CycleYear.Trim(), CycleMonth = r.CycleMonth.Trim(), + LocalAuthority = r.LocalAuthority.Trim(), LearnerId = r.LearnerId.Trim(), YearGroup = r.YearGroup.Trim(), + RemovalYear0 = r.RemovalYear0.Trim(), RemovalYear1 = r.RemovalYear1.Trim(), RemovalYear2 = r.RemovalYear2.Trim() + }; + if (item.NewRow is { } n) + item.NewRow = n with + { + CorrectionId = n.CorrectionId.Trim(), CorrectionType = n.CorrectionType.Trim(), KeyStage = n.KeyStage.Trim(), + LocalAuthority = n.LocalAuthority.Trim(), EstablishmentNumber = n.EstablishmentNumber.Trim(), Surname = n.Surname.Trim(), + Forename = n.Forename.Trim(), Sex = n.Sex.Trim(), DateOfBirth = n.DateOfBirth.Trim(), + AdmissionDate = n.AdmissionDate.Trim(), Postcode = n.Postcode.Trim(), CycleYear = n.CycleYear.Trim(), CycleMonth = n.CycleMonth.Trim(), + SchoolUrn = n.SchoolUrn.Trim(), Uln = n.Uln.Trim(), Upn = n.Upn.Trim(), LearnerId = n.LearnerId.Trim(), + YearGroup = n.YearGroup.Trim() + }; + } + + // v2.4 Remove Learner row 22: KS4 files carry Year_Group "for year group change requests only". + // The KS4 Remove journey asks higher-lower, then year-group-higher-moved-to (12/13) or + // year-group-lower-moved-to (8/9/10); a stale answer from the other branch can linger in the + // blob, so the branch actually chosen decides which one is read. + private static string YearGroupMovedTo(EgressSourceRecord s) + { + if (s.WindowType is not (CheckingWindowType.KS4June or CheckingWindowType.KS4Autumn)) return string.Empty; + if (!string.Equals(s.Answer("reason"), "year-group-change", StringComparison.OrdinalIgnoreCase)) return string.Empty; + var lower = string.Equals(s.Answer("higher-lower"), "lower", StringComparison.OrdinalIgnoreCase); + return (lower ? s.Answer("year-group-lower-moved-to") : s.Answer("year-group-higher-moved-to")) ?? string.Empty; + } + + // v2.4 Remove Learner rows 23-25: 16-19 files carry TRUE/FALSE per academic year. The Post16 + // Remove journey's years-to-remove checkbox uses labels "YYYY-YYYY" (EgressAnswers joins them + // with "|"); Year_0 is the academic year ending in Cycle_Year, Year_1 the one before, Year_2 + // the one before that. Only the "other" route asks the question — every other reason leaves + // all three blank (NULL-able in the spec) rather than guessing. + private static (string Year0, string Year1, string Year2) RemovalYears(EgressSourceRecord s, string cycleYear) + { + var answer = s.Answer("years-to-remove"); + if (s.WindowType != CheckingWindowType.Post16 || string.IsNullOrWhiteSpace(answer) || !int.TryParse(cycleYear, out var end)) + return (string.Empty, string.Empty, string.Empty); + var chosen = answer.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(StringComparer.Ordinal); + string Flag(int n) => chosen.Contains($"{end - 1 - n}-{end - n}") ? "TRUE" : "FALSE"; + return (Flag(0), Flag(1), Flag(2)); + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressRunService.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressRunService.cs new file mode 100644 index 000000000..aaa68b12f --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressRunService.cs @@ -0,0 +1,126 @@ +using DfE.CheckPerformanceData.Application.Journey; +using DfE.CheckPerformanceData.Application.RequestSubmission; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The Pull step (AB#294553). Refuses first, pulls second: every candidate request for the window +/// and output types is returned with the decision Zendesk holds for it — nothing is filtered here, +/// so the ops user can see the pull worked before anything is discarded. Zendesk supplies ONLY the +/// decision; every record value is CYPMD's own (design decision 1, 2026-09-14). +/// +public sealed class EgressRunService( + IEgressRunRepository repository, + IEgressTicketSource tickets, + IRequestStateBlobClient journeys, + IWindowService windows, + ILogger logger) : IEgressRunService +{ + public async Task StartAsync(Guid windowId, IReadOnlyList outputTypes, EgressActor actor, CancellationToken ct) + { + if (outputTypes.Count == 0) + throw new ArgumentException("At least one output type is required.", nameof(outputTypes)); + var types = outputTypes.Distinct().ToList(); + + var blockers = await BlockersAsync(windowId, types, ct); + if (blockers.Count > 0) return new EgressStartResult.Refused(blockers); + + var window = await windows.GetByIdAsync(windowId, ct); + if (window is null) return new EgressStartResult.WindowNotFound(); + + var outputs = new List(types.Count); + try + { + foreach (var type in types) + outputs.Add(new EgressRunOutputCreate(type, await PullAsync(window, type, ct))); + } + catch (EgressTicketSourceException ex) + { + logger.LogWarning(ex, "Egress pull for window {WindowId} could not read Zendesk decisions", windowId); + return new EgressStartResult.PullFailed(ex.Message); + } + + try + { + var runId = await repository.CreateRunAsync(new EgressRunCreate(windowId, actor.UserId, actor.DisplayName, actor.Email, outputs), ct); + return new EgressStartResult.Started(runId); + } + catch (EgressRunConflictException) + { + // Lost the race to another admin between the check and the insert: tell them who won. + return new EgressStartResult.Refused(await BlockersAsync(windowId, types, ct)); + } + } + + private async Task> BlockersAsync(Guid windowId, IReadOnlyList types, CancellationToken ct) + { + var blockers = new List<(EgressOutputType, EgressBlocker)>(); + foreach (var type in types) + { + var blocker = await repository.FindBlockerAsync(windowId, type, ct); + if (blocker is not null) blockers.Add((type, blocker)); + } + return blockers; + } + + private async Task> PullAsync(CheckingWindowDto window, EgressOutputType type, CancellationToken ct) + { + var candidates = await repository.GetCandidateRequestsAsync(window.Id, EgressOutputTypes.WhatToChangeFor(type), ct); + var ticketIds = candidates.Select(c => TicketId(c.CrmId)).OfType().ToList(); + var decisions = ticketIds.Count == 0 + ? new Dictionary() + : await tickets.GetDecisionStatusesAsync(ticketIds, ct); + + var records = new List(candidates.Count); + foreach (var candidate in candidates) + { + var ticketId = TicketId(candidate.CrmId); + var decision = ticketId is null ? EgressDecisions.NoTicket + : decisions.TryGetValue(ticketId.Value, out var d) ? d + : EgressDecisions.NotFound; + + RequestState? journey; + try { journey = await journeys.GetAsync(window.Id, candidate.ReferenceNumber); } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Egress could not read the journey blob for {Reference}", candidate.ReferenceNumber); + journey = null; + } + var pupil = journey?.SelectedPupil; + + records.Add(new EgressSourceRecord + { + ChangeRequestId = candidate.ChangeRequestId, + ReferenceNumber = candidate.ReferenceNumber, + TicketId = ticketId, + Decision = decision, + OutputType = type, + WindowType = window.CheckingWindowType, + SubmittedAtUtc = candidate.SubmittedAtUtc, + OrganisationUrn = candidate.OrganisationUrn, + OrganisationLaestab = candidate.OrganisationLaestab, + PupilFirstname = pupil?.Firstname, + PupilSurname = pupil?.Surname, + PupilDateOfBirth = pupil?.DateOfBirth, + PupilSex = pupil?.Sex, + PupilIdentifier = pupil?.Identifier, + PupilCypmdId = pupil?.Cypmd_Id, + PupilMatchRef = pupil?.MatchRef ?? 0, + PupilLaestab = pupil?.Laestab, + PupilEntryDate = pupil?.EntryDate, + JourneyFound = journey is not null, + Answers = journey is null ? new Dictionary() : EgressAnswers.Flatten(journey) + }); + } + return records; + } + + private static long? TicketId(string? crmId) => + long.TryParse(crmId, out var id) && id > 0 ? id : null; + + public Task GetAsync(Guid runId, CancellationToken ct) => repository.GetRunAsync(runId, ct); + public Task> ListAsync(CancellationToken ct) => repository.ListRunsAsync(ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressRunStatuses.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressRunStatuses.cs new file mode 100644 index 000000000..fb4b46f8b --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressRunStatuses.cs @@ -0,0 +1,24 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The one place that maps an EgressRunStatus to what a user reads — the saved-runs "Stage" +/// column (Index.cshtml) and any refusal copy that would otherwise leak the C# enum member name +/// (e.g. "PreprocessingFailed" rendering literally instead of "Preprocessing failed"). +/// +public static class EgressRunStatuses +{ + public static string Label(EgressRunStatus status) => status switch + { + EgressRunStatus.Pulled => "Data pulled", + EgressRunStatus.Preprocessing => "Preprocessing", + EgressRunStatus.PreprocessingFailed => "Preprocessing failed", + EgressRunStatus.Preprocessed => "Ready to transfer", + EgressRunStatus.Transferring => "Transferring", + EgressRunStatus.TransferFailed => "Transfer failed", + EgressRunStatus.Transferred => "Transferred", + EgressRunStatus.Abandoned => "Abandoned", + _ => status.ToString() + }; +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressSourceRecord.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressSourceRecord.cs new file mode 100644 index 000000000..8e23a2857 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressSourceRecord.cs @@ -0,0 +1,45 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// One request as pulled: the decision Zendesk holds for it plus every CYPMD-held value the LDS +/// files can need. This is what the Results screen shows and what a saved run keeps (as JSON on +/// EgressRunOutput), so resuming never re-pulls. Nothing here is transformed — raw dates, raw +/// laestab, raw reason values — transformation is the preprocessing pipeline's job and is shown +/// to the user as it happens. +/// +public sealed record EgressSourceRecord +{ + public required Guid ChangeRequestId { get; init; } + public required string ReferenceNumber { get; init; } + public long? TicketId { get; init; } + /// A Zendesk decision value, or EgressDecisions.NoTicket / NotFound. + public required string Decision { get; init; } + public required EgressOutputType OutputType { get; init; } + public required CheckingWindowType WindowType { get; init; } + public required DateTime SubmittedAtUtc { get; init; } + public required long OrganisationUrn { get; init; } + /// The school's LAESTAB as stamped on the request row at submit; null on rows written before the column existed. + public string? OrganisationLaestab { get; init; } + public string? PupilFirstname { get; init; } + public string? PupilSurname { get; init; } + public string? PupilDateOfBirth { get; init; } + public string? PupilSex { get; init; } + /// UPN for KS4, ULN for Post16 (PupilDto.Identifier). + public string? PupilIdentifier { get; init; } + public string? PupilCypmdId { get; init; } + public int PupilMatchRef { get; init; } + public string? PupilLaestab { get; init; } + public string? PupilEntryDate { get; init; } + /// False when the journey blob could not be read — the record then fails validation with a clear reason. + public bool JourneyFound { get; init; } + /// The journey's answers flattened by question id (EgressAnswers.Flatten). + public IReadOnlyDictionary Answers { get; init; } = new Dictionary(); + + public string? Answer(string questionId) => + Answers.TryGetValue(questionId, out var value) ? value : null; +} + +/// One record's failure at one preprocessing step. Every failure is shown to the ops user; one failure fails the batch. +public sealed record EgressRecordFailure(string Step, long? TicketId, string ReferenceNumber, string Field, string Reason); diff --git a/src/DfE.CheckPerformanceData.Application/Egress/EgressTransferService.cs b/src/DfE.CheckPerformanceData.Application/Egress/EgressTransferService.cs new file mode 100644 index 000000000..df86500ce --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/EgressTransferService.cs @@ -0,0 +1,260 @@ +using System.Security.Cryptography; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The Transfer step (AB#294553). Files are built from the persisted rows — the database is the +/// source of truth and a retry needs no re-pull. Uploads are create-only; if any file fails, every +/// file already written is deleted and the run is marked TransferFailed, so LDS never sees a +/// partial set. Success and failure each write an audit row (repository, same transaction as the +/// state change); only success is ever recorded as such. +/// +public sealed class EgressTransferService(IEgressRunRepository repository, IEgressBlobClient blobs, IWindowService windows, ILogger logger) : IEgressTransferService +{ + public async Task TransferAsync(Guid runId, EgressActor actor, CancellationToken ct) + { + var run = await repository.GetRunAsync(runId, ct); + if (run is null) return new EgressTransferResult.NotFound(); + if (run.Status is not (EgressRunStatus.Preprocessed or EgressRunStatus.TransferFailed)) + return new EgressTransferResult.NotTransferable(run.Status); + + // M3: an approved set that preprocessed to zero saved rows (every record rejected, + // undecided, or lost to a misconfigured ticket source) must never send a header-only file + // and lock the pair forever — refuse before touching storage or flipping status. + if (run.Outputs.All(o => (o.OutputRecordCount ?? 0) == 0)) + return new EgressTransferResult.NothingToTransfer(); + + var windowType = await WindowTypeAsync(run.WindowId, runId, ct); + var fromStatus = run.Status; + if (fromStatus == EgressRunStatus.TransferFailed) + { + var blocked = await repository.TryReactivateAsync(runId, ct); + if (blocked is { } b) return new EgressTransferResult.Refused(b.OutputType, b.Blocker); + } + + if (!blobs.IsConfigured) + return await FailAsync(runId, actor, "Egress storage is not configured for this environment (ConnectionStrings:EgressStorage).", [], null, fromStatus, ct); + + if (!await repository.TrySetStatusAsync(runId, fromStatus, EgressRunStatus.Transferring, ct)) + return new EgressTransferResult.NotTransferable(EgressRunStatus.Transferring); + + // M1: once the run has flipped to Transferring, the caller's token (RequestAborted — a + // closed browser tab) must not be able to leave the run half-transferred. Everything from + // here on, including compensation, runs to completion regardless of the caller's token. + var operationCt = CancellationToken.None; + + var files = new Dictionary(); + var uploaded = new List(); + string? inFlight = null; + try + { + foreach (var output in run.Outputs) + { + var (bytes, records) = await BuildAsync(runId, windowType, output.OutputType, operationCt); + var fileName = output.FileName ?? throw new InvalidOperationException($"Run {runId} has no file name for {output.OutputType}."); + inFlight = fileName; + var sha = Convert.ToHexString(SHA256.HashData(bytes)); + await UploadReclaimingLeftoversAsync(fileName, bytes, sha, runId, operationCt); + inFlight = null; + uploaded.Add(fileName); + files[output.OutputType] = (fileName, records, sha); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Egress run {RunId} transfer failed after {Uploaded} file(s); compensating", runId, uploaded.Count); + return await FailAsync(runId, actor, ex.Message, uploaded, inFlight, EgressRunStatus.Transferring, operationCt); + } + + DateTime at; + int rows; + try + { + at = DateTime.UtcNow; + rows = await repository.MarkTransferredAsync(runId, EgressRunStatus.Transferring, + new EgressTransferAudit(actor.UserId.ToString(), actor.DisplayName, blobs.TargetDescription, files), at, operationCt); + } + catch (Exception ex) + { + logger.LogError(ex, "Egress run {RunId}: MarkTransferredAsync failed after every file was uploaded; compensating", runId); + return await FailAsync(runId, actor, ex.Message, uploaded, null, EgressRunStatus.Transferring, operationCt); + } + if (rows == 0) + { + // M4: lost the race — something else (an Abandon) moved the run off Transferring + // between the CAS flip and this write. Compensate the uploads; no Succeeded audit row + // is written because MarkTransferredAsync's own guard already refused to write one. + logger.LogWarning("Egress run {RunId} was no longer Transferring when the transfer completed; compensating", runId); + return await FailAsync(runId, actor, "This run was abandoned while the transfer was in progress.", uploaded, null, EgressRunStatus.Transferring, operationCt); + } + return new EgressTransferResult.Transferred(files.Select(f => (f.Key, f.Value.FileName, f.Value.Records)).ToList(), at); + } + + // Follow-up to R1 (the Abandon crash window) and to a compensation delete that failed with + // "remove it by hand": a same-named file already in the container is reclaimed — deleted and + // the create-only upload retried exactly once — only when its egressRunId stamp names an + // ABANDONED run (a sweep the process died in the middle of) or this very run (an earlier + // attempt). Both are files nobody wants in LDS and neither has any other UI path out. A file + // stamped by any live run, or carrying no stamp at all, is a real collision: left untouched, + // and the exception propagates to FailAsync — with the reason spelling out who wrote the file + // and what actually gets the operator out, because the file name carries the preprocessing + // date, so simply retrying this run tomorrow collides on the same name again. + private async Task UploadReclaimingLeftoversAsync(string fileName, byte[] bytes, string sha, Guid runId, CancellationToken ct) + { + try + { + await blobs.UploadAsync(fileName, bytes, sha, runId, ct); + return; + } + catch (EgressBlobAlreadyExistsException ex) + { + if (await blobs.GetOwnerRunIdAsync(fileName, ct) is not { } owner) + throw new EgressBlobAlreadyExistsException(ex.BlobName, + "It was not written by this service, so it was left in place. Ask LDS to remove or rename it, then retry."); + var ownerRun = owner == runId ? null : await repository.GetRunAsync(owner, ct); + var reclaimable = owner == runId || ownerRun?.Status == EgressRunStatus.Abandoned; + if (!reclaimable) + throw new EgressBlobAlreadyExistsException(ex.BlobName, + $"Another egress run wrote it{(ownerRun is null ? "" : $" (its status is {EgressRunStatuses.Label(ownerRun.Status)})")}, so it was left in place. " + + "Two checking windows of the same key stage cannot transfer on the same day. " + + "Abandon this run and start a new one on a later day, or ask LDS to remove the file and then retry."); + // Ownership is re-checked at delete time; a false here means the file changed hands + // between the two calls, which is the live-collision case again. + if (!await blobs.DeleteIfOwnedByRunAsync(fileName, owner, ct)) throw; + logger.LogWarning("Egress run {RunId}: reclaimed {File}, a leftover stamped by run {Owner}, before uploading", runId, fileName, owner); + } + await blobs.UploadAsync(fileName, bytes, sha, runId, ct); + } + + // possiblyOrphaned (S3): the file that was mid-upload when the exception was thrown, if any — + // its PUT may have actually landed server-side with the response lost to the client, so it is + // worth a metadata-checked delete attempt distinct from `uploaded` (which this call definitely + // wrote, since a create-only PUT that returned success cannot belong to another run). + // expectedStatus (M4): the status MarkTransferFailedAsync's own guard requires — Transferring + // once the CAS flip has happened, or the run's pre-flip status for the unconfigured-storage + // refusal, which never flips at all. + private async Task FailAsync(Guid runId, EgressActor actor, string reason, IReadOnlyList uploaded, string? possiblyOrphaned, EgressRunStatus expectedStatus, CancellationToken ct) + { + foreach (var name in uploaded) + { + try { await blobs.DeleteIfExistsAsync(name, ct); } + catch (Exception ex) + { + logger.LogError(ex, "Egress run {RunId}: could not delete {File} while compensating a failed transfer", runId, name); + reason += $" Could not remove {name} from the target container — remove it by hand before retrying."; + } + } + if (possiblyOrphaned is not null) + { + try + { + if (await blobs.DeleteIfOwnedByRunAsync(possiblyOrphaned, runId, ct)) + logger.LogWarning("Egress run {RunId}: removed {File}, which had landed despite its upload response failing", runId, possiblyOrphaned); + } + catch (Exception ex) + { + logger.LogError(ex, "Egress run {RunId}: could not check/remove {File} while compensating a failed transfer", runId, possiblyOrphaned); + reason += $" Could not remove {possiblyOrphaned} from the target container — remove it by hand before retrying."; + } + } + await repository.MarkTransferFailedAsync(runId, expectedStatus, reason, actor.UserId.ToString(), ct); + return new EgressTransferResult.Failed(reason); + } + + // M1: the lock has no expiry, so a run stuck in Transferring (a pod restart mid-upload) must be + // releasable. Any blob this run actually wrote is swept first — never a blob owned by another + // run — so the freed pair does not collide with a same-named retry. + public async Task AbandonAsync(Guid runId, CancellationToken ct) + { + var run = await repository.GetRunAsync(runId, ct); + if (run is null) return new EgressAbandonResult.NotFound(); + if (run.Status == EgressRunStatus.Transferred) return new EgressAbandonResult.AlreadyTransferred(); + + // S8 / R1: the write must happen — and its rows-affected be read — before any blob is + // swept, not after. A concurrent TransferAsync can flip Transferring -> Transferred + // between the read above and this write; sweeping first (the earlier ordering) would then + // delete the files that transfer had just uploaded, while this call still went on to lose + // the race and report AlreadyTransferred — leaving the run Transferred, with a Succeeded + // audit row, but no files in LDS. Writing first and branching on what it actually affected + // is what tells us which case happened. The write itself also runs on CancellationToken. + // None, not the caller's token: per M1, once state starts changing, a closed tab must not + // be able to abandon this call half-way. + var wasTransferring = run.Status == EgressRunStatus.Transferring; + var rows = await repository.AbandonAsync(runId, CancellationToken.None); + if (rows == 0) return new EgressAbandonResult.AlreadyTransferred(); + + var removed = new List(); + if (wasTransferring) + { + foreach (var output in run.Outputs) + { + if (output.FileName is null) continue; + if (await blobs.DeleteIfOwnedByRunAsync(output.FileName, runId, CancellationToken.None)) + removed.Add(output.FileName); + } + } + return new EgressAbandonResult.Abandoned(removed); + } + + public async Task BuildFileAsync(Guid runId, EgressOutputType type, CancellationToken ct) => + (await BuildAsync(runId, await WindowTypeOfRunAsync(runId, ct), type, ct)).Bytes; + + public async Task<(IReadOnlyList Headers, IReadOnlyList> Rows)> GetPreviewAsync(Guid runId, EgressOutputType type, CancellationToken ct) + { + var windowType = await WindowTypeOfRunAsync(runId, ct); + switch (type) + { + case EgressOutputType.NewLearners: + { + var columns = EgressColumnSets.NewLearnersFor(windowType); + var rows = await repository.GetNewLearnersAsync(runId, ct); + return (columns.Select(c => c.Header).ToList(), rows.Select(r => (IReadOnlyList)columns.Select(c => c.Value(r)).ToList()).ToList()); + } + case EgressOutputType.RemoveLearners: + { + var columns = EgressColumnSets.RemoveLearnersFor(windowType); + var rows = await repository.GetRemoveLearnersAsync(runId, ct); + return (columns.Select(c => c.Header).ToList(), rows.Select(r => (IReadOnlyList)columns.Select(c => c.Value(r)).ToList()).ToList()); + } + default: + throw new ArgumentOutOfRangeException(nameof(type), type, "No preview builder for this output type."); + } + } + + // The column set is the WINDOW's (spec v2.4 adds columns for KS4 and 16-19 only), so every + // file, preview and download of a run resolves it the same way. + private async Task WindowTypeOfRunAsync(Guid runId, CancellationToken ct) + { + var run = await repository.GetRunAsync(runId, ct) ?? throw new InvalidOperationException($"Egress run {runId} no longer exists."); + return await WindowTypeAsync(run.WindowId, runId, ct); + } + + private async Task WindowTypeAsync(Guid windowId, Guid runId, CancellationToken ct) + { + var window = await windows.GetByIdAsync(windowId, ct) + ?? throw new InvalidOperationException($"Checking window {windowId} for egress run {runId} no longer exists."); + return window.CheckingWindowType; + } + + private async Task<(byte[] Bytes, int Records)> BuildAsync(Guid runId, CheckingWindowType windowType, EgressOutputType type, CancellationToken ct) + { + switch (type) + { + case EgressOutputType.NewLearners: + { + var rows = await repository.GetNewLearnersAsync(runId, ct); + return (EgressCsvWriter.Write(EgressColumnSets.NewLearnersFor(windowType), rows), rows.Count); + } + case EgressOutputType.RemoveLearners: + { + var rows = await repository.GetRemoveLearnersAsync(runId, ct); + return (EgressCsvWriter.Write(EgressColumnSets.RemoveLearnersFor(windowType), rows), rows.Count); + } + default: + throw new ArgumentOutOfRangeException(nameof(type), type, "No file builder for this output type."); + } + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressBlobClient.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressBlobClient.cs new file mode 100644 index 000000000..5f5f60233 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressBlobClient.cs @@ -0,0 +1,45 @@ +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// Where LDS picks files up: container "cypmd", folder "extracts_input" (AB#292610). The storage +/// ACCOUNT is the environment's ConnectionStrings:EgressStorage — derived from the CYPMD +/// environment's configuration, never chosen by the user or edited at runtime (AB#294553 +/// "Transfer"). These two names are bindable only so a test can point at a scratch container. +/// +public sealed class EgressStorageOptions +{ + public const string SectionName = "EgressStorage"; + public string Container { get; set; } = "cypmd"; + public string Prefix { get; set; } = "extracts_input/"; + public string TargetDescription => $"{Container}/{Prefix.TrimEnd('/')}"; +} + +public sealed class EgressBlobAlreadyExistsException(string blobName, string? detail = null) + : Exception(detail is null ? Sentence(blobName) : $"{Sentence(blobName)} {detail}") +{ + public string BlobName { get; } = blobName; + private static string Sentence(string blobName) => $"A file named {blobName} already exists in the LDS container."; +} + +public interface IEgressBlobClient +{ + bool IsConfigured { get; } + string TargetDescription { get; } + /// A blob with that name exists — never overwritten. + Task UploadAsync(string fileName, byte[] content, string sha256, Guid runId, CancellationToken ct); + Task DeleteIfExistsAsync(string fileName, CancellationToken ct); + /// + /// Deletes the blob only if it exists and its egressRunId metadata equals ; + /// a blob that is absent or stamped with a different run is left untouched. Returns whether + /// something was actually deleted (M1/S3: cleans up a write whose success response was lost, + /// and the M1 Abandon-during-Transferring sweep, without ever touching another run's file). + /// + Task DeleteIfOwnedByRunAsync(string fileName, Guid runId, CancellationToken ct); + /// + /// The egressRunId stamped on an existing blob, or null when the blob is absent or carries no + /// such stamp (a file this service never wrote). Lets a transfer that collides with a + /// same-named file decide whether it is reclaiming its own leftover, an abandoned run's + /// orphan, or colliding with a real file that must be left alone. + /// + Task GetOwnerRunIdAsync(string fileName, CancellationToken ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressPreprocessor.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressPreprocessor.cs new file mode 100644 index 000000000..72de26787 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressPreprocessor.cs @@ -0,0 +1,11 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +public sealed record EgressProgress(int Step, int TotalSteps, string StepName, string State, int RecordsIn, int RecordsOut, int FailureCount, bool IsComplete, bool IsError, string Message, EgressRunStatus? FinalStatus); + +public interface IEgressPreprocessor +{ + /// Runs the pipeline for a Pulled (or PreprocessingFailed/Preprocessed — re-run) run. Yields one event per step state change; the last event has IsComplete = true. + IAsyncEnumerable RunAsync(Guid runId, CancellationToken ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunRepository.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunRepository.cs new file mode 100644 index 000000000..89c9f0357 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunRepository.cs @@ -0,0 +1,47 @@ +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +public sealed record EgressBlocker(Guid RunId, EgressRunStatus Status, string StartedByName, DateTime StartedAtUtc, DateTime? TransferredAtUtc, string? TransferredByName); +public sealed record EgressCandidateRequest(Guid ChangeRequestId, string ReferenceNumber, string? CrmId, long OrganisationUrn, string? OrganisationLaestab, DateTime SubmittedAtUtc, RequestStatus Status); +public sealed record EgressRunCreate(Guid WindowId, Guid StartedById, string StartedByName, string? StartedByEmail, IReadOnlyList Outputs); +public sealed record EgressRunOutputCreate(EgressOutputType OutputType, IReadOnlyList Records); +public sealed record EgressRunOutputDto(Guid Id, EgressOutputType OutputType, bool IsActive, IReadOnlyList Records, int SourceRecordCount, int? OutputRecordCount, string? FileName, string? Sha256); +public sealed record EgressRunDto(Guid Id, Guid WindowId, EgressRunStatus Status, Guid StartedById, string StartedByName, DateTime StartedAtUtc, DateTime? PreprocessedAtUtc, DateOnly? ExportDate, DateTime? TransferredAtUtc, string? TransferredByName, IReadOnlyList Failures, string? TransferFailureReason, IReadOnlyList Outputs); +public sealed record EgressRunListItem(Guid Id, Guid WindowId, string WindowTitle, EgressRunStatus Status, IReadOnlyList OutputTypes, string StartedByName, DateTime StartedAtUtc, DateTime? TransferredAtUtc); +public sealed record EgressTransferAudit(string UserId, string UserName, string TargetContainer, IReadOnlyDictionary Files); +public sealed class EgressRunConflictException(string message) : Exception(message); + +/// +/// Persistence for egress runs (AB#294553). Two rules the implementation must keep: (1) the +/// concurrency lock is the database's partial unique index, so CreateRunAsync and +/// TryReactivateAsync translate a 23505 into EgressRunConflictException / a blocker rather than +/// pre-checking only; (2) every status change that ends a run (failed, abandoned) clears IsActive +/// on its outputs in the same statement, and a successful transfer leaves them active for good. +/// +public interface IEgressRunRepository +{ + Task FindBlockerAsync(Guid windowId, EgressOutputType outputType, CancellationToken ct); + Task> GetCandidateRequestsAsync(Guid windowId, WhatToChange amendmentType, CancellationToken ct); + /// Another active run holds one of the (window, output type) pairs. + Task CreateRunAsync(EgressRunCreate create, CancellationToken ct); + Task GetRunAsync(Guid runId, CancellationToken ct); + Task> ListRunsAsync(CancellationToken ct); + Task TrySetStatusAsync(Guid runId, EgressRunStatus from, EgressRunStatus to, CancellationToken ct); + /// M4: guarded by ; returns rows affected (0 = lost the race — e.g. the run was abandoned in the meantime — nothing was written). + Task MarkPreprocessingFailedAsync(Guid runId, EgressRunStatus expectedStatus, IReadOnlyList failures, CancellationToken ct); + /// M4: guarded by ; returns rows affected (0 = lost the race — nothing was saved). + Task SavePreprocessedAsync(Guid runId, EgressRunStatus expectedStatus, IReadOnlyList newLearners, IReadOnlyList removeLearners, DateOnly exportDate, IReadOnlyDictionary fileNames, CancellationToken ct); + Task> GetNewLearnersAsync(Guid runId, CancellationToken ct); + Task> GetRemoveLearnersAsync(Guid runId, CancellationToken ct); + /// Re-activates a TransferFailed run's outputs for a retry; returns the output type + /// and blocker if another run now holds one of the run's pairs. + Task<(EgressOutputType OutputType, EgressBlocker Blocker)?> TryReactivateAsync(Guid runId, CancellationToken ct); + /// M4: guarded by ; returns rows affected (0 = lost the race — no Succeeded audit row is written). + Task MarkTransferredAsync(Guid runId, EgressRunStatus expectedStatus, EgressTransferAudit audit, DateTime transferredAtUtc, CancellationToken ct); + /// M4: guarded by ; returns rows affected (0 = the run had already moved on, e.g. to Abandoned — nothing was overwritten). + Task MarkTransferFailedAsync(Guid runId, EgressRunStatus expectedStatus, string reason, string userId, CancellationToken ct); + /// Excludes only Transferred; admits Preprocessing/Transferring so a stuck run can always be released. Returns rows affected (0 = already Transferred). + Task AbandonAsync(Guid runId, CancellationToken ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunService.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunService.cs new file mode 100644 index 000000000..5cd487618 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressRunService.cs @@ -0,0 +1,20 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +public sealed record EgressActor(Guid UserId, string DisplayName, string? Email); + +public abstract record EgressStartResult +{ + public sealed record Started(Guid RunId) : EgressStartResult; + public sealed record Refused(IReadOnlyList<(EgressOutputType OutputType, EgressBlocker Blocker)> Blockers) : EgressStartResult; + public sealed record WindowNotFound : EgressStartResult; + public sealed record PullFailed(string Reason) : EgressStartResult; +} + +public interface IEgressRunService +{ + Task StartAsync(Guid windowId, IReadOnlyList outputTypes, EgressActor actor, CancellationToken ct); + Task GetAsync(Guid runId, CancellationToken ct); + Task> ListAsync(CancellationToken ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressTicketSource.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressTicketSource.cs new file mode 100644 index 000000000..310ac2124 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressTicketSource.cs @@ -0,0 +1,14 @@ +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The one thing egress asks Zendesk: what decision each ticket carries. Keyed by ticket id; +/// a ticket Zendesk did not return, or returned without a readable decision, is absent — the +/// caller shows it as EgressDecisions.NotFound, never as approved. +/// +public interface IEgressTicketSource +{ + /// The field is not configured for this environment or Zendesk could not be read. + Task> GetDecisionStatusesAsync(IReadOnlyCollection ticketIds, CancellationToken ct); +} + +public sealed class EgressTicketSourceException(string message, Exception? inner = null) : Exception(message, inner); diff --git a/src/DfE.CheckPerformanceData.Application/Egress/IEgressTransferService.cs b/src/DfE.CheckPerformanceData.Application/Egress/IEgressTransferService.cs new file mode 100644 index 000000000..c27f63705 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/IEgressTransferService.cs @@ -0,0 +1,36 @@ +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.Egress; + +public abstract record EgressTransferResult +{ + public sealed record Transferred(IReadOnlyList<(EgressOutputType OutputType, string FileName, int Records)> Files, DateTime TransferredAtUtc) : EgressTransferResult; + /// Nit: carries the specific output type the retry lost, rather than the caller + /// having to guess it via string surgery on a description built for a different type. + public sealed record Refused(EgressOutputType OutputType, EgressBlocker Blocker) : EgressTransferResult; + public sealed record Failed(string Reason) : EgressTransferResult; + public sealed record NotTransferable(EgressRunStatus Status) : EgressTransferResult; + /// M3: every output's saved row count is zero — nothing to send, so refused before any upload. + public sealed record NothingToTransfer : EgressTransferResult; + /// Nit: a missing run is its own outcome, not a guess dressed as NotTransferable(Abandoned). + public sealed record NotFound : EgressTransferResult; +} + +/// M1: Abandon needs blob access (to sweep a Transferring run's own files), which is why +/// it lives beside Transfer rather than on . +public abstract record EgressAbandonResult +{ + public sealed record Abandoned(IReadOnlyList RemovedFiles) : EgressAbandonResult; + public sealed record AlreadyTransferred : EgressAbandonResult; + public sealed record NotFound : EgressAbandonResult; +} + +public interface IEgressTransferService +{ + Task TransferAsync(Guid runId, EgressActor actor, CancellationToken ct); + Task BuildFileAsync(Guid runId, EgressOutputType type, CancellationToken ct); + /// Nit: the preview table built from the saved rows and EgressColumnSets directly, + /// not by splitting the generated CSV text (which shifts columns for any quoted value). + Task<(IReadOnlyList Headers, IReadOnlyList> Rows)> GetPreviewAsync(Guid runId, EgressOutputType type, CancellationToken ct); + Task AbandonAsync(Guid runId, CancellationToken ct); +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/LaestabSplitter.cs b/src/DfE.CheckPerformanceData.Application/Egress/LaestabSplitter.cs new file mode 100644 index 000000000..1f25a6593 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/LaestabSplitter.cs @@ -0,0 +1,24 @@ +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// LDS wants the 7-digit DfE establishment number as two fields: a 3-digit LA number and a +/// 4-digit establishment number (AB#292610). Non-digits (the "/" some sources carry) are +/// ignored; anything that is not exactly seven digits is refused rather than padded. +/// +public static class LaestabSplitter +{ + public static bool TrySplit(string? raw, out string localAuthority, out string establishment) + { + var digits = new string((raw ?? string.Empty).Where(char.IsDigit).ToArray()); + if (digits.Length != 7) + { + localAuthority = string.Empty; + establishment = string.Empty; + return false; + } + + localAuthority = digits[..3]; + establishment = digits[3..]; + return true; + } +} diff --git a/src/DfE.CheckPerformanceData.Application/Egress/LdsRows.cs b/src/DfE.CheckPerformanceData.Application/Egress/LdsRows.cs new file mode 100644 index 000000000..0b19f1ded --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/LdsRows.cs @@ -0,0 +1,64 @@ +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// One row of the Remove learners file, already in LDS shape (spec column order is decided by +/// EgressColumnSets, not by property order). The trailing identifiers are CYPMD's own bookkeeping +/// and never reach the file. The init-only properties are the key-stage-specific columns of +/// LDS_CYPMD_Data specification v2.4: Year_Group is in KS4 files only, Removal_Year_0..2 in 16-19 +/// files only; a row for another key stage leaves them blank and its column set never emits them. +/// +public sealed record RemoveLearnerRow( + string CorrectionId, + string CorrectionType, + string CorrectionReason, + string KeyStage, + string EstablishmentNumber, + string Surname, + string Forename, + string Sex, + string DateOfBirth, + string CycleYear, + string CycleMonth, + string LocalAuthority, + string LearnerId, + Guid ChangeRequestId, + long? TicketId, + string ReferenceNumber) +{ + /// KS4 only, and only for year-group-change removals: the year group the pupil moves to. + public string YearGroup { get; init; } = string.Empty; + /// 16-19 only: TRUE/FALSE for the current academic year; blank when the journey did not ask. + public string RemovalYear0 { get; init; } = string.Empty; + /// 16-19 only: TRUE/FALSE for the previous academic year. + public string RemovalYear1 { get; init; } = string.Empty; + /// 16-19 only: TRUE/FALSE for the academic year before that. + public string RemovalYear2 { get; init; } = string.Empty; +} + +/// +/// One row of the New learners file (LDS_CYPMD_Data specification v2.4 "New Learner" sheet). +/// Property names are CYPMD's; the CSV headings (Post_Code, URN, …) live in EgressColumnSets. +/// Middle_Name is struck through in v2.4 and SEN status is not in the spec, so neither is here. +/// +public sealed record NewLearnerRow( + string CorrectionId, + string CorrectionType, + string KeyStage, + string LocalAuthority, + string EstablishmentNumber, + string Surname, + string Forename, + string Sex, + string DateOfBirth, + string AdmissionDate, + string Postcode, + string CycleYear, + string CycleMonth, + string SchoolUrn, + string Uln, + string Upn, + string LearnerId, + string YearGroup, + Guid ChangeRequestId, + long? TicketId, + string ReferenceNumber); diff --git a/src/DfE.CheckPerformanceData.Application/Egress/LdsSpecValidator.cs b/src/DfE.CheckPerformanceData.Application/Egress/LdsSpecValidator.cs new file mode 100644 index 000000000..740c05b67 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Application/Egress/LdsSpecValidator.cs @@ -0,0 +1,154 @@ +using System.Text.RegularExpressions; + +namespace DfE.CheckPerformanceData.Application.Egress; + +/// +/// The "Validate against LDS spec" step. Required fields, permitted values and shapes, one failure +/// per offending field. Values and shapes are LDS_CYPMD_Data specification v2.4; nothing here +/// loosens as a work-around — a record that fails here fails the batch. +/// +public static partial class LdsSpecValidator +{ + public const string StepName = "Validate against LDS spec"; + + private static readonly HashSet Stages = ["KS2", "KS4", "16-19"]; + // v2.4: both sheets — "F" (female), "M" (male), "U" (unknown). + private static readonly HashSet Sexes = ["F", "M", "U"]; + private static readonly HashSet TrueFalse = ["TRUE", "FALSE"]; + + public static IReadOnlyList Validate(RemoveLearnerRow row) + { + var f = new Failures(row.TicketId, row.ReferenceNumber); + f.Digits("Correction_ID", row.CorrectionId); + f.Equals("Correction_Type", row.CorrectionType, "31"); + f.Digits("Correction_Reason", row.CorrectionReason); + f.OneOf("Key_Stage", row.KeyStage, Stages); + f.DigitsOfLength("Establishment_Number", row.EstablishmentNumber, 4); + f.FreeText("Surname", row.Surname); + f.FreeText("Forename", row.Forename); + f.OneOf("Sex", row.Sex, Sexes); + f.IsoDate("Date_of_Birth", row.DateOfBirth); + f.DigitsOfLength("Cycle_Year", row.CycleYear, 4); + f.Month("Cycle_Month", row.CycleMonth); + f.DigitsOfLength("Local_Authority", row.LocalAuthority, 3); + f.Digits("Learner_ID", row.LearnerId); + f.OptionalYearGroup("Year_Group", row.YearGroup); + f.OptionalOneOf("Removal_Year_0", row.RemovalYear0, TrueFalse); + f.OptionalOneOf("Removal_Year_1", row.RemovalYear1, TrueFalse); + f.OptionalOneOf("Removal_Year_2", row.RemovalYear2, TrueFalse); + return f.List; + } + + public static IReadOnlyList Validate(NewLearnerRow row) + { + var f = new Failures(row.TicketId, row.ReferenceNumber); + f.Digits("Correction_ID", row.CorrectionId); + f.Equals("Correction_Type", row.CorrectionType, "10"); + f.OneOf("Key_Stage", row.KeyStage, Stages); + f.DigitsOfLength("Establishment_Number", row.EstablishmentNumber, 4); + f.FreeText("Surname", row.Surname); + f.FreeText("Forename", row.Forename); + f.OneOf("Sex", row.Sex, Sexes); + f.IsoDate("Date_of_Birth", row.DateOfBirth); + f.IsoDate("Admission_Date", row.AdmissionDate); + f.OptionalMaxLength("Post_Code", row.Postcode, 8); + f.DigitsOfLength("Cycle_Year", row.CycleYear, 4); + f.Month("Cycle_Month", row.CycleMonth); + f.DigitsOfLength("Local_Authority", row.LocalAuthority, 3); + f.Digits("URN", row.SchoolUrn); + f.OptionalDigitsMaxLength("ULN", row.Uln, 11); + f.OptionalMaxLength("UPN", row.Upn, 13); + f.OptionalDigits("Learner_ID", row.LearnerId); + f.OptionalYearGroup("Year_Group", row.YearGroup); + return f.List; + } + + [GeneratedRegex(@"^\d+$")] + private static partial Regex DigitsOnly(); + + [GeneratedRegex(@"^\d{4}-\d{2}-\d{2}$")] + private static partial Regex IsoDateShape(); + + private sealed class Failures(long? ticketId, string reference) + { + public List List { get; } = []; + + private void Add(string field, string reason) => + List.Add(new EgressRecordFailure(StepName, ticketId, reference, field, reason)); + + public void Required(string field, string value) + { + if (string.IsNullOrWhiteSpace(value)) Add(field, "is required"); + } + + // The only cells a school types freely (Surname, Forename). The file leaves for an + // external organisation and may be opened in a spreadsheet, where a leading = + - or @ + // is evaluated as a formula (OWASP CSV injection). Refused here rather than rewritten + // with an apostrophe in the writer: LDS expects raw values, no real name begins with one + // of these, and a named failure tells the ops user which record to look at. + public void FreeText(string field, string value) + { + if (string.IsNullOrWhiteSpace(value)) Add(field, "is required"); + else if (value[0] is '=' or '+' or '-' or '@') Add(field, "must not begin with =, +, - or @"); + } + + public void Equals(string field, string value, string expected) + { + if (value != expected) Add(field, $"must be {expected}"); + } + + public void Digits(string field, string value) + { + if (string.IsNullOrWhiteSpace(value) || !DigitsOnly().IsMatch(value)) Add(field, "must be a number"); + } + + public void OptionalDigits(string field, string value) + { + if (value.Length > 0 && !DigitsOnly().IsMatch(value)) Add(field, "must be a number when supplied"); + } + + public void DigitsOfLength(string field, string value, int length) + { + if (value.Length != length || !DigitsOnly().IsMatch(value)) Add(field, $"must be {length} digits"); + } + + public void OptionalMaxLength(string field, string value, int max) + { + if (value.Length > max) Add(field, $"must be {max} characters or fewer"); + } + + public void OptionalDigitsMaxLength(string field, string value, int max) + { + if (value.Length == 0) return; + if (!DigitsOnly().IsMatch(value)) Add(field, "must be a number when supplied"); + else if (value.Length > max) Add(field, $"must be {max} digits or fewer"); + } + + public void OptionalOneOf(string field, string value, IReadOnlySet permitted) + { + if (value.Length > 0 && !permitted.Contains(value)) Add(field, $"must be blank or one of {string.Join(", ", permitted.Order())}"); + } + + // Spec: Year_Group "1-13", NULL allowed. + public void OptionalYearGroup(string field, string value) + { + if (value.Length > 0 && (!int.TryParse(value, out var y) || y is < 1 or > 13)) Add(field, "must be blank or a year group 1 to 13"); + } + + public void OneOf(string field, string value, IReadOnlySet permitted) + { + if (!permitted.Contains(value)) Add(field, $"must be one of {string.Join(", ", permitted.Order())}"); + } + + public void IsoDate(string field, string value) + { + if (!IsoDateShape().IsMatch(value) || !DateOnly.TryParseExact(value, "yyyy-MM-dd", out _)) + Add(field, "must be a real date in yyyy-MM-dd format"); + } + + public void Month(string field, string value) + { + if (!int.TryParse(value, out var m) || m is < 1 or > 12) Add(field, "must be a month number 1 to 12"); + } + } +} diff --git a/src/DfE.CheckPerformanceData.Application/RequestSubmission/ChangeRequestData.cs b/src/DfE.CheckPerformanceData.Application/RequestSubmission/ChangeRequestData.cs index 19646bb9d..72a384f33 100644 --- a/src/DfE.CheckPerformanceData.Application/RequestSubmission/ChangeRequestData.cs +++ b/src/DfE.CheckPerformanceData.Application/RequestSubmission/ChangeRequestData.cs @@ -27,4 +27,7 @@ public sealed class ChangeRequestData public required RequestType RequestType { get; init; } public required string RequestTypeDescription { get; init; } public WhatToChange? AmendmentType { get; init; } + + /// The school's LAESTAB from the DfE Sign-In claim; null when the claim is empty (AB#294553). + public string? OrganisationLaestab { get; init; } } diff --git a/src/DfE.CheckPerformanceData.Application/RequestSubmission/RequestService.cs b/src/DfE.CheckPerformanceData.Application/RequestSubmission/RequestService.cs index 4ddac0241..c2ed34f29 100644 --- a/src/DfE.CheckPerformanceData.Application/RequestSubmission/RequestService.cs +++ b/src/DfE.CheckPerformanceData.Application/RequestSubmission/RequestService.cs @@ -23,6 +23,10 @@ public sealed class RequestService( { private long OrganisationUrnLong => long.Parse(currentUserService.OrganisationUrn); + // Empty claim → null column, never an empty string, so "unknown" reads the same on every row. + private string? OrganisationLaestabOrNull => + string.IsNullOrWhiteSpace(currentUserService.OrganisationLaestab) ? null : currentUserService.OrganisationLaestab.Trim(); + // The exercise a journey belongs to is derived from its change type, never stored on the // session - a stored copy can disagree with the journey's own SelectedWhatToChange. The row is // where it becomes durable, so this is the one place a journey's exercise id is resolved. @@ -162,7 +166,8 @@ public async Task SubmitResultsEnquiryAsync( Status = RequestStatus.SubmittedUnCommitted, RequestType = RequestType.ResultsEnquiry, RequestTypeDescription = requestTypeDescription, - AmendmentType = journey.SelectedWhatToChange.Value + AmendmentType = journey.SelectedWhatToChange.Value, + OrganisationLaestab = OrganisationLaestabOrNull }); // The journey JSON is the enquiry's full record — it carries the selected result and every @@ -219,7 +224,8 @@ await requestRepository.UpsertAsync(new ChangeRequestData SubmittedByEmail = currentUserService.Email, Status = RequestStatus.SubmittedUnCommitted, RequestType = RequestType.ConfirmCorrect, - RequestTypeDescription = "Confirm Pupil Data Declaration" + RequestTypeDescription = "Confirm Pupil Data Declaration", + OrganisationLaestab = OrganisationLaestabOrNull }); await requestNotificationService.NotifyDataCheckConfirmedAsync( @@ -328,7 +334,8 @@ private ChangeRequestData BuildChangeRequestData(Guid windowId, RequestState jou Status = status, RequestType = RequestType.Amendment, RequestTypeDescription = BuildRequestTypeDescription(journey, config), - AmendmentType = journey.SelectedWhatToChange + AmendmentType = journey.SelectedWhatToChange, + OrganisationLaestab = OrganisationLaestabOrNull }; private RequestDocument BuildRequestDocument(JourneySubmissionContext context, QuestionFlowConfig config, Guid changeRequestId) diff --git a/src/DfE.CheckPerformanceData.Domain/Enums/EgressOutputType.cs b/src/DfE.CheckPerformanceData.Domain/Enums/EgressOutputType.cs new file mode 100644 index 000000000..32a66d3c3 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Domain/Enums/EgressOutputType.cs @@ -0,0 +1,12 @@ +namespace DfE.CheckPerformanceData.Domain.Enums; + +/// +/// The LDS output files the egress can produce (AB#294553). Merged learners is deliberately absent — +/// it needs its own column set and the KS4 June 20→21 correction-code rule (AB#292610) and is a +/// follow-up. Stored as a string, so the order here is display order, not identity. +/// +public enum EgressOutputType +{ + NewLearners, + RemoveLearners +} diff --git a/src/DfE.CheckPerformanceData.Domain/Enums/EgressRunStatus.cs b/src/DfE.CheckPerformanceData.Domain/Enums/EgressRunStatus.cs new file mode 100644 index 000000000..6951a79a0 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Domain/Enums/EgressRunStatus.cs @@ -0,0 +1,19 @@ +namespace DfE.CheckPerformanceData.Domain.Enums; + +/// +/// Lifecycle of one egress run. Pulled → Preprocessing → (PreprocessingFailed | Preprocessed) → +/// Transferring → (TransferFailed | Transferred); Abandoned from any non-terminal state. Only +/// Pulled, Preprocessing, Preprocessed, Transferring and Transferred keep the run's outputs active +/// (see EgressRunOutput.IsActive) — a failed or abandoned run never blocks a new one. +/// +public enum EgressRunStatus +{ + Pulled, + Preprocessing, + PreprocessingFailed, + Preprocessed, + Transferring, + TransferFailed, + Transferred, + Abandoned +} diff --git a/src/DfE.CheckPerformanceData.Infrastructure/Egress/DevOutboxEgressTicketSource.cs b/src/DfE.CheckPerformanceData.Infrastructure/Egress/DevOutboxEgressTicketSource.cs new file mode 100644 index 000000000..b11b9b201 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Infrastructure/Egress/DevOutboxEgressTicketSource.cs @@ -0,0 +1,59 @@ +using System.Text.Json; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Persistence.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +namespace DfE.CheckPerformanceData.Infrastructure.Egress; + +/// +/// Development stand-in: reads decisions from the DevZendeskTickets outbox the worker's fake +/// Zendesk writes. The captured ticket's "Decision status" custom field wins (so a dev seeder can +/// stage a human "approved"); otherwise the subject prefix the ticket builder always writes +/// decides. Selected when Zendesk:UseFake is true (the default), exactly like the worker's fake. +/// +public sealed class DevOutboxEgressTicketSource(IPortalDbContext db, IOptions fieldSettings) : IEgressTicketSource +{ + // The dev/preprod Zendesk instance's field id, used when the environment configures none. + public const long WellKnownDecisionFieldId = 19056253670034; + + public async Task> GetDecisionStatusesAsync(IReadOnlyCollection ticketIds, CancellationToken ct) + { + var result = new Dictionary(); + if (ticketIds.Count == 0) return result; + + var fieldId = fieldSettings.Value.DecisionStatusId is > 0 and var configured ? configured : WellKnownDecisionFieldId; + var ids = ticketIds.Distinct().ToList(); + var rows = await db.DevZendeskTickets.AsNoTracking().Where(t => ids.Contains(t.TicketId)).ToListAsync(ct); + + foreach (var row in rows) + { + var decision = FromCustomField(row.RawJson, fieldId) ?? FromSubject(row.Subject); + if (decision is not null) result[row.TicketId] = decision; + } + return result; + } + + private static string? FromCustomField(string rawJson, long fieldId) + { + try + { + var request = JsonSerializer.Deserialize(rawJson); + var value = request?.Ticket.CustomFields.FirstOrDefault(f => f.Id == fieldId)?.Value?.ToString(); + return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); + } + catch (JsonException) + { + return null; + } + } + + private static string? FromSubject(string subject) + { + if (subject.StartsWith("CPMD Auto-Approved:", StringComparison.OrdinalIgnoreCase)) return ZendeskTicketFieldOptions.DecisionStatus.AutoApproved; + if (subject.StartsWith("CPMD Auto-Rejected:", StringComparison.OrdinalIgnoreCase)) return ZendeskTicketFieldOptions.DecisionStatus.AutoRejected; + if (subject.StartsWith("CPMD Requires Scrutiny:", StringComparison.OrdinalIgnoreCase)) return ZendeskTicketFieldOptions.DecisionStatus.Scrutiny; + return null; + } +} diff --git a/src/DfE.CheckPerformanceData.Infrastructure/Egress/EgressBlobClient.cs b/src/DfE.CheckPerformanceData.Infrastructure/Egress/EgressBlobClient.cs new file mode 100644 index 000000000..75d12cdb2 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Infrastructure/Egress/EgressBlobClient.cs @@ -0,0 +1,74 @@ +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using DfE.CheckPerformanceData.Application.Egress; +using Microsoft.Extensions.Options; + +namespace DfE.CheckPerformanceData.Infrastructure.Egress; + +/// Uploads egress files to the LDS account's cypmd/extracts_input, never overwriting. +public sealed class EgressBlobClient(IReadOnlyDictionary clients, IOptions options) : IEgressBlobClient +{ + public const string ClientKey = "egress"; + + public bool IsConfigured => clients.ContainsKey(ClientKey); + public string TargetDescription => options.Value.TargetDescription; + + public async Task UploadAsync(string fileName, byte[] content, string sha256, Guid runId, CancellationToken ct) + { + var container = clients[ClientKey].GetBlobContainerClient(options.Value.Container); + await container.CreateIfNotExistsAsync(cancellationToken: ct); + var blob = container.GetBlobClient(options.Value.Prefix + fileName); + try + { + await blob.UploadAsync(new BinaryData(content), new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "text/csv" }, + Metadata = new Dictionary { ["sha256"] = sha256, ["egressRunId"] = runId.ToString() }, + // IfNoneMatch "*" = create only. The same window and type can never be sent twice, + // and a leftover from a failed compensation must be looked at, not silently replaced. + Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All } + }, ct); + } + // S7: only the conflict this code actually handles — a blob already exists — is worth its + // own exception; a lease held on the blob or the container mid-delete also returns 409 but + // means something else entirely, and must propagate rather than being misreported as + // "remove it by hand". + catch (RequestFailedException ex) when (ex.Status == 409 && ex.ErrorCode == BlobErrorCode.BlobAlreadyExists.ToString()) + { + throw new EgressBlobAlreadyExistsException(options.Value.Prefix + fileName); + } + } + + public async Task DeleteIfExistsAsync(string fileName, CancellationToken ct) + { + var container = clients[ClientKey].GetBlobContainerClient(options.Value.Container); + await container.GetBlobClient(options.Value.Prefix + fileName).DeleteIfExistsAsync(cancellationToken: ct); + } + + public async Task DeleteIfOwnedByRunAsync(string fileName, Guid runId, CancellationToken ct) + { + if (await OwnerAsync(fileName, ct) != runId) return false; + await Blob(fileName).DeleteIfExistsAsync(cancellationToken: ct); + return true; + } + + public Task GetOwnerRunIdAsync(string fileName, CancellationToken ct) => OwnerAsync(fileName, ct); + + private BlobClient Blob(string fileName) => + clients[ClientKey].GetBlobContainerClient(options.Value.Container).GetBlobClient(options.Value.Prefix + fileName); + + private async Task OwnerAsync(string fileName, CancellationToken ct) + { + BlobProperties properties; + try + { + properties = await Blob(fileName).GetPropertiesAsync(cancellationToken: ct); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + return properties.Metadata.TryGetValue("egressRunId", out var owner) && Guid.TryParse(owner, out var id) ? id : null; + } +} diff --git a/src/DfE.CheckPerformanceData.Infrastructure/Egress/ZendeskEgressTicketSource.cs b/src/DfE.CheckPerformanceData.Infrastructure/Egress/ZendeskEgressTicketSource.cs new file mode 100644 index 000000000..45c22dadc --- /dev/null +++ b/src/DfE.CheckPerformanceData.Infrastructure/Egress/ZendeskEgressTicketSource.cs @@ -0,0 +1,56 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Infrastructure.Resilience; +using DfE.CheckPerformanceData.Infrastructure.ZendeskClient; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Polly; + +namespace DfE.CheckPerformanceData.Infrastructure.Egress; + +/// +/// Reads the "Decision status" custom field from real Zendesk tickets, by id, 100 at a time. +/// The field id comes from configuration (ZendeskTicketFields:DecisionStatusId); production +/// currently configures 0, and this refuses rather than treating every ticket as undecided. +/// +public sealed class ZendeskEgressTicketSource( + IZendeskApi api, + IOptions fieldSettings, + IOptions pollySettings, + ILogger logger) : IEgressTicketSource +{ + public const int BatchSize = 100; + private readonly ResiliencePipeline _retry = ResiliencePipelineFactory.CreateRetryPipeline(pollySettings.Value, logger); + + public async Task> GetDecisionStatusesAsync(IReadOnlyCollection ticketIds, CancellationToken ct) + { + var fieldId = fieldSettings.Value.DecisionStatusId; + if (fieldId is null or <= 0) + throw new EgressTicketSourceException( + "The Zendesk \"Decision status\" field id is not configured for this environment (ZendeskTicketFields:DecisionStatusId), so decisions cannot be read."); + + var result = new Dictionary(); + if (ticketIds.Count == 0) return result; + + foreach (var batch in ticketIds.Distinct().Chunk(BatchSize)) + { + var ids = string.Join(",", batch); + var response = await _retry.ExecuteAsync(async token => + { + try { return await api.ShowManyTickets(ids); } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new EgressTicketSourceException($"Zendesk could not return tickets {ids}.", ex); + } + }, ct); + + foreach (var ticket in response.Tickets) + { + var value = ticket.AllCustomFields.FirstOrDefault(f => f.Id == fieldId)?.Value?.ToString(); + if (!string.IsNullOrWhiteSpace(value)) + result[ticket.Id] = value.Trim().ToLowerInvariant(); + } + } + return result; + } +} diff --git a/src/DfE.CheckPerformanceData.Infrastructure/ZendeskClient/IZendeskApi.cs b/src/DfE.CheckPerformanceData.Infrastructure/ZendeskClient/IZendeskApi.cs index 2f09a2ffe..52ae5b8b1 100644 --- a/src/DfE.CheckPerformanceData.Infrastructure/ZendeskClient/IZendeskApi.cs +++ b/src/DfE.CheckPerformanceData.Infrastructure/ZendeskClient/IZendeskApi.cs @@ -20,6 +20,11 @@ public interface IZendeskApi [Get("/api/v2/tickets/{ticket_id}")] Task GetTicket(long ticket_id); + // Bulk read for the LDS egress (AB#294553): up to 100 comma-separated ids per call. Returns + // the same {"tickets":[...]} envelope as a view listing, so the model is reused. + [Get("/api/v2/tickets/show_many.json")] + Task ShowManyTickets([Query] string ids); + [Get("/api/v2/user_fields.json")] Task GetUserFields(); diff --git a/src/DfE.CheckPerformanceData.Persistence/Configurations/ChangeRequestConfiguration.cs b/src/DfE.CheckPerformanceData.Persistence/Configurations/ChangeRequestConfiguration.cs index 628fc4577..a893f4faa 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Configurations/ChangeRequestConfiguration.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Configurations/ChangeRequestConfiguration.cs @@ -25,6 +25,9 @@ public void Configure(EntityTypeBuilder builder) builder.Property(x => x.PupilSurname) .HasMaxLength(100); + builder.Property(x => x.OrganisationLaestab) + .HasMaxLength(20); + builder.Property(x => x.Submitted) .IsRequired() .HasColumnType("timestamp without time zone"); diff --git a/src/DfE.CheckPerformanceData.Persistence/Contexts/IPortalDbContext.cs b/src/DfE.CheckPerformanceData.Persistence/Contexts/IPortalDbContext.cs index 1f62aecec..dff2a8c88 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Contexts/IPortalDbContext.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Contexts/IPortalDbContext.cs @@ -2,6 +2,7 @@ using DfE.CheckPerformanceData.Persistence.Entities; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; namespace DfE.CheckPerformanceData.Persistence.Contexts; @@ -9,6 +10,7 @@ namespace DfE.CheckPerformanceData.Persistence.Contexts; public interface IPortalDbContext { DatabaseFacade Database { get; } + ChangeTracker ChangeTracker { get; } DbSet ChangeRequests { get; } DbSet AuditEntries { get; } @@ -24,6 +26,10 @@ public interface IPortalDbContext DbSet QueueMessages { get; } DbSet DeadLetters { get; } DbSet DevZendeskTickets { get; } + DbSet EgressRuns { get; } + DbSet EgressRunOutputs { get; } + DbSet EgressNewLearners { get; } + DbSet EgressRemoveLearners { get; } DbSet QueueMetricEvents { get; } DbSet ShareTokens { get; } DbSet PageNodes { get; } @@ -37,4 +43,5 @@ public interface IPortalDbContext Task SaveChangesAsync(CancellationToken cancellationToken = default); Task ExecuteInTransactionAsync(Func work, CancellationToken cancellationToken = default); + Task ExecuteInTransactionAsync(Func> work, CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/DfE.CheckPerformanceData.Persistence/Contexts/PortalDbContext.cs b/src/DfE.CheckPerformanceData.Persistence/Contexts/PortalDbContext.cs index f0fb855b2..b17d6b18f 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Contexts/PortalDbContext.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Contexts/PortalDbContext.cs @@ -27,6 +27,10 @@ public sealed class PortalDbContext( public DbSet QueueMessages => Set(); public DbSet DeadLetters => Set(); public DbSet DevZendeskTickets => Set(); + public DbSet EgressRuns => Set(); + public DbSet EgressRunOutputs => Set(); + public DbSet EgressNewLearners => Set(); + public DbSet EgressRemoveLearners => Set(); public DbSet QueueMetricEvents => Set(); public DbSet ShareTokens => Set(); public DbSet PageNodes => Set(); @@ -54,6 +58,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new QueueMessageConfiguration()); modelBuilder.ApplyConfiguration(new DeadLetterConfiguration()); modelBuilder.ApplyConfiguration(new DevZendeskTicketConfiguration()); + modelBuilder.ApplyConfiguration(new EgressRunConfiguration()); + modelBuilder.ApplyConfiguration(new EgressRunOutputConfiguration()); + modelBuilder.ApplyConfiguration(new EgressNewLearnerConfiguration()); + modelBuilder.ApplyConfiguration(new EgressRemoveLearnerConfiguration()); modelBuilder.ApplyConfiguration(new QueueMetricEventConfiguration()); modelBuilder.ApplyConfiguration(new ShareTokenConfiguration()); modelBuilder.ApplyConfiguration(new PageNodeConfiguration()); @@ -101,6 +109,25 @@ await strategy.ExecuteAsync(async () => }); } + // M4: the same re-entrant/execution-strategy shape as the void overload, for callers (like the + // egress terminal writes) that need to know how many rows a status-guarded update affected. + public async Task ExecuteInTransactionAsync(Func> work, CancellationToken cancellationToken = default) + { + if (Database.CurrentTransaction is not null) + { + return await work(); + } + + var strategy = Database.CreateExecutionStrategy(); + return await strategy.ExecuteAsync(async () => + { + await using var transaction = await Database.BeginTransactionAsync(cancellationToken); + var result = await work(); + await transaction.CommitAsync(cancellationToken); + return result; + }); + } + private List<(AuditEntry Audit, EntityEntry Entry, bool HasTempKey)> CollectAuditEntries() { ChangeTracker.DetectChanges(); @@ -124,6 +151,17 @@ await strategy.ExecuteAsync(async () => // login and keep a second copy of organisation data in audit_entries, which has // no retention purge. if (entry.Entity is OrganisationLogin) continue; + + // Processed egress learner rows are derived data written in bulk from one deliberate + // admin action; the run-level AuditEntry written at transfer (EgressRun / Transfer) + // is the audit record AB#294553 asks for. Auditing each row would add hundreds of + // audit_entries per run that say nothing the run row does not. + if (entry.Entity is EgressNewLearner) continue; + if (entry.Entity is EgressRemoveLearner) continue; + // S1 (review): EgressRunOutput.RawRecordsJson is the raw pulled payload — names, DOB, + // sex, UPN, every journey answer for every candidate — for the same reason as the two + // learner rows above, and because AuditEntries has no retention purge of its own. + if (entry.Entity is EgressRunOutput) continue; if (entry.State is EntityState.Detached or EntityState.Unchanged) continue; var audit = new AuditEntry diff --git a/src/DfE.CheckPerformanceData.Persistence/DependencyManager.cs b/src/DfE.CheckPerformanceData.Persistence/DependencyManager.cs index a0ca0cf36..d4324b1a9 100644 --- a/src/DfE.CheckPerformanceData.Persistence/DependencyManager.cs +++ b/src/DfE.CheckPerformanceData.Persistence/DependencyManager.cs @@ -57,6 +57,7 @@ public static IServiceCollection AddPersistenceDependencies( services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/DfE.CheckPerformanceData.Persistence/Entities/ChangeRequest.cs b/src/DfE.CheckPerformanceData.Persistence/Entities/ChangeRequest.cs index 7c17e3b59..1b4dab95e 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Entities/ChangeRequest.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Entities/ChangeRequest.cs @@ -43,6 +43,15 @@ public class ChangeRequest public WhatToChange? AmendmentType { get; init; } public string? CrmId { get; init; } + /// + /// The submitting school's 7-digit DfE establishment number (LAESTAB), from the DfE Sign-In + /// organisation_laestab claim at submit time (AB#294553). The LDS egress splits it into LA and + /// establishment for a new learner, whose synthetic pupil record carries no LAESTAB of its own. + /// Null on rows written before the column existed; egress then falls back to the pupil's and + /// fails the record, with a reason, when neither is known. + /// + public string? OrganisationLaestab { get; init; } + // Written by the rules engine consumer once it has decided on the request, and read // back by the Zendesk consumer; all stay null until the rules engine has run. public DecisionStatus? Outcome { get; set; } diff --git a/src/DfE.CheckPerformanceData.Persistence/Entities/EgressLearners.cs b/src/DfE.CheckPerformanceData.Persistence/Entities/EgressLearners.cs new file mode 100644 index 000000000..067555de3 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Entities/EgressLearners.cs @@ -0,0 +1,104 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace DfE.CheckPerformanceData.Persistence.Entities; + +/// +/// A processed new-learner record — the spec-conformant version, never the raw Zendesk shape +/// (AB#292610 "Data storage in CYPMD"). Written only when the whole batch preprocessed cleanly; +/// the transfer reads the file rows from here, not from the pulled payload. +/// +public sealed class EgressNewLearner +{ + public Guid Id { get; set; } + public Guid RunId { get; set; } + public Guid ChangeRequestId { get; set; } + public long? TicketId { get; set; } + public string ReferenceNumber { get; set; } = string.Empty; + public string CorrectionId { get; set; } = string.Empty; + public string CorrectionType { get; set; } = string.Empty; + public string KeyStage { get; set; } = string.Empty; + public string LocalAuthority { get; set; } = string.Empty; + public string EstablishmentNumber { get; set; } = string.Empty; + public string Surname { get; set; } = string.Empty; + public string Forename { get; set; } = string.Empty; + public string Sex { get; set; } = string.Empty; + public string DateOfBirth { get; set; } = string.Empty; + public string AdmissionDate { get; set; } = string.Empty; + public string Postcode { get; set; } = string.Empty; + public string CycleYear { get; set; } = string.Empty; + public string CycleMonth { get; set; } = string.Empty; + public string SchoolUrn { get; set; } = string.Empty; + public string Uln { get; set; } = string.Empty; + public string Upn { get; set; } = string.Empty; + public string LearnerId { get; set; } = string.Empty; + public string YearGroup { get; set; } = string.Empty; +} + +/// A processed remove-learner record; same rules as . +public sealed class EgressRemoveLearner +{ + public Guid Id { get; set; } + public Guid RunId { get; set; } + public Guid ChangeRequestId { get; set; } + public long? TicketId { get; set; } + public string ReferenceNumber { get; set; } = string.Empty; + public string CorrectionId { get; set; } = string.Empty; + public string CorrectionType { get; set; } = string.Empty; + public string CorrectionReason { get; set; } = string.Empty; + public string KeyStage { get; set; } = string.Empty; + public string EstablishmentNumber { get; set; } = string.Empty; + public string Surname { get; set; } = string.Empty; + public string Forename { get; set; } = string.Empty; + public string Sex { get; set; } = string.Empty; + public string DateOfBirth { get; set; } = string.Empty; + public string CycleYear { get; set; } = string.Empty; + public string CycleMonth { get; set; } = string.Empty; + public string LocalAuthority { get; set; } = string.Empty; + public string LearnerId { get; set; } = string.Empty; + public string YearGroup { get; set; } = string.Empty; + public string RemovalYear0 { get; set; } = string.Empty; + public string RemovalYear1 { get; set; } = string.Empty; + public string RemovalYear2 { get; set; } = string.Empty; +} + +public sealed class EgressNewLearnerConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("new_learners"); + builder.HasKey(x => x.Id); + builder.Property(x => x.ReferenceNumber).IsRequired().HasMaxLength(50); + foreach (var name in new[] { nameof(EgressNewLearner.CorrectionId), nameof(EgressNewLearner.CorrectionType), nameof(EgressNewLearner.KeyStage), + nameof(EgressNewLearner.LocalAuthority), nameof(EgressNewLearner.EstablishmentNumber), nameof(EgressNewLearner.Surname), + nameof(EgressNewLearner.Forename), nameof(EgressNewLearner.Sex), nameof(EgressNewLearner.DateOfBirth), + nameof(EgressNewLearner.AdmissionDate), nameof(EgressNewLearner.Postcode), nameof(EgressNewLearner.CycleYear), nameof(EgressNewLearner.CycleMonth), + nameof(EgressNewLearner.SchoolUrn), nameof(EgressNewLearner.Uln), nameof(EgressNewLearner.Upn), nameof(EgressNewLearner.LearnerId), + nameof(EgressNewLearner.YearGroup) }) + { + builder.Property(name).IsRequired().HasMaxLength(200); + } + builder.HasOne().WithMany().HasForeignKey(x => x.RunId).OnDelete(DeleteBehavior.Cascade); + builder.HasIndex(x => x.RunId); + } +} + +public sealed class EgressRemoveLearnerConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("remove_learners"); + builder.HasKey(x => x.Id); + builder.Property(x => x.ReferenceNumber).IsRequired().HasMaxLength(50); + foreach (var name in new[] { nameof(EgressRemoveLearner.CorrectionId), nameof(EgressRemoveLearner.CorrectionType), nameof(EgressRemoveLearner.CorrectionReason), + nameof(EgressRemoveLearner.KeyStage), nameof(EgressRemoveLearner.EstablishmentNumber), nameof(EgressRemoveLearner.Surname), + nameof(EgressRemoveLearner.Forename), nameof(EgressRemoveLearner.Sex), nameof(EgressRemoveLearner.DateOfBirth), nameof(EgressRemoveLearner.CycleYear), + nameof(EgressRemoveLearner.CycleMonth), nameof(EgressRemoveLearner.LocalAuthority), nameof(EgressRemoveLearner.LearnerId), + nameof(EgressRemoveLearner.YearGroup), nameof(EgressRemoveLearner.RemovalYear0), nameof(EgressRemoveLearner.RemovalYear1), nameof(EgressRemoveLearner.RemovalYear2) }) + { + builder.Property(name).IsRequired().HasMaxLength(200); + } + builder.HasOne().WithMany().HasForeignKey(x => x.RunId).OnDelete(DeleteBehavior.Cascade); + builder.HasIndex(x => x.RunId); + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Entities/EgressRun.cs b/src/DfE.CheckPerformanceData.Persistence/Entities/EgressRun.cs new file mode 100644 index 000000000..d9ab979f6 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Entities/EgressRun.cs @@ -0,0 +1,94 @@ +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace DfE.CheckPerformanceData.Persistence.Entities; + +/// +/// One data-egress run (AB#294553): a pull of Zendesk decisions for one window and one or more +/// output types, carried through preprocessing to transfer. Persisted at pull time so "save" is +/// nothing more than leaving the page and a resume never re-pulls. +/// +public sealed class EgressRun +{ + public Guid Id { get; set; } + public Guid WindowId { get; set; } + public EgressRunStatus Status { get; set; } + public Guid StartedById { get; set; } + public string StartedByName { get; set; } = string.Empty; + public string? StartedByEmail { get; set; } + public DateTime StartedAtUtc { get; set; } + public DateTime? PreprocessedAtUtc { get; set; } + /// The London calendar date stamped into every file name, fixed when preprocessing completes so summary and transfer agree. + public DateOnly? ExportDate { get; set; } + public DateTime? TransferredAtUtc { get; set; } + public string? TransferredByName { get; set; } + /// JSON array of EgressRecordFailure when preprocessing failed; null otherwise. + public string? FailureJson { get; set; } + /// Why the last transfer failed, for the summary page; null otherwise. + public string? TransferFailureReason { get; set; } + public List Outputs { get; set; } = []; +} + +/// +/// One output type within a run. IsActive is the concurrency lock: true from pull until the run +/// fails or is abandoned, and it STAYS true after a successful transfer so the same window and +/// type can never be sent twice. A partial unique index on (WindowId, OutputType) WHERE IsActive +/// is what refuses the second run; WindowId is duplicated here for that index. +/// +public sealed class EgressRunOutput +{ + public Guid Id { get; set; } + public Guid RunId { get; set; } + public Guid WindowId { get; set; } + public EgressOutputType OutputType { get; set; } + public bool IsActive { get; set; } + /// JSON array of EgressSourceRecord — the "data pulled" the Results screen shows and a resume reuses. + public string RawRecordsJson { get; set; } = "[]"; + public int SourceRecordCount { get; set; } + public int? OutputRecordCount { get; set; } + public string? FileName { get; set; } + public string? Sha256 { get; set; } +} + +public sealed class EgressRunConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("egress_runs"); + builder.HasKey(x => x.Id); + builder.Property(x => x.Status).IsRequired().HasConversion().HasMaxLength(30); + builder.Property(x => x.StartedByName).IsRequired().HasMaxLength(200); + builder.Property(x => x.StartedByEmail).HasMaxLength(256); + builder.Property(x => x.TransferredByName).HasMaxLength(200); + builder.Property(x => x.StartedAtUtc).HasColumnType("timestamp with time zone"); + builder.Property(x => x.PreprocessedAtUtc).HasColumnType("timestamp with time zone"); + builder.Property(x => x.TransferredAtUtc).HasColumnType("timestamp with time zone"); + builder.Property(x => x.TransferFailureReason).HasMaxLength(1000); + builder.HasOne().WithMany().HasForeignKey(x => x.WindowId).OnDelete(DeleteBehavior.Restrict); + builder.HasMany(x => x.Outputs).WithOne().HasForeignKey(o => o.RunId).OnDelete(DeleteBehavior.Cascade); + builder.HasIndex(x => x.WindowId); + builder.HasIndex(x => x.StartedAtUtc); + } +} + +public sealed class EgressRunOutputConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("egress_run_outputs"); + builder.HasKey(x => x.Id); + builder.Property(x => x.OutputType).IsRequired().HasConversion().HasMaxLength(20); + builder.Property(x => x.RawRecordsJson).IsRequired(); + builder.Property(x => x.FileName).HasMaxLength(100); + builder.Property(x => x.Sha256).HasMaxLength(64); + // The concurrency rule (AB#294553): one ACTIVE output per window and type. Partial, so a + // failed or abandoned run (IsActive = false) never blocks a new one, while a transferred + // run (still active) blocks forever. + builder.HasIndex(x => new { x.WindowId, x.OutputType }) + .IsUnique() + .HasFilter("\"IsActive\" = TRUE") + .HasDatabaseName("ix_egress_run_outputs_active_window_output"); + builder.HasIndex(x => x.RunId); + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.Designer.cs b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.Designer.cs new file mode 100644 index 000000000..8aea36a68 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.Designer.cs @@ -0,0 +1,1827 @@ +// +using System; +using DfE.CheckPerformanceData.Persistence.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using NpgsqlTypes; + +#nullable disable + +namespace DfE.CheckPerformanceData.Persistence.Migrations +{ + [DbContext(typeof(PortalDbContext))] + [Migration("20260914160145_AddDataEgress")] + partial class AddDataEgress + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.AuditEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Action") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ChangedColumns") + .HasColumnType("text"); + + b.Property("EntityId") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NewValues") + .HasColumnType("text"); + + b.Property("OldValues") + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("EntityType"); + + b.HasIndex("Timestamp"); + + b.ToTable("AuditEntries"); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.ContentStagingSession", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BundleJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("bundle_json"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("CreatedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("ExpiresAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at_utc"); + + b.HasKey("Id"); + + b.HasIndex("ExpiresAtUtc") + .HasDatabaseName("ix_content_staging_sessions_expires_at_utc"); + + b.ToTable("content_staging_sessions", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.QueueMetricEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DecisionStatus") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("decision_status"); + + b.Property("LatencyMs") + .HasColumnType("double precision") + .HasColumnName("latency_ms"); + + b.Property("MessageId") + .HasColumnType("uuid") + .HasColumnName("message_id"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("RecordedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at_utc"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("reference_number"); + + b.Property("RulesVersion") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("rules_version"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("stage"); + + b.HasKey("Id"); + + b.HasIndex("RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_recorded_at"); + + b.HasIndex("QueueName", "RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_queue_recorded"); + + b.HasIndex("ReferenceNumber", "RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_reference"); + + b.ToTable("queue_metrics_events", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("LatencyMs") + .HasColumnType("integer") + .HasColumnName("latency_ms"); + + b.Property("OccurredAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("occurred_at_utc"); + + b.Property("QueryNormalised") + .HasColumnType("text") + .HasColumnName("query_normalised"); + + b.Property("QueryRaw") + .HasColumnType("text") + .HasColumnName("query_raw"); + + b.Property("ResultsBlocks") + .HasColumnType("integer") + .HasColumnName("results_blocks"); + + b.Property("ResultsPages") + .HasColumnType("integer") + .HasColumnName("results_pages"); + + b.Property("ResultsTotal") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("integer") + .HasColumnName("results_total") + .HasComputedColumnSql("results_pages + results_blocks", true); + + b.Property("Scope") + .HasColumnType("text") + .HasColumnName("scope"); + + b.Property("SessionId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("session_id"); + + b.Property("ZeroResults") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("zero_results") + .HasComputedColumnSql("(results_pages + results_blocks) = 0", true); + + b.HasKey("Id"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_events_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("OccurredAtUtc") + .HasDatabaseName("ix_search_events_occurred_at"); + + b.HasIndex("QueryNormalised") + .HasDatabaseName("ix_search_events_query_normalised"); + + b.HasIndex("SessionId") + .HasDatabaseName("ix_search_events_session_id"); + + b.HasIndex("OccurredAtUtc", "QueryNormalised") + .HasDatabaseName("ix_search_events_occurred_at_query_normalised") + .HasFilter("query_normalised IS NOT NULL"); + + b.HasIndex("OccurredAtUtc", "SessionId") + .HasDatabaseName("ix_search_events_occurred_at_session_id"); + + b.HasIndex("ZeroResults", "OccurredAtUtc") + .HasDatabaseName("ix_search_events_zero_results_occurred_at") + .HasFilter("zero_results = true"); + + b.ToTable("search_events", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEventResult", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("Position") + .HasColumnType("integer") + .HasColumnName("position"); + + b.Property("Rank") + .HasColumnType("real") + .HasColumnName("rank"); + + b.Property("ResultKey") + .IsRequired() + .HasColumnType("text") + .HasColumnName("result_key"); + + b.Property("ResultKind") + .IsRequired() + .HasColumnType("text") + .HasColumnName("result_kind"); + + b.Property("SearchEventId") + .HasColumnType("bigint") + .HasColumnName("search_event_id"); + + b.HasKey("Id"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_event_results_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("SearchEventId") + .HasDatabaseName("ix_search_event_results_search_event_id"); + + b.ToTable("search_event_results", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Email") + .HasColumnType("text") + .HasColumnName("email"); + + b.Property("IsRead") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_read"); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("ReadAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("read_at_utc"); + + b.Property("ReadByAdminSub") + .HasColumnType("text") + .HasColumnName("read_by_admin_sub"); + + b.Property("SessionId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("session_id"); + + b.Property("SubmittedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted_at_utc"); + + b.Property("WhatGot") + .HasColumnType("text") + .HasColumnName("what_got"); + + b.Property("WhatLookingFor") + .IsRequired() + .HasColumnType("text") + .HasColumnName("what_looking_for"); + + b.HasKey("Id"); + + b.HasIndex("IsRead") + .HasDatabaseName("ix_search_messages_is_read"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_messages_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("SessionId") + .HasDatabaseName("ix_search_messages_session_id"); + + b.HasIndex("SubmittedAtUtc") + .HasDatabaseName("ix_search_messages_submitted_at"); + + b.ToTable("search_messages", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.ShareToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("label"); + + b.Property("RevokedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("revoked_at_utc"); + + b.Property("Surface") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("surface"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash"); + + b.HasKey("Id"); + + b.HasIndex("TokenHash") + .HasDatabaseName("ix_share_tokens_token_hash"); + + b.ToTable("share_tokens", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.AdminSectionAccess", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("RoleName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("SectionKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("Id"); + + b.HasIndex("RoleName", "SectionKey") + .IsUnique(); + + b.ToTable("AdminSectionAccesses"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.AppLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("CorrelationId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("EventId") + .HasColumnType("integer"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("RequestPath") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("StateJson") + .HasColumnType("jsonb"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("Category"); + + b.HasIndex("Level"); + + b.HasIndex("Timestamp") + .IsDescending(); + + b.ToTable("AppLogs"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ChangeRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AmendmentType") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("CheckingExerciseId") + .HasColumnType("uuid"); + + b.Property("CrmId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("DecidedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("DecisionTrace") + .HasColumnType("text"); + + b.Property("MatchedRuleId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("OrganisationLaestab") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("OrganisationUrn") + .HasColumnType("bigint"); + + b.Property("Outcome") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("OutcomeKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilFirstname") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilId") + .HasColumnType("uuid"); + + b.Property("PupilSurname") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilUpn") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RequestType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RequestTypeDescription") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("RulesVersion") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("Submitted") + .HasColumnType("timestamp without time zone"); + + b.Property("SubmittedByEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("SubmittedById") + .HasColumnType("uuid"); + + b.Property("SubmittedByName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.Property("WithdrawnAt") + .HasColumnType("timestamp with time zone"); + + b.Property("WithdrawnByEmail") + .HasColumnType("text"); + + b.Property("WorkerStatus") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("CheckingExerciseId"); + + b.HasIndex("CrmId") + .IsUnique() + .HasFilter("\"CrmId\" IS NOT NULL"); + + b.HasIndex("ReferenceNumber") + .IsUnique(); + + b.HasIndex("Status"); + + b.HasIndex("WindowId", "OrganisationUrn"); + + b.ToTable("ChangeRequests"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingWindowId") + .HasColumnType("uuid"); + + b.Property("EndDate") + .HasColumnType("timestamp without time zone"); + + b.Property("ExerciseType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("StartDate") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("CheckingWindowId", "ExerciseType") + .IsUnique(); + + b.ToTable("CheckingExercises", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingWindowType") + .IsRequired() + .HasColumnType("text"); + + b.Property("EndDate") + .HasColumnType("timestamp without time zone"); + + b.Property("IngressFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("IngressFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("KeyStage") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextOpportunity") + .HasColumnType("timestamp without time zone"); + + b.Property("Published") + .HasColumnType("boolean"); + + b.Property("SchemaFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("SchemaFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("StartDate") + .HasColumnType("timestamp without time zone"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TurnaroundCommitment") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.ToTable("CheckingWindows"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindowDataset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingExerciseId") + .HasColumnType("uuid"); + + b.Property("CheckingWindowId") + .HasColumnType("uuid"); + + b.Property("Included") + .HasColumnType("boolean"); + + b.Property("IngressFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("IngressFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Required") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("SchemaFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("SchemaFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("SourceFile") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("CheckingExerciseId", "Name") + .IsUnique(); + + b.ToTable("CheckingWindowDatasets"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AppearInSearch") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("BlockType") + .IsRequired() + .HasColumnType("text"); + + b.Property("ContentId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("Keywords") + .HasColumnType("text"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSeenPath") + .HasColumnType("text"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"Keywords\", '')), 'A') || setweight(to_tsvector('english', coalesce(\"ValuePlainText\", '')), 'B')", true); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("ValuePlainText") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.HasKey("Id"); + + b.HasIndex("ContentId") + .IsUnique(); + + b.HasIndex("Key") + .IsUnique(); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.ToTable("ContentBlocks"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ContentBlockId") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("VersionNumber") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId", "VersionNumber") + .IsUnique(); + + b.ToTable("ContentBlockVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.Country", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("OfficialName") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("Name"); + + b.ToTable("Countries"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.DeadLetterEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("DeadLetteredAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("dead_lettered_at_utc"); + + b.Property("EnqueuedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("enqueued_at_utc"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text") + .HasColumnName("payload"); + + b.Property("PayloadHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("payload_hash"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("Reason") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("reason"); + + b.HasKey("Id"); + + b.HasIndex("DeadLetteredAtUtc") + .HasDatabaseName("ix_queue_dead_letters_dead_lettered_at"); + + b.ToTable("queue_dead_letters", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.DevZendeskTicket", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("Priority") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("priority"); + + b.Property("RawJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("raw_json"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("reference_number"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("status"); + + b.Property("Subject") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("subject"); + + b.Property("TicketId") + .HasColumnType("bigint") + .HasColumnName("ticket_id"); + + b.HasKey("Id"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("ix_dev_zendesk_outbox_created_at"); + + b.ToTable("dev_zendesk_outbox", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdmissionDate") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("MiddleName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Postcode") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("SchoolUrn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("SenStatus") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.Property("Uln") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Upn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("YearGroup") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("new_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionReason") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("remove_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ExportDate") + .HasColumnType("date"); + + b.Property("FailureJson") + .HasColumnType("text"); + + b.Property("PreprocessedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedByEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("StartedById") + .HasColumnType("uuid"); + + b.Property("StartedByName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("TransferFailureReason") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("TransferredAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("TransferredByName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("StartedAtUtc"); + + b.HasIndex("WindowId"); + + b.ToTable("egress_runs", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("OutputRecordCount") + .HasColumnType("integer"); + + b.Property("OutputType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RawRecordsJson") + .IsRequired() + .HasColumnType("text"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sha256") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SourceRecordCount") + .HasColumnType("integer"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.HasIndex("WindowId", "OutputType") + .IsUnique() + .HasDatabaseName("ix_egress_run_outputs_active_window_output") + .HasFilter("\"IsActive\" = TRUE"); + + b.ToTable("egress_run_outputs", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.OrganisationLogin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("Laestab") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("LoggedInAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("OrganisationName") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("OrganisationUrn") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("LoggedInAtUtc"); + + b.HasIndex("OrganisationUrn", "LoggedInAtUtc"); + + b.ToTable("OrganisationLogins"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AppearInSearch") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("DeletedBy") + .HasColumnType("text"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Keywords") + .HasColumnType("text"); + + b.Property("PageName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PageType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"Keywords\", '')), 'A') || setweight(to_tsvector('english', coalesce(\"Title\", '')), 'B') || setweight(to_tsvector('english', coalesce(\"Subtitle\", '')), 'C')", true); + + b.Property("Segment") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("ShowInMenu") + .HasColumnType("boolean"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("Subtitle") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ParentId"); + + b.HasIndex("Path") + .IsUnique() + .HasFilter("\"DeletedDate\" IS NULL"); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.ToTable("PageNodes"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNodeVersion", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("BodyPlainText") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("IsCurrent") + .HasColumnType("boolean"); + + b.Property("MinorVersion") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("PageNodeId") + .HasColumnType("uuid"); + + b.Property("PublishFrom") + .HasColumnType("timestamp with time zone"); + + b.Property("PublishTo") + .HasColumnType("timestamp with time zone"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"BodyPlainText\", '')), 'D')", true); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("VersionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.HasIndex("PageNodeId", "IsCurrent"); + + b.HasIndex("PageNodeId", "VersionId") + .IsUnique(); + + b.ToTable("PageNodeVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.QueueMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("EnqueuedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("enqueued_at_utc"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text") + .HasColumnName("payload"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("status"); + + b.Property("VisibleAfterUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("visible_after_utc"); + + b.HasKey("Id"); + + b.HasIndex("QueueName", "Status", "VisibleAfterUtc") + .HasDatabaseName("ix_queue_messages_claim"); + + b.ToTable("queue_messages", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.RulesConfigVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConfigType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("VersionNumber") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ConfigType", "VersionNumber") + .IsUnique(); + + b.ToTable("RulesConfigVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.Setting", b => + { + b.Property("Key") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Value") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEventResult", b => + { + b.HasOne("DfE.CheckPerformance.Persistence.Entities.SearchEvent", null) + .WithMany() + .HasForeignKey("SearchEventId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ChangeRequest", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", null) + .WithMany() + .HasForeignKey("CheckingExerciseId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany() + .HasForeignKey("WindowId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany("CheckingExercises") + .HasForeignKey("CheckingWindowId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("DfE.CheckPerformanceData.Persistence.Entities.ExerciseValidated", "Validated", b1 => + { + b1.Property("CheckingExerciseId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("IngressValidationChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b1.Property("SchemaValidationChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b1.Property("ValidatedAt") + .HasColumnType("timestamp with time zone"); + + b1.HasKey("CheckingExerciseId"); + + b1.ToTable("CheckingExercises"); + + b1.WithOwner() + .HasForeignKey("CheckingExerciseId"); + }); + + b.Navigation("Validated"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindowDataset", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", null) + .WithMany("Datasets") + .HasForeignKey("CheckingExerciseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlockVersion", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", "ContentBlock") + .WithMany("Versions") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany() + .HasForeignKey("WindowId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany("Outputs") + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.PageNode", null) + .WithMany() + .HasForeignKey("ParentId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_PageNode_PageNode_ParentId"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNodeVersion", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.PageNode", "PageNode") + .WithMany("Versions") + .HasForeignKey("PageNodeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("PageNode"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.Navigation("Datasets"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", b => + { + b.Navigation("CheckingExercises"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Navigation("Outputs"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.Navigation("Versions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.cs b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.cs new file mode 100644 index 000000000..39f3dd276 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260914160145_AddDataEgress.cs @@ -0,0 +1,204 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DfE.CheckPerformanceData.Persistence.Migrations +{ + /// + public partial class AddDataEgress : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "OrganisationLaestab", + table: "ChangeRequests", + type: "character varying(20)", + maxLength: 20, + nullable: true); + + migrationBuilder.CreateTable( + name: "egress_runs", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + WindowId = table.Column(type: "uuid", nullable: false), + Status = table.Column(type: "character varying(30)", maxLength: 30, nullable: false), + StartedById = table.Column(type: "uuid", nullable: false), + StartedByName = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + StartedByEmail = table.Column(type: "character varying(256)", maxLength: 256, nullable: true), + StartedAtUtc = table.Column(type: "timestamp with time zone", nullable: false), + PreprocessedAtUtc = table.Column(type: "timestamp with time zone", nullable: true), + ExportDate = table.Column(type: "date", nullable: true), + TransferredAtUtc = table.Column(type: "timestamp with time zone", nullable: true), + TransferredByName = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + FailureJson = table.Column(type: "text", nullable: true), + TransferFailureReason = table.Column(type: "character varying(1000)", maxLength: 1000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_egress_runs", x => x.Id); + table.ForeignKey( + name: "FK_egress_runs_CheckingWindows_WindowId", + column: x => x.WindowId, + principalTable: "CheckingWindows", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "egress_run_outputs", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + RunId = table.Column(type: "uuid", nullable: false), + WindowId = table.Column(type: "uuid", nullable: false), + OutputType = table.Column(type: "character varying(20)", maxLength: 20, nullable: false), + IsActive = table.Column(type: "boolean", nullable: false), + RawRecordsJson = table.Column(type: "text", nullable: false), + SourceRecordCount = table.Column(type: "integer", nullable: false), + OutputRecordCount = table.Column(type: "integer", nullable: true), + FileName = table.Column(type: "character varying(100)", maxLength: 100, nullable: true), + Sha256 = table.Column(type: "character varying(64)", maxLength: 64, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_egress_run_outputs", x => x.Id); + table.ForeignKey( + name: "FK_egress_run_outputs_egress_runs_RunId", + column: x => x.RunId, + principalTable: "egress_runs", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "new_learners", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + RunId = table.Column(type: "uuid", nullable: false), + ChangeRequestId = table.Column(type: "uuid", nullable: false), + TicketId = table.Column(type: "bigint", nullable: true), + ReferenceNumber = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), + CorrectionId = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CorrectionType = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + KeyStage = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + LocalAuthority = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + EstablishmentNumber = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Surname = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + MiddleName = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Forename = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Sex = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + DateOfBirth = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + AdmissionDate = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Postcode = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CycleYear = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CycleMonth = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + SchoolUrn = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Uln = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Upn = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + LearnerId = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + YearGroup = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + SenStatus = table.Column(type: "character varying(200)", maxLength: 200, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_new_learners", x => x.Id); + table.ForeignKey( + name: "FK_new_learners_egress_runs_RunId", + column: x => x.RunId, + principalTable: "egress_runs", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "remove_learners", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + RunId = table.Column(type: "uuid", nullable: false), + ChangeRequestId = table.Column(type: "uuid", nullable: false), + TicketId = table.Column(type: "bigint", nullable: true), + ReferenceNumber = table.Column(type: "character varying(50)", maxLength: 50, nullable: false), + CorrectionId = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CorrectionType = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CorrectionReason = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + KeyStage = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + EstablishmentNumber = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Surname = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Forename = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + Sex = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + DateOfBirth = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CycleYear = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + CycleMonth = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + LocalAuthority = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + LearnerId = table.Column(type: "character varying(200)", maxLength: 200, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_remove_learners", x => x.Id); + table.ForeignKey( + name: "FK_remove_learners_egress_runs_RunId", + column: x => x.RunId, + principalTable: "egress_runs", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_egress_run_outputs_active_window_output", + table: "egress_run_outputs", + columns: new[] { "WindowId", "OutputType" }, + unique: true, + filter: "\"IsActive\" = TRUE"); + + migrationBuilder.CreateIndex( + name: "IX_egress_run_outputs_RunId", + table: "egress_run_outputs", + column: "RunId"); + + migrationBuilder.CreateIndex( + name: "IX_egress_runs_StartedAtUtc", + table: "egress_runs", + column: "StartedAtUtc"); + + migrationBuilder.CreateIndex( + name: "IX_egress_runs_WindowId", + table: "egress_runs", + column: "WindowId"); + + migrationBuilder.CreateIndex( + name: "IX_new_learners_RunId", + table: "new_learners", + column: "RunId"); + + migrationBuilder.CreateIndex( + name: "IX_remove_learners_RunId", + table: "remove_learners", + column: "RunId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "egress_run_outputs"); + + migrationBuilder.DropTable( + name: "new_learners"); + + migrationBuilder.DropTable( + name: "remove_learners"); + + migrationBuilder.DropTable( + name: "egress_runs"); + + migrationBuilder.DropColumn( + name: "OrganisationLaestab", + table: "ChangeRequests"); + } + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.Designer.cs b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.Designer.cs new file mode 100644 index 000000000..bc5d86da4 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.Designer.cs @@ -0,0 +1,1837 @@ +// +using System; +using DfE.CheckPerformanceData.Persistence.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using NpgsqlTypes; + +#nullable disable + +namespace DfE.CheckPerformanceData.Persistence.Migrations +{ + [DbContext(typeof(PortalDbContext))] + [Migration("20260916103225_AlignEgressWithLdsSpecV24")] + partial class AlignEgressWithLdsSpecV24 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.AuditEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Action") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ChangedColumns") + .HasColumnType("text"); + + b.Property("EntityId") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NewValues") + .HasColumnType("text"); + + b.Property("OldValues") + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("EntityType"); + + b.HasIndex("Timestamp"); + + b.ToTable("AuditEntries"); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.ContentStagingSession", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BundleJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("bundle_json"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("CreatedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("ExpiresAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at_utc"); + + b.HasKey("Id"); + + b.HasIndex("ExpiresAtUtc") + .HasDatabaseName("ix_content_staging_sessions_expires_at_utc"); + + b.ToTable("content_staging_sessions", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.QueueMetricEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DecisionStatus") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("decision_status"); + + b.Property("LatencyMs") + .HasColumnType("double precision") + .HasColumnName("latency_ms"); + + b.Property("MessageId") + .HasColumnType("uuid") + .HasColumnName("message_id"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("RecordedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at_utc"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("reference_number"); + + b.Property("RulesVersion") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("rules_version"); + + b.Property("Stage") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("stage"); + + b.HasKey("Id"); + + b.HasIndex("RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_recorded_at"); + + b.HasIndex("QueueName", "RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_queue_recorded"); + + b.HasIndex("ReferenceNumber", "RecordedAtUtc") + .HasDatabaseName("ix_queue_metrics_events_reference"); + + b.ToTable("queue_metrics_events", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("LatencyMs") + .HasColumnType("integer") + .HasColumnName("latency_ms"); + + b.Property("OccurredAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("occurred_at_utc"); + + b.Property("QueryNormalised") + .HasColumnType("text") + .HasColumnName("query_normalised"); + + b.Property("QueryRaw") + .HasColumnType("text") + .HasColumnName("query_raw"); + + b.Property("ResultsBlocks") + .HasColumnType("integer") + .HasColumnName("results_blocks"); + + b.Property("ResultsPages") + .HasColumnType("integer") + .HasColumnName("results_pages"); + + b.Property("ResultsTotal") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("integer") + .HasColumnName("results_total") + .HasComputedColumnSql("results_pages + results_blocks", true); + + b.Property("Scope") + .HasColumnType("text") + .HasColumnName("scope"); + + b.Property("SessionId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("session_id"); + + b.Property("ZeroResults") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("boolean") + .HasColumnName("zero_results") + .HasComputedColumnSql("(results_pages + results_blocks) = 0", true); + + b.HasKey("Id"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_events_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("OccurredAtUtc") + .HasDatabaseName("ix_search_events_occurred_at"); + + b.HasIndex("QueryNormalised") + .HasDatabaseName("ix_search_events_query_normalised"); + + b.HasIndex("SessionId") + .HasDatabaseName("ix_search_events_session_id"); + + b.HasIndex("OccurredAtUtc", "QueryNormalised") + .HasDatabaseName("ix_search_events_occurred_at_query_normalised") + .HasFilter("query_normalised IS NOT NULL"); + + b.HasIndex("OccurredAtUtc", "SessionId") + .HasDatabaseName("ix_search_events_occurred_at_session_id"); + + b.HasIndex("ZeroResults", "OccurredAtUtc") + .HasDatabaseName("ix_search_events_zero_results_occurred_at") + .HasFilter("zero_results = true"); + + b.ToTable("search_events", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEventResult", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("Position") + .HasColumnType("integer") + .HasColumnName("position"); + + b.Property("Rank") + .HasColumnType("real") + .HasColumnName("rank"); + + b.Property("ResultKey") + .IsRequired() + .HasColumnType("text") + .HasColumnName("result_key"); + + b.Property("ResultKind") + .IsRequired() + .HasColumnType("text") + .HasColumnName("result_kind"); + + b.Property("SearchEventId") + .HasColumnType("bigint") + .HasColumnName("search_event_id"); + + b.HasKey("Id"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_event_results_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("SearchEventId") + .HasDatabaseName("ix_search_event_results_search_event_id"); + + b.ToTable("search_event_results", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Email") + .HasColumnType("text") + .HasColumnName("email"); + + b.Property("IsRead") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_read"); + + b.Property("IsSeeded") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_seeded"); + + b.Property("JobId") + .HasColumnType("text") + .HasColumnName("job_id"); + + b.Property("ReadAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("read_at_utc"); + + b.Property("ReadByAdminSub") + .HasColumnType("text") + .HasColumnName("read_by_admin_sub"); + + b.Property("SessionId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("session_id"); + + b.Property("SubmittedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("submitted_at_utc"); + + b.Property("WhatGot") + .HasColumnType("text") + .HasColumnName("what_got"); + + b.Property("WhatLookingFor") + .IsRequired() + .HasColumnType("text") + .HasColumnName("what_looking_for"); + + b.HasKey("Id"); + + b.HasIndex("IsRead") + .HasDatabaseName("ix_search_messages_is_read"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_search_messages_job_id") + .HasFilter("job_id IS NOT NULL"); + + b.HasIndex("SessionId") + .HasDatabaseName("ix_search_messages_session_id"); + + b.HasIndex("SubmittedAtUtc") + .HasDatabaseName("ix_search_messages_submitted_at"); + + b.ToTable("search_messages", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.ShareToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("label"); + + b.Property("RevokedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("revoked_at_utc"); + + b.Property("Surface") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("surface"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash"); + + b.HasKey("Id"); + + b.HasIndex("TokenHash") + .HasDatabaseName("ix_share_tokens_token_hash"); + + b.ToTable("share_tokens", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.AdminSectionAccess", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("RoleName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("SectionKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.HasKey("Id"); + + b.HasIndex("RoleName", "SectionKey") + .IsUnique(); + + b.ToTable("AdminSectionAccesses"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.AppLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("CorrelationId") + .HasMaxLength(128) + .HasColumnType("character varying(128)"); + + b.Property("EventId") + .HasColumnType("integer"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("RequestPath") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("StateJson") + .HasColumnType("jsonb"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("Category"); + + b.HasIndex("Level"); + + b.HasIndex("Timestamp") + .IsDescending(); + + b.ToTable("AppLogs"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ChangeRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AmendmentType") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("CheckingExerciseId") + .HasColumnType("uuid"); + + b.Property("CrmId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("DecidedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("DecisionTrace") + .HasColumnType("text"); + + b.Property("MatchedRuleId") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("OrganisationLaestab") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("OrganisationUrn") + .HasColumnType("bigint"); + + b.Property("Outcome") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("OutcomeKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilFirstname") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilId") + .HasColumnType("uuid"); + + b.Property("PupilSurname") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PupilUpn") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RequestType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RequestTypeDescription") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("RulesVersion") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("Submitted") + .HasColumnType("timestamp without time zone"); + + b.Property("SubmittedByEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("SubmittedById") + .HasColumnType("uuid"); + + b.Property("SubmittedByName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.Property("WithdrawnAt") + .HasColumnType("timestamp with time zone"); + + b.Property("WithdrawnByEmail") + .HasColumnType("text"); + + b.Property("WorkerStatus") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("CheckingExerciseId"); + + b.HasIndex("CrmId") + .IsUnique() + .HasFilter("\"CrmId\" IS NOT NULL"); + + b.HasIndex("ReferenceNumber") + .IsUnique(); + + b.HasIndex("Status"); + + b.HasIndex("WindowId", "OrganisationUrn"); + + b.ToTable("ChangeRequests"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingWindowId") + .HasColumnType("uuid"); + + b.Property("EndDate") + .HasColumnType("timestamp without time zone"); + + b.Property("ExerciseType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("StartDate") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("CheckingWindowId", "ExerciseType") + .IsUnique(); + + b.ToTable("CheckingExercises", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingWindowType") + .IsRequired() + .HasColumnType("text"); + + b.Property("EndDate") + .HasColumnType("timestamp without time zone"); + + b.Property("IngressFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("IngressFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("KeyStage") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextOpportunity") + .HasColumnType("timestamp without time zone"); + + b.Property("Published") + .HasColumnType("boolean"); + + b.Property("SchemaFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("SchemaFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("StartDate") + .HasColumnType("timestamp without time zone"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TurnaroundCommitment") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.ToTable("CheckingWindows"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindowDataset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckingExerciseId") + .HasColumnType("uuid"); + + b.Property("CheckingWindowId") + .HasColumnType("uuid"); + + b.Property("Included") + .HasColumnType("boolean"); + + b.Property("IngressFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("IngressFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Required") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("SchemaFile") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("SchemaFileChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("SourceFile") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("CheckingExerciseId", "Name") + .IsUnique(); + + b.ToTable("CheckingWindowDatasets"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AppearInSearch") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("BlockType") + .IsRequired() + .HasColumnType("text"); + + b.Property("ContentId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("Keywords") + .HasColumnType("text"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSeenPath") + .HasColumnType("text"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"Keywords\", '')), 'A') || setweight(to_tsvector('english', coalesce(\"ValuePlainText\", '')), 'B')", true); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("ValuePlainText") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.HasKey("Id"); + + b.HasIndex("ContentId") + .IsUnique(); + + b.HasIndex("Key") + .IsUnique(); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.ToTable("ContentBlocks"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ContentBlockId") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.Property("VersionNumber") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId", "VersionNumber") + .IsUnique(); + + b.ToTable("ContentBlockVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.Country", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("OfficialName") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("Name"); + + b.ToTable("Countries"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.DeadLetterEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("DeadLetteredAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("dead_lettered_at_utc"); + + b.Property("EnqueuedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("enqueued_at_utc"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text") + .HasColumnName("payload"); + + b.Property("PayloadHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("payload_hash"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("Reason") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("reason"); + + b.HasKey("Id"); + + b.HasIndex("DeadLetteredAtUtc") + .HasDatabaseName("ix_queue_dead_letters_dead_lettered_at"); + + b.ToTable("queue_dead_letters", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.DevZendeskTicket", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at_utc"); + + b.Property("Priority") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("priority"); + + b.Property("RawJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("raw_json"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("reference_number"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("status"); + + b.Property("Subject") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("subject"); + + b.Property("TicketId") + .HasColumnType("bigint") + .HasColumnName("ticket_id"); + + b.HasKey("Id"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("ix_dev_zendesk_outbox_created_at"); + + b.ToTable("dev_zendesk_outbox", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdmissionDate") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Postcode") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("SchoolUrn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.Property("Uln") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Upn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("YearGroup") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("new_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionReason") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RemovalYear0") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RemovalYear1") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RemovalYear2") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.Property("YearGroup") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("remove_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ExportDate") + .HasColumnType("date"); + + b.Property("FailureJson") + .HasColumnType("text"); + + b.Property("PreprocessedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedByEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("StartedById") + .HasColumnType("uuid"); + + b.Property("StartedByName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("TransferFailureReason") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("TransferredAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("TransferredByName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("StartedAtUtc"); + + b.HasIndex("WindowId"); + + b.ToTable("egress_runs", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("OutputRecordCount") + .HasColumnType("integer"); + + b.Property("OutputType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RawRecordsJson") + .IsRequired() + .HasColumnType("text"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sha256") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SourceRecordCount") + .HasColumnType("integer"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.HasIndex("WindowId", "OutputType") + .IsUnique() + .HasDatabaseName("ix_egress_run_outputs_active_window_output") + .HasFilter("\"IsActive\" = TRUE"); + + b.ToTable("egress_run_outputs", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.OrganisationLogin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("Laestab") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("LoggedInAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("OrganisationName") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("OrganisationUrn") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("LoggedInAtUtc"); + + b.HasIndex("OrganisationUrn", "LoggedInAtUtc"); + + b.ToTable("OrganisationLogins"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AppearInSearch") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("DeletedBy") + .HasColumnType("text"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Keywords") + .HasColumnType("text"); + + b.Property("PageName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PageType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("character varying(2048)"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"Keywords\", '')), 'A') || setweight(to_tsvector('english', coalesce(\"Title\", '')), 'B') || setweight(to_tsvector('english', coalesce(\"Subtitle\", '')), 'C')", true); + + b.Property("Segment") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("ShowInMenu") + .HasColumnType("boolean"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.Property("Subtitle") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ParentId"); + + b.HasIndex("Path") + .IsUnique() + .HasFilter("\"DeletedDate\" IS NULL"); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.ToTable("PageNodes"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNodeVersion", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("BodyPlainText") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("CreatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("IsCurrent") + .HasColumnType("boolean"); + + b.Property("MinorVersion") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0); + + b.Property("PageNodeId") + .HasColumnType("uuid"); + + b.Property("PublishFrom") + .HasColumnType("timestamp with time zone"); + + b.Property("PublishTo") + .HasColumnType("timestamp with time zone"); + + b.Property("SearchVector") + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("tsvector") + .HasComputedColumnSql("setweight(to_tsvector('english', coalesce(\"BodyPlainText\", '')), 'D')", true); + + b.Property("UpdatedBy") + .HasColumnType("text"); + + b.Property("UpdatedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("VersionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("SearchVector"); + + NpgsqlIndexBuilderExtensions.HasMethod(b.HasIndex("SearchVector"), "gin"); + + b.HasIndex("PageNodeId", "IsCurrent"); + + b.HasIndex("PageNodeId", "VersionId") + .IsUnique(); + + b.ToTable("PageNodeVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.QueueMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("EnqueuedAtUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("enqueued_at_utc"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("text") + .HasColumnName("payload"); + + b.Property("QueueName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("queue_name"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("status"); + + b.Property("VisibleAfterUtc") + .HasColumnType("timestamp with time zone") + .HasColumnName("visible_after_utc"); + + b.HasKey("Id"); + + b.HasIndex("QueueName", "Status", "VisibleAfterUtc") + .HasDatabaseName("ix_queue_messages_claim"); + + b.ToTable("queue_messages", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.RulesConfigVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ConfigType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedBy") + .HasColumnType("text"); + + b.Property("VersionNumber") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ConfigType", "VersionNumber") + .IsUnique(); + + b.ToTable("RulesConfigVersions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.Setting", b => + { + b.Property("Key") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Value") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue(""); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DfE.CheckPerformance.Persistence.Entities.SearchEventResult", b => + { + b.HasOne("DfE.CheckPerformance.Persistence.Entities.SearchEvent", null) + .WithMany() + .HasForeignKey("SearchEventId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ChangeRequest", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", null) + .WithMany() + .HasForeignKey("CheckingExerciseId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany() + .HasForeignKey("WindowId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany("CheckingExercises") + .HasForeignKey("CheckingWindowId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("DfE.CheckPerformanceData.Persistence.Entities.ExerciseValidated", "Validated", b1 => + { + b1.Property("CheckingExerciseId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b1.Property("IngressValidationChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b1.Property("SchemaValidationChecksum") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b1.Property("ValidatedAt") + .HasColumnType("timestamp with time zone"); + + b1.HasKey("CheckingExerciseId"); + + b1.ToTable("CheckingExercises"); + + b1.WithOwner() + .HasForeignKey("CheckingExerciseId"); + }); + + b.Navigation("Validated"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindowDataset", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", null) + .WithMany("Datasets") + .HasForeignKey("CheckingExerciseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlockVersion", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", "ContentBlock") + .WithMany("Versions") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany() + .HasForeignKey("WindowId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany("Outputs") + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.PageNode", null) + .WithMany() + .HasForeignKey("ParentId") + .OnDelete(DeleteBehavior.Restrict) + .HasConstraintName("FK_PageNode_PageNode_ParentId"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNodeVersion", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.PageNode", "PageNode") + .WithMany("Versions") + .HasForeignKey("PageNodeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("PageNode"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingExercise", b => + { + b.Navigation("Datasets"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", b => + { + b.Navigation("CheckingExercises"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.ContentBlock", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Navigation("Outputs"); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => + { + b.Navigation("Versions"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.cs b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.cs new file mode 100644 index 000000000..f3af86e53 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Migrations/20260916103225_AlignEgressWithLdsSpecV24.cs @@ -0,0 +1,90 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DfE.CheckPerformanceData.Persistence.Migrations +{ + /// + public partial class AlignEgressWithLdsSpecV24 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "MiddleName", + table: "new_learners"); + + migrationBuilder.DropColumn( + name: "SenStatus", + table: "new_learners"); + + migrationBuilder.AddColumn( + name: "RemovalYear0", + table: "remove_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "RemovalYear1", + table: "remove_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "RemovalYear2", + table: "remove_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "YearGroup", + table: "remove_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "RemovalYear0", + table: "remove_learners"); + + migrationBuilder.DropColumn( + name: "RemovalYear1", + table: "remove_learners"); + + migrationBuilder.DropColumn( + name: "RemovalYear2", + table: "remove_learners"); + + migrationBuilder.DropColumn( + name: "YearGroup", + table: "remove_learners"); + + migrationBuilder.AddColumn( + name: "MiddleName", + table: "new_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "SenStatus", + table: "new_learners", + type: "character varying(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Migrations/PortalDbContextModelSnapshot.cs b/src/DfE.CheckPerformanceData.Persistence/Migrations/PortalDbContextModelSnapshot.cs index 2b2f05e88..af60ac9fa 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Migrations/PortalDbContextModelSnapshot.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Migrations/PortalDbContextModelSnapshot.cs @@ -581,6 +581,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(100) .HasColumnType("character varying(100)"); + b.Property("OrganisationLaestab") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + b.Property("OrganisationUrn") .HasColumnType("bigint"); @@ -1076,6 +1080,343 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("dev_zendesk_outbox", (string)null); }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AdmissionDate") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Postcode") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("SchoolUrn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.Property("Uln") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Upn") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("YearGroup") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("new_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ChangeRequestId") + .HasColumnType("uuid"); + + b.Property("CorrectionId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionReason") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CorrectionType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleMonth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("CycleYear") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("DateOfBirth") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("EstablishmentNumber") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Forename") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("KeyStage") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LearnerId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("LocalAuthority") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("ReferenceNumber") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("RemovalYear0") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RemovalYear1") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RemovalYear2") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sex") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Surname") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("TicketId") + .HasColumnType("bigint"); + + b.Property("YearGroup") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.ToTable("remove_learners", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ExportDate") + .HasColumnType("date"); + + b.Property("FailureJson") + .HasColumnType("text"); + + b.Property("PreprocessedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("StartedByEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("StartedById") + .HasColumnType("uuid"); + + b.Property("StartedByName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("TransferFailureReason") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("TransferredAtUtc") + .HasColumnType("timestamp with time zone"); + + b.Property("TransferredByName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("StartedAtUtc"); + + b.HasIndex("WindowId"); + + b.ToTable("egress_runs", (string)null); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("FileName") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("OutputRecordCount") + .HasColumnType("integer"); + + b.Property("OutputType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RawRecordsJson") + .IsRequired() + .HasColumnType("text"); + + b.Property("RunId") + .HasColumnType("uuid"); + + b.Property("Sha256") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("SourceRecordCount") + .HasColumnType("integer"); + + b.Property("WindowId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RunId"); + + b.HasIndex("WindowId", "OutputType") + .IsUnique() + .HasDatabaseName("ix_egress_run_outputs_active_window_output") + .HasFilter("\"IsActive\" = TRUE"); + + b.ToTable("egress_run_outputs", (string)null); + }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.OrganisationLogin", b => { b.Property("Id") @@ -1439,6 +1780,42 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("ContentBlock"); }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressNewLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRemoveLearner", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany() + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.CheckingWindow", null) + .WithMany() + .HasForeignKey("WindowId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRunOutput", b => + { + b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", null) + .WithMany("Outputs") + .HasForeignKey("RunId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => { b.HasOne("DfE.CheckPerformanceData.Persistence.Entities.PageNode", null) @@ -1474,6 +1851,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Versions"); }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.EgressRun", b => + { + b.Navigation("Outputs"); + }); + modelBuilder.Entity("DfE.CheckPerformanceData.Persistence.Entities.PageNode", b => { b.Navigation("Versions"); diff --git a/src/DfE.CheckPerformanceData.Persistence/Repositories/EgressRunRepository.cs b/src/DfE.CheckPerformanceData.Persistence/Repositories/EgressRunRepository.cs new file mode 100644 index 000000000..bcc30b9a3 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Persistence/Repositories/EgressRunRepository.cs @@ -0,0 +1,286 @@ +using System.Text.Json; +using DfE.CheckPerformance.Persistence.Entities; +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.Persistence.Contexts; +using DfE.CheckPerformanceData.Persistence.Entities; +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace DfE.CheckPerformanceData.Persistence.Repositories; + +public sealed class EgressRunRepository(IPortalDbContext db) : IEgressRunRepository +{ + private const string UniqueViolation = "23505"; + // Nit: match the specific constraint, not any 23505 — a coincidental unrelated unique + // violation must not be misreported as "another run holds this pair". + private const string ActiveWindowOutputConstraint = "ix_egress_run_outputs_active_window_output"; + private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web); + + public async Task FindBlockerAsync(Guid windowId, EgressOutputType outputType, CancellationToken ct) => + await db.EgressRunOutputs.AsNoTracking() + .Where(o => o.WindowId == windowId && o.OutputType == outputType && o.IsActive) + .Join(db.EgressRuns.AsNoTracking(), o => o.RunId, r => r.Id, + (o, r) => new EgressBlocker(r.Id, r.Status, r.StartedByName, r.StartedAtUtc, r.TransferredAtUtc, r.TransferredByName)) + .FirstOrDefaultAsync(ct); + + // Every amendment of the type the window holds, whatever its ticket state: a row with no CrmId + // is shown as "No Zendesk ticket" and filtered out later, not hidden here — the ops user must + // be able to see that a request exists that scrutiny never saw. Withdrawn/draft rows are not + // amendments any more. + public async Task> GetCandidateRequestsAsync(Guid windowId, WhatToChange amendmentType, CancellationToken ct) => + await db.ChangeRequests.AsNoTracking() + .Where(r => r.WindowId == windowId + && r.RequestType == RequestType.Amendment + && r.AmendmentType == amendmentType + && (r.Status == RequestStatus.SubmittedCommitted || r.Status == RequestStatus.SubmittedUnCommitted)) + .OrderBy(r => r.Submitted) + .Select(r => new EgressCandidateRequest(r.Id, r.ReferenceNumber, r.CrmId, r.OrganisationUrn, r.OrganisationLaestab, + DateTime.SpecifyKind(r.Submitted, DateTimeKind.Utc), r.Status)) + .ToListAsync(ct); + + public async Task CreateRunAsync(EgressRunCreate create, CancellationToken ct) + { + var run = new EgressRun + { + Id = Guid.NewGuid(), WindowId = create.WindowId, Status = EgressRunStatus.Pulled, + StartedById = create.StartedById, StartedByName = create.StartedByName, StartedByEmail = create.StartedByEmail, + StartedAtUtc = DateTime.UtcNow, + Outputs = create.Outputs.Select(o => new EgressRunOutput + { + Id = Guid.NewGuid(), WindowId = create.WindowId, OutputType = o.OutputType, IsActive = true, + RawRecordsJson = JsonSerializer.Serialize(o.Records, Json), SourceRecordCount = o.Records.Count + }).ToList() + }; + db.EgressRuns.Add(run); + try + { + await db.SaveChangesAsync(ct); + } + catch (DbUpdateException ex) when (ex.InnerException is PostgresException { SqlState: UniqueViolation, ConstraintName: ActiveWindowOutputConstraint }) + { + throw new EgressRunConflictException("Another egress run already holds this checking window and output type."); + } + return run.Id; + } + + public async Task GetRunAsync(Guid runId, CancellationToken ct) + { + var run = await db.EgressRuns.AsNoTracking().Include(r => r.Outputs).FirstOrDefaultAsync(r => r.Id == runId, ct); + if (run is null) return null; + return new EgressRunDto(run.Id, run.WindowId, run.Status, run.StartedById, run.StartedByName, run.StartedAtUtc, + run.PreprocessedAtUtc, run.ExportDate, run.TransferredAtUtc, run.TransferredByName, + run.FailureJson is null ? [] : JsonSerializer.Deserialize>(run.FailureJson, Json) ?? [], + run.TransferFailureReason, + run.Outputs.OrderBy(o => o.OutputType).Select(o => new EgressRunOutputDto(o.Id, o.OutputType, o.IsActive, + JsonSerializer.Deserialize>(o.RawRecordsJson, Json) ?? [], + o.SourceRecordCount, o.OutputRecordCount, o.FileName, o.Sha256)).ToList()); + } + + public async Task> ListRunsAsync(CancellationToken ct) + { + var rows = await db.EgressRuns.AsNoTracking() + .OrderByDescending(r => r.StartedAtUtc) + .Select(r => new + { + r.Id, r.WindowId, r.Status, r.StartedByName, r.StartedAtUtc, r.TransferredAtUtc, + WindowTitle = db.CheckingWindows.Where(w => w.Id == r.WindowId).Select(w => w.Title).FirstOrDefault() ?? string.Empty, + Types = r.Outputs.OrderBy(o => o.OutputType).Select(o => o.OutputType).ToList() + }) + .ToListAsync(ct); + return rows.Select(r => new EgressRunListItem(r.Id, r.WindowId, r.WindowTitle, r.Status, r.Types, r.StartedByName, r.StartedAtUtc, r.TransferredAtUtc)).ToList(); + } + + public async Task TrySetStatusAsync(Guid runId, EgressRunStatus from, EgressRunStatus to, CancellationToken ct) => + await db.EgressRuns.Where(r => r.Id == runId && r.Status == from) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, to), ct) == 1; + + public Task MarkPreprocessingFailedAsync(Guid runId, EgressRunStatus expectedStatus, IReadOnlyList failures, CancellationToken ct) => + db.ExecuteInTransactionAsync(async () => + { + var rows = await db.EgressRuns.Where(r => r.Id == runId && r.Status == expectedStatus).ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, EgressRunStatus.PreprocessingFailed) + .SetProperty(r => r.FailureJson, JsonSerializer.Serialize(failures, Json)), ct); + if (rows > 0) + await db.EgressRunOutputs.Where(o => o.RunId == runId).ExecuteUpdateAsync(s => s.SetProperty(o => o.IsActive, false), ct); + return rows; + }, ct); + + public Task SavePreprocessedAsync(Guid runId, EgressRunStatus expectedStatus, IReadOnlyList newLearners, IReadOnlyList removeLearners, + DateOnly exportDate, IReadOnlyDictionary fileNames, CancellationToken ct) => + db.ExecuteInTransactionAsync(async () => + { + // S2: EnableRetryOnFailure re-runs this whole delegate on a transient fault. Without + // clearing first, a prior attempt's AddRange calls stay tracked as Added (they were + // never persisted, so ExecuteDeleteAsync below — a bulk operation that bypasses the + // tracker — cannot remove them), and the retry's own AddRange would save both sets: + // 2N learner rows for an N-row run, which is what ends up in the CSV. + db.ChangeTracker.Clear(); + + // M4: guard first — if the run moved on (e.g. Abandoned) while preprocessing ran, + // nothing below must be written. + var rows = await db.EgressRuns.Where(r => r.Id == runId && r.Status == expectedStatus).ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, EgressRunStatus.Preprocessed) + .SetProperty(r => r.PreprocessedAtUtc, DateTime.UtcNow) + .SetProperty(r => r.ExportDate, exportDate) + .SetProperty(r => r.FailureJson, (string?)null), ct); + if (rows == 0) return 0; + + await db.EgressNewLearners.Where(x => x.RunId == runId).ExecuteDeleteAsync(ct); + await db.EgressRemoveLearners.Where(x => x.RunId == runId).ExecuteDeleteAsync(ct); + db.EgressNewLearners.AddRange(newLearners.Select(r => new EgressNewLearner + { + Id = Guid.NewGuid(), RunId = runId, ChangeRequestId = r.ChangeRequestId, TicketId = r.TicketId, ReferenceNumber = r.ReferenceNumber, + CorrectionId = r.CorrectionId, CorrectionType = r.CorrectionType, KeyStage = r.KeyStage, LocalAuthority = r.LocalAuthority, + EstablishmentNumber = r.EstablishmentNumber, Surname = r.Surname, Forename = r.Forename, Sex = r.Sex, + DateOfBirth = r.DateOfBirth, AdmissionDate = r.AdmissionDate, Postcode = r.Postcode, CycleYear = r.CycleYear, CycleMonth = r.CycleMonth, + SchoolUrn = r.SchoolUrn, Uln = r.Uln, Upn = r.Upn, LearnerId = r.LearnerId, YearGroup = r.YearGroup + })); + db.EgressRemoveLearners.AddRange(removeLearners.Select(r => new EgressRemoveLearner + { + Id = Guid.NewGuid(), RunId = runId, ChangeRequestId = r.ChangeRequestId, TicketId = r.TicketId, ReferenceNumber = r.ReferenceNumber, + CorrectionId = r.CorrectionId, CorrectionType = r.CorrectionType, CorrectionReason = r.CorrectionReason, KeyStage = r.KeyStage, + EstablishmentNumber = r.EstablishmentNumber, Surname = r.Surname, Forename = r.Forename, Sex = r.Sex, DateOfBirth = r.DateOfBirth, + CycleYear = r.CycleYear, CycleMonth = r.CycleMonth, LocalAuthority = r.LocalAuthority, LearnerId = r.LearnerId, + YearGroup = r.YearGroup, RemovalYear0 = r.RemovalYear0, RemovalYear1 = r.RemovalYear1, RemovalYear2 = r.RemovalYear2 + })); + await db.SaveChangesAsync(ct); + + foreach (var (type, name) in fileNames) + { + var count = type == EgressOutputType.NewLearners ? newLearners.Count : removeLearners.Count; + await db.EgressRunOutputs.Where(o => o.RunId == runId && o.OutputType == type) + .ExecuteUpdateAsync(s => s.SetProperty(o => o.FileName, name).SetProperty(o => o.OutputRecordCount, count), ct); + } + return rows; + }, ct); + + public async Task> GetNewLearnersAsync(Guid runId, CancellationToken ct) => + await db.EgressNewLearners.AsNoTracking().Where(x => x.RunId == runId).OrderBy(x => x.Surname).ThenBy(x => x.Forename).ThenBy(x => x.ReferenceNumber) + .Select(r => new NewLearnerRow(r.CorrectionId, r.CorrectionType, r.KeyStage, r.LocalAuthority, r.EstablishmentNumber, r.Surname, + r.Forename, r.Sex, r.DateOfBirth, r.AdmissionDate, r.Postcode, r.CycleYear, r.CycleMonth, r.SchoolUrn, r.Uln, r.Upn, r.LearnerId, + r.YearGroup, r.ChangeRequestId, r.TicketId, r.ReferenceNumber)) + .ToListAsync(ct); + + public async Task> GetRemoveLearnersAsync(Guid runId, CancellationToken ct) => + await db.EgressRemoveLearners.AsNoTracking().Where(x => x.RunId == runId).OrderBy(x => x.Surname).ThenBy(x => x.Forename).ThenBy(x => x.ReferenceNumber) + .Select(r => new RemoveLearnerRow(r.CorrectionId, r.CorrectionType, r.CorrectionReason, r.KeyStage, r.EstablishmentNumber, r.Surname, r.Forename, + r.Sex, r.DateOfBirth, r.CycleYear, r.CycleMonth, r.LocalAuthority, r.LearnerId, r.ChangeRequestId, r.TicketId, r.ReferenceNumber) + { + YearGroup = r.YearGroup, RemovalYear0 = r.RemovalYear0, RemovalYear1 = r.RemovalYear1, RemovalYear2 = r.RemovalYear2 + }) + .ToListAsync(ct); + + public async Task<(EgressOutputType OutputType, EgressBlocker Blocker)?> TryReactivateAsync(Guid runId, CancellationToken ct) + { + try + { + await db.EgressRunOutputs.Where(o => o.RunId == runId).ExecuteUpdateAsync(s => s.SetProperty(o => o.IsActive, true), ct); + return null; + } + catch (DbUpdateException ex) when (ex.InnerException is PostgresException { SqlState: UniqueViolation }) + { + return await BlockerForRunPairsAsync(runId, ct); + } + catch (PostgresException ex) when (ex.SqlState == UniqueViolation) + { + return await BlockerForRunPairsAsync(runId, ct); + } + } + + private async Task<(EgressOutputType, EgressBlocker)?> BlockerForRunPairsAsync(Guid runId, CancellationToken ct) + { + var pairs = await db.EgressRunOutputs.AsNoTracking().Where(o => o.RunId == runId).Select(o => new { o.WindowId, o.OutputType }).ToListAsync(ct); + foreach (var pair in pairs) + { + var blocker = await FindBlockerAsync(pair.WindowId, pair.OutputType, ct); + if (blocker is not null && blocker.RunId != runId) return (pair.OutputType, blocker); + } + return null; + } + + public Task MarkTransferredAsync(Guid runId, EgressRunStatus expectedStatus, EgressTransferAudit audit, DateTime transferredAtUtc, CancellationToken ct) => + db.ExecuteInTransactionAsync(async () => + { + // S2: same retry-duplication hazard as SavePreprocessedAsync — a stale tracked + // AuditEntry from a prior attempt would otherwise be saved a second time alongside + // this attempt's. + db.ChangeTracker.Clear(); + + // M4: guard first — if the run moved on (e.g. Abandoned mid-transfer) since the + // Transferring flip, no file/output row is touched and no Succeeded audit is written. + var rows = await db.EgressRuns.Where(r => r.Id == runId && r.Status == expectedStatus).ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, EgressRunStatus.Transferred) + .SetProperty(r => r.TransferredAtUtc, transferredAtUtc) + .SetProperty(r => r.TransferredByName, audit.UserName) + .SetProperty(r => r.TransferFailureReason, (string?)null), ct); + if (rows == 0) return 0; + + foreach (var (type, file) in audit.Files) + { + await db.EgressRunOutputs.Where(o => o.RunId == runId && o.OutputType == type) + .ExecuteUpdateAsync(s => s.SetProperty(o => o.Sha256, file.Sha256).SetProperty(o => o.FileName, file.FileName), ct); + } + var run = await db.EgressRuns.AsNoTracking().FirstAsync(r => r.Id == runId, ct); + db.AuditEntries.Add(new AuditEntry + { + EntityType = "EgressRun", + EntityId = runId.ToString(), + Action = "Transfer", + Timestamp = transferredAtUtc, + UserId = audit.UserId, + NewValues = JsonSerializer.Serialize(new + { + Outcome = "Succeeded", + run.WindowId, + OutputTypes = audit.Files.Keys.Select(k => k.ToString()).Order().ToList(), + Files = audit.Files.Select(f => new { OutputType = f.Key.ToString(), f.Value.FileName, f.Value.Records, f.Value.Sha256 }).ToList(), + audit.TargetContainer, + TransferredBy = audit.UserName, + TransferredAtUtc = transferredAtUtc + }, Json) + }); + await db.SaveChangesAsync(ct); + return rows; + }, ct); + + public Task MarkTransferFailedAsync(Guid runId, EgressRunStatus expectedStatus, string reason, string userId, CancellationToken ct) => + db.ExecuteInTransactionAsync(async () => + { + // S2: same retry-duplication hazard as SavePreprocessedAsync and MarkTransferredAsync + // — a stale tracked AuditEntry from a prior attempt would otherwise be saved a second + // time alongside this attempt's, leaving two TransferFailed audit rows for one failure. + db.ChangeTracker.Clear(); + + var clipped = reason.Length > 1000 ? reason[..1000] : reason; + // M4: guard first — a run that has already moved on (e.g. Abandoned) must not be + // overwritten to TransferFailed, and must not gain a spurious audit row for it. + var rows = await db.EgressRuns.Where(r => r.Id == runId && r.Status == expectedStatus).ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, EgressRunStatus.TransferFailed) + .SetProperty(r => r.TransferFailureReason, clipped), ct); + if (rows == 0) return 0; + + await db.EgressRunOutputs.Where(o => o.RunId == runId).ExecuteUpdateAsync(s => s.SetProperty(o => o.IsActive, false), ct); + db.AuditEntries.Add(new AuditEntry + { + EntityType = "EgressRun", EntityId = runId.ToString(), Action = "TransferFailed", + Timestamp = DateTime.UtcNow, UserId = userId, + NewValues = JsonSerializer.Serialize(new { Outcome = "Failed", Reason = clipped }, Json) + }); + await db.SaveChangesAsync(ct); + return rows; + }, ct); + + public Task AbandonAsync(Guid runId, CancellationToken ct) => + db.ExecuteInTransactionAsync(async () => + { + // Excludes only Transferred — Preprocessing and Transferring are both admitted, so a + // run stuck there (a pod restart mid-pipeline) can always be released (M4). + var rows = await db.EgressRuns.Where(r => r.Id == runId && r.Status != EgressRunStatus.Transferred) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, EgressRunStatus.Abandoned), ct); + if (rows > 0) + await db.EgressRunOutputs.Where(o => o.RunId == runId).ExecuteUpdateAsync(s => s.SetProperty(o => o.IsActive, false), ct); + return rows; + }, ct); +} diff --git a/src/DfE.CheckPerformanceData.Persistence/Repositories/RequestRepository.cs b/src/DfE.CheckPerformanceData.Persistence/Repositories/RequestRepository.cs index 184a6093b..6c3808a00 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Repositories/RequestRepository.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Repositories/RequestRepository.cs @@ -124,6 +124,7 @@ await db.ChangeRequests .SetProperty(r => r.RequestType, data.RequestType) .SetProperty(r => r.RequestTypeDescription, data.RequestTypeDescription) .SetProperty(r => r.AmendmentType, data.AmendmentType) + .SetProperty(r => r.OrganisationLaestab, data.OrganisationLaestab) .SetProperty(r => r.CheckingExerciseId, data.CheckingExerciseId)); } else @@ -147,7 +148,8 @@ await db.ChangeRequests Status = data.Status, RequestType = data.RequestType, RequestTypeDescription = data.RequestTypeDescription, - AmendmentType = data.AmendmentType + AmendmentType = data.AmendmentType, + OrganisationLaestab = data.OrganisationLaestab }); await db.SaveChangesAsync(); } @@ -181,6 +183,7 @@ await db.ChangeRequests .SetProperty(r => r.RequestType, data.RequestType) .SetProperty(r => r.RequestTypeDescription, data.RequestTypeDescription) .SetProperty(r => r.AmendmentType, data.AmendmentType) + .SetProperty(r => r.OrganisationLaestab, data.OrganisationLaestab) .SetProperty(r => r.CheckingExerciseId, data.CheckingExerciseId)); return draftExistingId; } @@ -204,7 +207,8 @@ await db.ChangeRequests.AddAsync(new ChangeRequest Status = data.Status, RequestType = data.RequestType, RequestTypeDescription = data.RequestTypeDescription, - AmendmentType = data.AmendmentType + AmendmentType = data.AmendmentType, + OrganisationLaestab = data.OrganisationLaestab }); await db.SaveChangesAsync(); return newId; diff --git a/src/DfE.CheckPerformanceData.Persistence/Seeding/SeedCheckingWindows.cs b/src/DfE.CheckPerformanceData.Persistence/Seeding/SeedCheckingWindows.cs index ccbdc1e22..ce83f325e 100644 --- a/src/DfE.CheckPerformanceData.Persistence/Seeding/SeedCheckingWindows.cs +++ b/src/DfE.CheckPerformanceData.Persistence/Seeding/SeedCheckingWindows.cs @@ -62,6 +62,12 @@ private static List ExercisesFor( public static async Task ExecuteSeed(IPortalDbContext dbContext, Guid openKs4WindowId, Guid closedKs4WindowId, Guid post16WindowId, Guid closedPupilDataPost16WindowId) { + // Egress runs first: egress_runs → CheckingWindows is a RESTRICT foreign key (an egress + // is an audit record and must never vanish because a window was deleted), so a run left + // behind — an E2E cleanup that failed part-way is enough — made the window wipe below + // throw, the host terminated before it listened, and the review app's new pod never became + // Ready while the old one kept serving. Outputs and learner rows cascade from the run. + await dbContext.EgressRuns.ExecuteDeleteAsync(); await dbContext.ChangeRequests.ExecuteDeleteAsync(); await dbContext.CheckingWindows.ExecuteDeleteAsync(); diff --git a/src/DfE.CheckPerformanceData.Web/Admin/Nav/AdminNavKeys.cs b/src/DfE.CheckPerformanceData.Web/Admin/Nav/AdminNavKeys.cs index d1414302b..63958a1a2 100644 --- a/src/DfE.CheckPerformanceData.Web/Admin/Nav/AdminNavKeys.cs +++ b/src/DfE.CheckPerformanceData.Web/Admin/Nav/AdminNavKeys.cs @@ -68,4 +68,10 @@ public static class AdminNavKeys // form + POST action for pumping plausible search events + feedback messages into the // sink so the search-analytics dashboard has something to show during demos. public const string SeedSampleSearchData = "seed-sample-search-data"; + + // Data egress to LDS (AB#294553). EgressGroup is a container (not in AllSections); Egress is + // the tile's own key and the [RequireAdminSection] gate on EgressController, so it IS in + // DefaultAdminAccessSeeder.AllSections. + public const string EgressGroup = "egress-group"; + public const string Egress = "egress"; } diff --git a/src/DfE.CheckPerformanceData.Web/Admin/Nav/EgressGroupNavEntry.cs b/src/DfE.CheckPerformanceData.Web/Admin/Nav/EgressGroupNavEntry.cs new file mode 100644 index 000000000..54f5924ee --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Admin/Nav/EgressGroupNavEntry.cs @@ -0,0 +1,12 @@ +namespace DfE.CheckPerformanceData.Web.Admin.Nav; + +public sealed record EgressGroupNavEntry : IAdminNavEntry +{ + public string Key => AdminNavKeys.EgressGroup; + public string? ParentKey => null; + public string Title => "Data egress"; + public string Description => "Pull approved amendment decisions from Zendesk, prepare them to the LDS specification and transfer them."; + public string Url => string.Empty; + public bool Enabled => true; + public int Order => 35; +} diff --git a/src/DfE.CheckPerformanceData.Web/Admin/Nav/StartEgressNavEntry.cs b/src/DfE.CheckPerformanceData.Web/Admin/Nav/StartEgressNavEntry.cs new file mode 100644 index 000000000..16371d1c0 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Admin/Nav/StartEgressNavEntry.cs @@ -0,0 +1,12 @@ +namespace DfE.CheckPerformanceData.Web.Admin.Nav; + +public sealed record StartEgressNavEntry : IAdminNavEntry +{ + public string Key => AdminNavKeys.Egress; + public string? ParentKey => AdminNavKeys.EgressGroup; + public string Title => "Start a new egress"; + public string Description => "Choose a checking window and output types, pull the decisions and transfer the files to LDS. Saved runs are listed here too."; + public string Url => "/admin/egress"; + public bool Enabled => true; + public int Order => 10; +} diff --git a/src/DfE.CheckPerformanceData.Web/Controllers/DevEgressController.cs b/src/DfE.CheckPerformanceData.Web/Controllers/DevEgressController.cs new file mode 100644 index 000000000..ce959d28e --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Controllers/DevEgressController.cs @@ -0,0 +1,171 @@ +using System.Text.Json; +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.Journey; +using DfE.CheckPerformanceData.Application.RequestSubmission; +using DfE.CheckPerformanceData.Application.Settings; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.Persistence.Contexts; +using DfE.CheckPerformanceData.Persistence.Entities; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +namespace DfE.CheckPerformanceData.Web.Controllers; + +// Dev-only fixture for the LDS egress (AB#294553): stages committed requests with a Zendesk ticket +// id, a journey blob and a dev-outbox ticket carrying a chosen decision, so the whole egress can be +// walked locally and by the E2E suite without a worker or a real Zendesk. 404 in Production and +// wherever Dev:ToolsEnabled is off, like DevPipelineController; [AllowAnonymous] for the same +// reason — E2E callers arrive with no cookie and it only touches the local dev database. +[AllowAnonymous] +public sealed class DevEgressController( + IConfiguration configuration, + IPortalDbContext dbContext, + IRequestStateBlobClient journeys, + IEgressBlobClient egressBlobs, + IHostEnvironment? hostEnvironment = null) : Controller +{ + private const string ReferencePrefix = "DEV-EGRESS-"; + private static long _nextTicketId = 900_000_000 + DateTimeOffset.UtcNow.ToUnixTimeSeconds() % 100_000_000; + + private bool IsAllowed => + configuration.GetValue(SettingKeys.DevToolsEnabled) && hostEnvironment?.IsProduction() != true; + + [HttpPost("dev/egress/seed")] + public async Task Seed(Guid windowId, string outputType, string decision, int count, string laestab, long urn, string? reason, CancellationToken cancellationToken) + { + if (!IsAllowed) return NotFound(); + if (!Enum.TryParse(outputType, ignoreCase: true, out var type)) return BadRequest("outputType must be NewLearners or RemoveLearners"); + count = Math.Clamp(count, 1, 50); + + var window = await dbContext.CheckingWindows.AsNoTracking().FirstOrDefaultAsync(w => w.Id == windowId, cancellationToken); + if (window is null) return BadRequest("Unknown window"); + + var references = new List(); + var ticketIds = new List(); + for (var i = 0; i < count; i++) + { + var reference = $"{ReferencePrefix}{Guid.NewGuid():N}"[..(ReferencePrefix.Length + 8)]; + var ticketId = decision == "none" ? (long?)null : Interlocked.Increment(ref _nextTicketId); + var pupilId = Guid.NewGuid(); + var surname = $"Egress{i + 1}"; + var forename = type == EgressOutputType.NewLearners ? "Newbie" : "Removal"; + + dbContext.ChangeRequests.Add(new ChangeRequest + { + Id = Guid.NewGuid(), WindowId = windowId, OrganisationUrn = urn, OrganisationLaestab = laestab, + PupilId = pupilId, PupilUpn = "A860407000011", PupilFirstname = forename, PupilSurname = surname, + Submitted = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified), SubmittedById = Guid.NewGuid(), + SubmittedByName = "Dev Egress Harness", SubmittedByEmail = "dev.egress@education.gov.uk", + Status = RequestStatus.SubmittedCommitted, ReferenceNumber = reference, RequestType = RequestType.Amendment, + RequestTypeDescription = type == EgressOutputType.NewLearners ? "Add" : $"Remove - {reason ?? "pupil-died"}", + AmendmentType = EgressOutputTypes.WhatToChangeFor(type), CrmId = ticketId?.ToString() + }); + + var journey = new RequestState + { + SelectedWhatToChange = EgressOutputTypes.WhatToChangeFor(type), + ReferenceNumber = reference, + SelectedPupil = new PupilDto + { + Id = pupilId, Firstname = forename, Surname = surname, Sex = "F", DateOfBirth = "07/09/2010", Age = 15, + Cypmd_Id = type == EgressOutputType.NewLearners ? "" : $"50{i:D4}", Identifier = "A860407000011", + MatchRef = type == EgressOutputType.NewLearners ? 0 : 555000 + i, + Laestab = type == EgressOutputType.NewLearners ? "" : laestab + } + }; + if (type == EgressOutputType.NewLearners) + { + journey.QuestionAnswers["first-name"] = new QuestionAnswer { TextValue = forename }; + journey.QuestionAnswers["last-name"] = new QuestionAnswer { TextValue = surname }; + journey.QuestionAnswers["date-of-birth"] = new QuestionAnswer { DateValue = new DateAnswer { Day = 7, Month = 9, Year = 2010 } }; + journey.QuestionAnswers["sex"] = new QuestionAnswer { TextValue = "F" }; + journey.QuestionAnswers["upn"] = new QuestionAnswer { TextValue = "A860407000011" }; + journey.QuestionAnswers["admission-date"] = new QuestionAnswer { DateValue = new DateAnswer { Day = 4, Month = 9, Year = 2018 } }; + journey.QuestionAnswers["year-group"] = new QuestionAnswer { TextValue = window.CheckingWindowType == CheckingWindowType.KS2 ? "6" : "10" }; + journey.QuestionAnswers["sen-status"] = new QuestionAnswer { TextValue = "N" }; + } + else + { + journey.QuestionAnswers["reason"] = new QuestionAnswer { TextValue = reason ?? "pupil-died" }; + journey.QuestionAnswers["date-removed-from-roll"] = new QuestionAnswer { DateValue = new DateAnswer { Day = 1, Month = 6, Year = 2026 } }; + } + await journeys.SaveAsync(windowId, reference, journey); + + if (ticketId is { } id) + { + var subject = decision switch + { + "auto_approved" => $"CPMD Auto-Approved: Seed ({reference})", + "auto_rejected" => $"CPMD Auto-Rejected: Seed ({reference})", + _ => $"CPMD Requires Scrutiny: Seed ({reference})" + }; + var fieldId = configuration.GetValue("ZendeskTicketFields:DecisionStatusId") is > 0 and var configured + ? configured : DevOutboxEgressTicketSource.WellKnownDecisionFieldId; + var raw = new CreateTicketRequestDto + { + Ticket = new CreateTicketDto { Subject = subject, Status = "open", Priority = "normal", Type = "task", + CustomFields = [new CustomFieldDto { Id = fieldId, Value = decision }] } + }; + dbContext.DevZendeskTickets.Add(new DevZendeskTicket + { + Id = Guid.NewGuid(), CreatedAtUtc = DateTime.UtcNow, ReferenceNumber = reference, Subject = subject, + Priority = "normal", Status = "open", TicketId = id, RawJson = JsonSerializer.Serialize(raw) + }); + ticketIds.Add(id); + } + references.Add(reference); + } + await dbContext.SaveChangesAsync(cancellationToken); + return Json(new { references, ticketIds }); + } + + [HttpPost("dev/egress/cleanup")] + public async Task Cleanup(Guid windowId, CancellationToken cancellationToken) + { + if (!IsAllowed) return NotFound(); + + // Nit: delete only blobs owned by a run this cleanup is actually removing, matched by + // egressRunId metadata (the M1 sweep helper) — plain by-filename deletion could remove + // another run's blob if two windows' files collided on name (Q3). + var owners = await dbContext.EgressRunOutputs.AsNoTracking() + .Where(o => o.WindowId == windowId && o.FileName != null) + .Select(o => new { o.RunId, FileName = o.FileName! }).Distinct().ToListAsync(cancellationToken); + // Best effort, per blob: a reset must reset. This used to throw straight out of the action, + // so on an environment whose egress account was unreachable every cleanup answered 500 and + // left the runs behind — and a leftover run is what stopped the dev seeder (and so the pod) + // starting on the next deploy. A failed delete is reported in the response, not fatal. + var blobs = 0; + var blobErrors = 0; + if (egressBlobs.IsConfigured) + { + foreach (var owner in owners) + { + try + { + if (await egressBlobs.DeleteIfOwnedByRunAsync(owner.FileName, owner.RunId, cancellationToken)) blobs++; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + blobErrors++; + } + } + } + + var runs = await dbContext.EgressRuns.Where(r => r.WindowId == windowId).ExecuteDeleteAsync(cancellationToken); + + var devRequests = await dbContext.ChangeRequests.AsNoTracking() + .Where(r => r.WindowId == windowId && EF.Functions.Like(r.ReferenceNumber, ReferencePrefix + "%")) + .Select(r => new { r.Id, r.ReferenceNumber }).ToListAsync(cancellationToken); + foreach (var r in devRequests) await journeys.DeleteAsync(windowId, r.ReferenceNumber); + var refs = devRequests.Select(r => r.ReferenceNumber).ToList(); + await dbContext.DevZendeskTickets.Where(t => refs.Contains(t.ReferenceNumber)).ExecuteDeleteAsync(cancellationToken); + var ids = devRequests.Select(r => r.Id).ToList(); + var requests = ids.Count == 0 ? 0 : await dbContext.ChangeRequests.Where(r => ids.Contains(r.Id)).ExecuteDeleteAsync(cancellationToken); + + return Json(new { runs, requests, blobs, blobErrors }); + } +} diff --git a/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressController.cs b/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressController.cs new file mode 100644 index 000000000..5e0f21ec8 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressController.cs @@ -0,0 +1,321 @@ +using System.Runtime.CompilerServices; +using DfE.CheckPerformanceData.Application.CurrentUser; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.Web.Admin; +using DfE.CheckPerformanceData.Web.Admin.Nav; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ModelBinding; + +namespace DfE.CheckPerformanceData.Web.Controllers.Egress; + +/// +/// Data egress to LDS (AB#294553): Pull → Results → Preprocessing → Summary → Complete. A run is +/// persisted at pull time, so every later page is a GET on the run id and "save" is just leaving; +/// Resume sends the user to whichever page the run's status implies. Preprocessing streams progress +/// over server-sent events with a plain POST fallback driving the same enumerable +/// (ValidateWindowController pattern). +/// +[RequireAdminSection(AdminNavKeys.Egress)] +[Route("admin/egress")] +public sealed class EgressController( + IEgressRunService runs, + IEgressPreprocessor preprocessor, + IEgressTransferService transfer, + IEgressBlobClient blobs, + IWindowService windows, + ICurrentUserService currentUser) : Controller +{ + public const string BannerKey = "EgressBanner"; + public const string TransferErrorKey = "EgressTransferError"; + + private static readonly EgressRunStatus[] SavedStatuses = + [EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, EgressRunStatus.PreprocessingFailed, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, EgressRunStatus.TransferFailed]; + + private EgressActor Actor => new(Guid.TryParse(currentUser.UserId, out var id) ? id : Guid.Empty, currentUser.DisplayName, currentUser.Email); + + // ---------------------------------------------------------------- Pull + + [HttpGet("")] + public async Task Index(CancellationToken cancellationToken) => + View("Index", await PullModelAsync(new PullForm(), null, null, [], null, cancellationToken)); + + [HttpPost("")] + [ValidateAntiForgeryToken] + public async Task Start(PullForm form, CancellationToken cancellationToken) + { + var windowError = form.WindowId is null || form.WindowId == Guid.Empty ? "Select a checking window" : null; + // S9: an unbindable OutputTypes value (e.g. OutputTypes=garbage) previously bound as + // default(EgressOutputType) — NewLearners — with a ModelState error nobody read, so the + // request silently proceeded as if NewLearners had been ticked. Checking the field's own + // validation state catches that case without mislabelling an unrelated WindowId failure + // (already handled above) as an output-types error. + var typesError = form.OutputTypes.Count == 0 + || ModelState.GetValidationState(nameof(PullForm.OutputTypes)) == ModelValidationState.Invalid + ? "Select at least one output type" : null; + if (windowError is not null || typesError is not null) + return View("Index", await PullModelAsync(form, windowError, typesError, [], null, cancellationToken)); + + var result = await runs.StartAsync(form.WindowId!.Value, form.OutputTypes.Distinct().ToList(), Actor, cancellationToken); + return result switch + { + EgressStartResult.Started started => RedirectToAction(nameof(Results), new { id = started.RunId }), + EgressStartResult.Refused refused => View("Index", await PullModelAsync(form, null, null, refused.Blockers.Select(Describe).ToList(), null, cancellationToken)), + EgressStartResult.WindowNotFound => View("Index", await PullModelAsync(form, "Select a checking window", null, [], null, cancellationToken)), + EgressStartResult.PullFailed failed => View("Index", await PullModelAsync(form, null, null, [], failed.Reason, cancellationToken)), + _ => throw new InvalidOperationException("Unknown start result") + }; + } + + // FLAGGED copy: the two refusal sentences (AB#294553 "Concurrency"). + private static string Describe((EgressOutputType OutputType, EgressBlocker Blocker) refusal) + { + var (type, b) = refusal; + var label = EgressOutputTypes.Label(type); + return b.Status == EgressRunStatus.Transferred + ? $"{label} for this checking window has already been transferred to LDS by {b.TransferredByName ?? b.StartedByName} on {b.TransferredAtUtc:d MMMM yyyy 'at' HH:mm} UTC. It cannot be sent again." + : $"{label} for this checking window is already being processed by {b.StartedByName}, started {b.StartedAtUtc:d MMMM yyyy 'at' HH:mm} UTC. Wait for that run to finish or be abandoned."; + } + + private async Task PullModelAsync(PullForm form, string? windowError, string? typesError, IReadOnlyList refusals, string? pullError, CancellationToken ct) + { + var all = (await windows.GetAllDataAsync(ct))?.Windows ?? []; + var list = await runs.ListAsync(ct); + return new PullViewModel + { + Windows = all.OrderByDescending(w => w.StartDate).Select(w => new WindowChoice(w.Id, $"{w.Title} ({EgressOutputTypes.StageToken(w.CheckingWindowType)})")).ToList(), + OutputTypes = EgressOutputTypes.All, + SelectedWindowId = form.WindowId, + SelectedOutputTypes = form.OutputTypes, + WindowError = windowError, + OutputTypesError = typesError, + Refusals = refusals, + PullError = pullError, + SavedRuns = list.Where(r => SavedStatuses.Contains(r.Status)).ToList(), + CompletedRuns = list.Where(r => r.Status == EgressRunStatus.Transferred).ToList(), + Banner = TempData[BannerKey] as string, + StorageNotConfigured = !blobs.IsConfigured + }; + } + + // ---------------------------------------------------------------- Resume + + [HttpGet("runs/{id:guid}")] + public async Task Resume(Guid id, CancellationToken cancellationToken) + { + var run = await runs.GetAsync(id, cancellationToken); + if (run is null) return NotFound(); + return run.Status switch + { + EgressRunStatus.Pulled => RedirectToAction(nameof(Results), new { id }), + EgressRunStatus.Preprocessing => RedirectToAction(nameof(Preprocessing), new { id }), + EgressRunStatus.PreprocessingFailed => RedirectToAction(nameof(Failed), new { id }), + EgressRunStatus.Preprocessed or EgressRunStatus.TransferFailed or EgressRunStatus.Transferring => RedirectToAction(nameof(Summary), new { id }), + EgressRunStatus.Transferred => RedirectToAction(nameof(Complete), new { id }), + _ => AbandonedHome() + }; + } + + // Second-pass nit: Transfer's "abandoned while the transfer was in progress" Failed result + // (EgressTransferService.FailAsync) sets TransferErrorKey then redirects to Summary — but + // Summary never renders for an Abandoned run, it redirects straight here instead, so that + // TempData entry is never read on this request. Left alone it would surface incorrectly as a + // transfer error on the next unrelated run's Summary page. This is the one place every + // Abandoned run lands, so clearing it here (rather than at every caller of Home) closes the + // leak without guessing which Failed result caused it. + private IActionResult AbandonedHome() + { + TempData.Remove(TransferErrorKey); + return Home("That egress run was abandoned. Start a new one if it is still needed."); + } + + private IActionResult Home(string banner) + { + TempData[BannerKey] = banner; + return RedirectToAction(nameof(Index)); + } + + // ---------------------------------------------------------------- Results + + [HttpGet("runs/{id:guid}/results")] + public async Task Results(Guid id, CancellationToken cancellationToken) + { + var page = await PageAsync(id, cancellationToken); + if (page is null) return NotFound(); + if (page.Run.Status is not (EgressRunStatus.Pulled or EgressRunStatus.PreprocessingFailed or EgressRunStatus.Preprocessed)) + return RedirectToAction(nameof(Resume), new { id }); + return View("Results", page); + } + + // ---------------------------------------------------------------- Preprocessing + + [HttpGet("runs/{id:guid}/preprocessing")] + public async Task Preprocessing(Guid id, CancellationToken cancellationToken) + { + var page = await PageAsync(id, cancellationToken); + if (page is null) return NotFound(); + // Nit: a Transferred or Abandoned run has nothing left to preprocess — without this guard + // the page rendered a live "Run preprocessing" button for a run that is already finished. + // PreprocessingFailed is the same story: the POST already refuses it (EgressPreprocessor), + // so the GET must not keep offering the button either. + if (page.Run.Status is EgressRunStatus.Transferred or EgressRunStatus.Abandoned or EgressRunStatus.PreprocessingFailed) + return RedirectToAction(nameof(Resume), new { id }); + if (page.Run.Outputs.Sum(o => o.SourceRecordCount) == 0) + return RedirectToAction(nameof(Results), new { id }); + return View("Preprocessing", page with { StreamUrl = Url.Action(nameof(PreprocessingStream), new { id }) }); + } + + // EventSource can only GET, so the pipeline runs here when JS drives it. + [HttpGet("runs/{id:guid}/preprocessing/stream")] + public IResult PreprocessingStream(Guid id, CancellationToken cancellationToken) => + // Fully qualified: this controller has its own action named Results, which shadows + // Microsoft.AspNetCore.Http.Results at an unqualified call site. + Microsoft.AspNetCore.Http.Results.ServerSentEvents(RunWithNextUrl(id, cancellationToken), eventType: "progress"); + + // No-JS fallback: run to completion, then land where the outcome says. + [HttpPost("runs/{id:guid}/preprocessing")] + [ValidateAntiForgeryToken] + public async Task PreprocessingRun(Guid id, CancellationToken cancellationToken) + { + EgressProgress? last = null; + await foreach (var progress in preprocessor.RunAsync(id, cancellationToken)) last = progress; + return last?.FinalStatus switch + { + EgressRunStatus.Preprocessed => RedirectToAction(nameof(Summary), new { id }), + EgressRunStatus.PreprocessingFailed => RedirectToAction(nameof(Failed), new { id }), + _ => Home(last?.Message ?? "Preprocessing did not complete.") + }; + } + + public sealed record ProgressEvent(int Step, int TotalSteps, string StepName, string State, int RecordsIn, int RecordsOut, int FailureCount, bool IsComplete, bool IsError, string Message, string? NextUrl); + + private async IAsyncEnumerable RunWithNextUrl(Guid id, [EnumeratorCancellation] CancellationToken ct) + { + await foreach (var p in preprocessor.RunAsync(id, ct)) + { + var next = p.FinalStatus switch + { + EgressRunStatus.Preprocessed => Url.Action(nameof(Summary), new { id }), + EgressRunStatus.PreprocessingFailed => Url.Action(nameof(Failed), new { id }), + _ => p.IsComplete ? Url.Action(nameof(Index)) : null + }; + yield return new ProgressEvent(p.Step, p.TotalSteps, p.StepName, p.State, p.RecordsIn, p.RecordsOut, p.FailureCount, p.IsComplete, p.IsError, p.Message, next); + } + } + + [HttpGet("runs/{id:guid}/failed")] + public async Task Failed(Guid id, CancellationToken cancellationToken) + { + var page = await PageAsync(id, cancellationToken); + if (page is null) return NotFound(); + if (page.Run.Status != EgressRunStatus.PreprocessingFailed) return RedirectToAction(nameof(Resume), new { id }); + return View("Failed", page); + } + + // ---------------------------------------------------------------- Summary / preview / transfer + + [HttpGet("runs/{id:guid}/summary")] + public async Task Summary(Guid id, CancellationToken cancellationToken) + { + var page = await PageAsync(id, cancellationToken); + if (page is null) return NotFound(); + if (page.Run.Status is not (EgressRunStatus.Preprocessed or EgressRunStatus.TransferFailed or EgressRunStatus.Transferring)) + return RedirectToAction(nameof(Resume), new { id }); + return View("Summary", page with { TransferError = TempData[TransferErrorKey] as string ?? page.Run.TransferFailureReason }); + } + + [HttpGet("runs/{id:guid}/preview/{outputType}")] + public async Task Preview(Guid id, EgressOutputType outputType, CancellationToken cancellationToken) + { + var run = await runs.GetAsync(id, cancellationToken); + var output = run?.Outputs.FirstOrDefault(o => o.OutputType == outputType); + if (run is null || output?.FileName is null) return NotFound(); + var (headers, rows) = await transfer.GetPreviewAsync(id, outputType, cancellationToken); + return View("Preview", new PreviewViewModel + { + RunId = id, OutputType = outputType, FileName = output.FileName, + Headers = headers, Rows = rows + }); + } + + [HttpGet("runs/{id:guid}/download/{outputType}")] + public async Task Download(Guid id, EgressOutputType outputType, CancellationToken cancellationToken) + { + var run = await runs.GetAsync(id, cancellationToken); + var output = run?.Outputs.FirstOrDefault(o => o.OutputType == outputType); + if (run is null || output?.FileName is null) return NotFound(); + return File(await transfer.BuildFileAsync(id, outputType, cancellationToken), "text/csv; charset=utf-8", output.FileName); + } + + [HttpPost("runs/{id:guid}/transfer")] + [ValidateAntiForgeryToken] + public async Task Transfer(Guid id, CancellationToken cancellationToken) + { + var result = await transfer.TransferAsync(id, Actor, cancellationToken); + switch (result) + { + case EgressTransferResult.Transferred: + return RedirectToAction(nameof(Complete), new { id }); + case EgressTransferResult.Failed failed: + TempData[TransferErrorKey] = failed.Reason; + return RedirectToAction(nameof(Summary), new { id }); + // M3: never Complete for an empty approved set — Summary derives the same "nothing to + // transfer" state from the run's own saved row counts, so no TempData is needed here. + case EgressTransferResult.NothingToTransfer: + return RedirectToAction(nameof(Summary), new { id }); + case EgressTransferResult.Refused refused: + TempData[TransferErrorKey] = Describe((refused.OutputType, refused.Blocker)); + return RedirectToAction(nameof(Summary), new { id }); + // Nit: a missing run is its own outcome rather than a NotTransferable(Abandoned) guess. + case EgressTransferResult.NotFound: + return NotFound(); + default: + return RedirectToAction(nameof(Resume), new { id }); + } + } + + [HttpGet("runs/{id:guid}/complete")] + public async Task Complete(Guid id, CancellationToken cancellationToken) + { + var page = await PageAsync(id, cancellationToken); + if (page is null) return NotFound(); + if (page.Run.Status != EgressRunStatus.Transferred) return RedirectToAction(nameof(Resume), new { id }); + return View("Complete", page); + } + + [HttpPost("runs/{id:guid}/abandon")] + [ValidateAntiForgeryToken] + public async Task Abandon(Guid id, CancellationToken cancellationToken) + { + var result = await transfer.AbandonAsync(id, cancellationToken); + return result switch + { + // M1: a Transferring run's own files may have been swept — the banner must say so + // rather than always claiming nothing was transferred. + EgressAbandonResult.Abandoned { RemovedFiles.Count: > 0 } abandoned => + Home($"The egress run was abandoned. Removed from the target container: {string.Join(", ", abandoned.RemovedFiles)}."), + EgressAbandonResult.Abandoned => + Home("The egress run was abandoned. Nothing was transferred."), + EgressAbandonResult.AlreadyTransferred => + Home("That egress run has already been transferred to LDS and cannot be abandoned."), + _ => NotFound() + }; + } + + private async Task PageAsync(Guid id, CancellationToken ct) + { + var run = await runs.GetAsync(id, ct); + if (run is null) return null; + var window = await windows.GetByIdAsync(run.WindowId, ct); + return new RunPageViewModel + { + Run = run, + WindowTitle = window?.Title ?? "Unknown window", + WindowType = window?.CheckingWindowType ?? CheckingWindowType.KS4June, + TargetDescription = blobs.TargetDescription, + CurrentUserName = currentUser.DisplayName, + StorageNotConfigured = !blobs.IsConfigured + }; + } +} diff --git a/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressViewModels.cs b/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressViewModels.cs new file mode 100644 index 000000000..b551edc5b --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Controllers/Egress/EgressViewModels.cs @@ -0,0 +1,77 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Web.Controllers.Egress; + +/// The Pull form's posted fields. +public sealed class PullForm +{ + public Guid? WindowId { get; set; } + public List OutputTypes { get; set; } = []; +} + +public sealed record WindowChoice(Guid Id, string Label); + +public sealed class PullViewModel +{ + public required IReadOnlyList Windows { get; init; } + public required IReadOnlyList OutputTypes { get; init; } + public Guid? SelectedWindowId { get; init; } + public IReadOnlyList SelectedOutputTypes { get; init; } = []; + public string? WindowError { get; init; } + public string? OutputTypesError { get; init; } + /// One sentence per refused output type: who holds it and since when, or when it was transferred and by whom. + public IReadOnlyList Refusals { get; init; } = []; + public string? PullError { get; init; } + public IReadOnlyList SavedRuns { get; init; } = []; + public IReadOnlyList CompletedRuns { get; init; } = []; + public string? Banner { get; init; } + /// True when ConnectionStrings:EgressStorage is absent — the view warns up front that a transfer will refuse. + public bool StorageNotConfigured { get; init; } + public bool IsValid => WindowError is null && OutputTypesError is null && Refusals.Count == 0 && PullError is null; +} + +public sealed record RunPageViewModel +{ + public required EgressRunDto Run { get; init; } + public required string WindowTitle { get; init; } + public required CheckingWindowType WindowType { get; init; } + public string TargetDescription { get; init; } = string.Empty; + public string? TransferError { get; init; } + public string? StreamUrl { get; init; } + public IReadOnlyList StepNames { get; init; } = EgressPreprocessor.StepNames; + public string CurrentUserName { get; init; } = string.Empty; + /// True when ConnectionStrings:EgressStorage is absent — the Summary warns before Confirm rather than after. + public bool StorageNotConfigured { get; init; } + /// The Results page's "raw data" columns — one shared set; a column a record has no value for renders blank. + public static readonly IReadOnlyList<(string Header, Func Value)> RawColumns = + [ + ("Ticket ID", r => r.TicketId?.ToString() ?? ""), + ("Reference", r => r.ReferenceNumber), + ("Decision", r => EgressDecisions.Label(r.Decision)), + ("Reason", r => r.Answer("reason") ?? ""), + ("Key stage", r => EgressOutputTypes.KeyStageValue(r.WindowType)), + ("DfE establishment number", r => string.IsNullOrWhiteSpace(r.PupilLaestab) ? r.OrganisationLaestab ?? "" : r.PupilLaestab), + ("Surname", r => r.Answer("last-name") ?? r.PupilSurname ?? ""), + ("Forename", r => r.Answer("first-name") ?? r.PupilFirstname ?? ""), + ("Sex", r => r.Answer("sex") ?? r.PupilSex ?? ""), + ("Date of birth", r => r.Answer("date-of-birth") ?? r.PupilDateOfBirth ?? ""), + ("Admission date", r => r.Answer("admission-date") ?? r.PupilEntryDate ?? ""), + ("Year group", r => r.Answer("year-group") ?? ""), + ("SEN status", r => r.Answer("sen-status") ?? ""), + ("UPN or ULN", r => r.Answer("upn") ?? r.PupilIdentifier ?? ""), + ("CYPMD ID", r => r.PupilCypmdId ?? ""), + ("LDS matched pupil ID", r => r.PupilMatchRef > 0 ? r.PupilMatchRef.ToString() : ""), + ("School URN", r => r.OrganisationUrn.ToString()), + ("Submitted", r => r.SubmittedAtUtc.ToString("yyyy-MM-dd HH:mm") + " UTC") + ]; +} + +public sealed class PreviewViewModel +{ + public required Guid RunId { get; init; } + public required EgressOutputType OutputType { get; init; } + public required string FileName { get; init; } + public required IReadOnlyList Headers { get; init; } + public required IReadOnlyList> Rows { get; init; } +} diff --git a/src/DfE.CheckPerformanceData.Web/Extensions/AdminNavServiceCollectionExtensions.cs b/src/DfE.CheckPerformanceData.Web/Extensions/AdminNavServiceCollectionExtensions.cs index 75cbb059d..ec2ce3a7d 100644 --- a/src/DfE.CheckPerformanceData.Web/Extensions/AdminNavServiceCollectionExtensions.cs +++ b/src/DfE.CheckPerformanceData.Web/Extensions/AdminNavServiceCollectionExtensions.cs @@ -59,6 +59,8 @@ public static IServiceCollection AddAdminNavEntries( services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/DfE.CheckPerformanceData.Web/Program.cs b/src/DfE.CheckPerformanceData.Web/Program.cs index d08e70f63..2389a7366 100644 --- a/src/DfE.CheckPerformanceData.Web/Program.cs +++ b/src/DfE.CheckPerformanceData.Web/Program.cs @@ -42,6 +42,7 @@ .AddCpdQueue(configuration) .AddCpdJourneyAndCmsServices() .AddCpdBlobStorage(configuration) + .AddCpdEgress(configuration, builder.Environment) .AddCpdBigQueryAnalytics(configuration); builder.AddCpdDevImpersonation(); diff --git a/src/DfE.CheckPerformanceData.Web/Properties/launchSettings.json b/src/DfE.CheckPerformanceData.Web/Properties/launchSettings.json index ddd7b1baf..a382f01b5 100644 --- a/src/DfE.CheckPerformanceData.Web/Properties/launchSettings.json +++ b/src/DfE.CheckPerformanceData.Web/Properties/launchSettings.json @@ -8,7 +8,9 @@ "applicationUrl": "http://localhost:5146", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", - "Dev__ToolsEnabled": "true" + "Dev__ToolsEnabled": "true", + "Zendesk__UseFake": "true", + "ConnectionStrings__EgressStorage": "DefaultEndpointsProtocol=http;AccountName=egressaccount;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/egressaccount;QueueEndpoint=http://127.0.0.1:10001/egressaccount;" } }, "https": { @@ -18,7 +20,9 @@ "applicationUrl": "https://localhost:7223;http://localhost:5146", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", - "Dev__ToolsEnabled": "true" + "Dev__ToolsEnabled": "true", + "Zendesk__UseFake": "true", + "ConnectionStrings__EgressStorage": "DefaultEndpointsProtocol=http;AccountName=egressaccount;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/egressaccount;QueueEndpoint=http://127.0.0.1:10001/egressaccount;" } } } diff --git a/src/DfE.CheckPerformanceData.Web/Startup/BlobStorageExtensions.cs b/src/DfE.CheckPerformanceData.Web/Startup/BlobStorageExtensions.cs index d119956d6..6afc1c0ce 100644 --- a/src/DfE.CheckPerformanceData.Web/Startup/BlobStorageExtensions.cs +++ b/src/DfE.CheckPerformanceData.Web/Startup/BlobStorageExtensions.cs @@ -35,6 +35,11 @@ public static IServiceCollection AddCpdBlobStorage(this IServiceCollection servi var ingressConn = configuration.GetConnectionString("IngressStorage"); if (!string.IsNullOrEmpty(ingressConn)) clients["ingress"] = new BlobServiceClient(ingressConn); + // The LDS interface account (AB#294553): container cypmd, folder extracts_input. Absent + // locally unless the compose file supplies it; the egress refuses to transfer without it. + var egressConn = configuration.GetConnectionString("EgressStorage"); + if (!string.IsNullOrEmpty(egressConn)) + clients["egress"] = new BlobServiceClient(egressConn); return clients; }); services.Configure( diff --git a/src/DfE.CheckPerformanceData.Web/Startup/EgressExtensions.cs b/src/DfE.CheckPerformanceData.Web/Startup/EgressExtensions.cs new file mode 100644 index 000000000..e49c1b869 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Startup/EgressExtensions.cs @@ -0,0 +1,49 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.Settings; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Infrastructure; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.Persistence.Repositories; +using Microsoft.Extensions.Hosting; + +namespace DfE.CheckPerformanceData.Web.Startup; + +public static class EgressExtensions +{ + // Registers the LDS data egress (AB#294553). Review finding B1: the safe default is the real + // Zendesk client, matching the worker's own configured default (RulesEngineWorker's + // appsettings.json pins Zendesk:UseFake=false; Web/appsettings.json now does the same) — the + // dev outbox fake is opt-in only (Zendesk__UseFake=true), and refused outright in Production + // regardless of what configuration says, the same way DevEgressController is unreachable there. + public static IServiceCollection AddCpdEgress( + this IServiceCollection services, IConfiguration configuration, IHostEnvironment environment) + { + services.Configure(configuration.GetSection(EgressStorageOptions.SectionName)); + services.Configure(configuration.GetSection(ZendeskTicketFieldSettings.SectionName)); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + + var useFake = configuration.GetValue(SettingKeys.ZendeskUseFake, defaultValue: false); + if (useFake) + { + if (environment.IsProduction()) + { + throw new InvalidOperationException( + $"{SettingKeys.ZendeskUseFake}=true is not permitted in Production: the dev " + + "outbox ticket source must never be reachable there. Remove the setting (or " + + "set it to false) for this environment."); + } + services.AddScoped(); + } + else + { + services.AddZendeskApiClient(configuration, requireRealClient: true); + services.AddScoped(); + } + return services; + } +} diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Complete.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Complete.cshtml new file mode 100644 index 000000000..7b647f77f --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Complete.cshtml @@ -0,0 +1,49 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@using DfE.CheckPerformanceData.Application.Egress +@{ + ViewData["Title"] = "Transfer complete"; +} + +Back to data egress + +
+

Transfer complete

+
Files successfully transferred to LDS blob storage
+
+ +

Transfer summary

+
+
+
Checking window
+
@Model.WindowTitle
+
+
+
Transferred by
+
@Model.Run.TransferredByName
+
+
+
Transferred at
+
@Model.Run.TransferredAtUtc?.ToString("d MMMM yyyy 'at' HH:mm") UTC
+
+
+
Target container
+
@Model.TargetDescription
+
+
+
Files transferred
+
+
    + @foreach (var output in Model.Run.Outputs) + { +
  • @output.FileName (@output.OutputRecordCount records, SHA-256 @output.Sha256)
    • + } +
+
+
+
+
Status
+
Success
+
+
+ +Start a new egress diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Failed.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Failed.cshtml new file mode 100644 index 000000000..d2262f317 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Failed.cshtml @@ -0,0 +1,48 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@{ + ViewData["Title"] = "Preprocessing failed"; + ViewData["HasError"] = true; +} + +Back to data egress + +

Preprocessing failed

+ +
+
+

No records were transferred

+
+

@Model.Run.Failures.Count problem(s) were found across @Model.Run.Failures.Select(f => f.ReferenceNumber).Distinct().Count() record(s). Because one failing record fails the whole batch, nothing was saved and nothing was sent. Correct the source data, then start a new run.

+
+
+
+ + + + + + + + + + + + + + @foreach (var failure in Model.Run.Failures) + { + + + + + + + + } + +
Records that failed
Ticket IDReferenceStepFieldProblem
@failure.TicketId@failure.ReferenceNumber@failure.Step@failure.Field@failure.Reason
+ +
+ View the pulled data + Start a new egress +
diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Index.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Index.cshtml new file mode 100644 index 000000000..f41b2209c --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Index.cshtml @@ -0,0 +1,192 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.PullViewModel +@using DfE.CheckPerformanceData.Application.Egress +@using DfE.CheckPerformanceData.Domain.Enums +@{ + ViewData["Title"] = "Data egress"; + ViewData["HasError"] = !Model.IsValid; +} + +Back to administration + +@if (Model.Banner is not null) +{ + @* Nit: GOV.UK's default notification banner is role="region" — "alert" is reserved for a + success banner, and every banner this page shows today ("abandoned", "already + transferred", refusals) is neutral, not success. *@ +
+
+

Important

+
+
+

@Model.Banner

+
+
+} + +

Data egress

+ +@if (Model.StorageNotConfigured) +{ + @* Review finding (18 Sep): say so on the first screen, not only after Confirm. Pull, preprocess, + preview and download still work without a target account; only Transfer refuses. *@ +
+ + + Warning + Egress storage is not configured for this environment (ConnectionStrings:EgressStorage). You can pull, preprocess and download files, but a transfer to LDS will be refused until it is set. + +
+} + +@* The library's govuk-checkboxes-error-message tag helper below registers its error with + GovUk.Frontend.AspNetCore's page-level error-summary aggregator; using the library's own + govuk-error-summary tag helper here (rather than hand-written markup) is what "claims" that + aggregator, so it does not also auto-render a second, unclaimed summary. *@ +@if (!Model.IsValid) +{ + + @if (Model.WindowError is not null) { @Model.WindowError } + @if (Model.OutputTypesError is not null) { @Model.OutputTypesError } + @foreach (var refusal in Model.Refusals) { @refusal } + @if (Model.PullError is not null) { Zendesk could not be read: @Model.PullError } + +} + +
+ @Html.AntiForgeryToken() + +
+ +
Choose the checking window you want to export data for.
+ @if (Model.WindowError is not null) + { +

Error: @Model.WindowError

+ } + +
+ + @* S5: still deliberately not the govuk-checkboxes-error-message tag helper — in this version + of GovUk.Frontend.AspNetCore that tag helper unconditionally auto-renders its OWN page-level + error summary at the top of
in addition to this page's single, consolidated + govuk-error-summary above, and it only binds via asp-validation-for against ModelState, + which this field does not use. The field-level error is plain markup instead, but now placed + inside the form group between the hint and the checkboxes (the GOV.UK pattern) and referenced + from the fieldset's aria-describedby, matching the window select's error above. *@ +
+ + @* The library already appends its own auto-generated OutputTypes-hint id to whatever + aria-describedby is supplied here (verified live), so only the error id — which it + does not know about — needs adding, and only when there is one. *@ + + 2. Select data to pull from Zendesk + Select one or more output types to retrieve. + @* govuk-checkboxes-before-inputs is the fieldset's one slot for plain markup between + the hint and the items — govuk-checkboxes-fieldset itself only allows tag-helper + children, so a bare

here (as tried before) is RZ2009. *@ + + @if (Model.OutputTypesError is not null) + { +

Error: @Model.OutputTypesError

+ } + + @foreach (var type in Model.OutputTypes) + { + @EgressOutputTypes.Label(type) + } + + +
+ + Pull data from Zendesk + + +

Saved runs

+@if (Model.SavedRuns.Count == 0) +{ +

There are no saved runs.

+} +else +{ + + + + + + + + + + + + + + @foreach (var run in Model.SavedRuns) + { + + + + + + + + + } + +
Saved runs
Checking windowOutput typesStageStarted byStartedAction
@run.WindowTitle@string.Join(", ", run.OutputTypes.Select(EgressOutputTypes.Label))@StageLabel(run.Status)@run.StartedByName@run.StartedAtUtc.ToString("d MMM yyyy HH:mm") UTCResume run for @run.WindowTitle started @run.StartedAtUtc.ToString("d MMM yyyy HH:mm")
+} + +

Completed runs

+@if (Model.CompletedRuns.Count == 0) +{ +

Nothing has been transferred yet.

+} +else +{ + + + + + + + + + + + + + + @foreach (var run in Model.CompletedRuns) + { + + + + + + + + + } + +
Completed runs
Checking windowOutput typesStatusStarted byTransferredAction
@run.WindowTitle@string.Join(", ", run.OutputTypes.Select(EgressOutputTypes.Label))Transferred@run.StartedByName@run.TransferredAtUtc?.ToString("d MMM yyyy HH:mm") UTCView run for @run.WindowTitle
+} + +@functions { + // Where a saved run will reopen. FLAGGED copy. + // Second-pass nit: delegates to EgressRunStatuses.Label — the single source of truth this + // mapping now shares with the preprocessing refusal copy — instead of keeping its own copy. + private static string StageLabel(EgressRunStatus status) => EgressRunStatuses.Label(status); + + // Nit: the saved-runs list gave every stage the same blue tag, including the two failed ones. + private static string StageTagClass(EgressRunStatus status) => status switch + { + EgressRunStatus.PreprocessingFailed or EgressRunStatus.TransferFailed => "govuk-tag--red", + _ => "govuk-tag--blue" + }; +} diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Preprocessing.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Preprocessing.cshtml new file mode 100644 index 000000000..548698ba3 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Preprocessing.cshtml @@ -0,0 +1,55 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@{ + ViewData["Title"] = "Preprocessing data"; +} + +Back to pulled data + +

Preprocessing data

+

@Model.WindowTitle: do not navigate away from this page while preprocessing runs. If you do, the run goes back to the pulled data and nothing is saved.

+ +
+ + @* No-JS path: the button posts and the server runs every step before redirecting. JS hides + this form and opens the stream instead, ticking the steps below as they complete. *@ +
+ @Html.AntiForgeryToken() + Run preprocessing +
+ +
+

Preprocessing progress

+ @* S6: role="progressbar" only permits presentational children per the ARIA spec, so an + aria-live region nested inside it may never be announced by assistive tech. Nothing + visual is drawn inside the bar itself, so the status paragraph is a sibling instead. *@ +
+

Starting…

+
+ +

Steps

+
    + @{ var index = 0; } + @foreach (var step in Model.StepNames) + { + index++; +
  1. + @step + Not started + +
    2. + } +
+ +
+ Save and exit + @* M4: the lock has no expiry, so a run stuck here after a restart must be releasable. *@ +
+ @Html.AntiForgeryToken() + +
+
+
+ +@section Scripts { + +} diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Preview.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Preview.cshtml new file mode 100644 index 000000000..d6c2caeb1 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Preview.cshtml @@ -0,0 +1,33 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.PreviewViewModel +@{ + ViewData["Title"] = $"Preview {Model.FileName}"; +} + +Back to confirm transfer +

@Model.FileName

+

@Model.Rows.Count data row(s). Download the file.

+ +
+ + + + + @foreach (var header in Model.Headers) + { + + } + + + + @foreach (var row in Model.Rows) + { + + @foreach (var cell in row) + { + + } + + } + +
File contents
@header
@cell
+
diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Results.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Results.cshtml new file mode 100644 index 000000000..b38d94739 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Results.cshtml @@ -0,0 +1,65 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@using DfE.CheckPerformanceData.Application.Egress +@using DfE.CheckPerformanceData.Domain.Enums +@using DfE.CheckPerformanceData.Web.Controllers.Egress +@{ + ViewData["Title"] = "Data pulled from Zendesk"; + var total = Model.Run.Outputs.Sum(o => o.SourceRecordCount); +} + +Back to data egress + +

Data pulled from Zendesk

+@* Nit: rewritten from developer-note phrasing into plain user-facing language. + FLAGGED copy — see PR notes §5. *@ +

Every request for the selected output types is listed below, whatever its decision. Only approved and auto-approved requests will be included when you proceed to preprocessing.

+ +@if (total == 0) +{ +

There are no requests to process for this checking window and these output types.

+} + + + @foreach (var output in Model.Run.Outputs) + { + +

@EgressOutputTypes.Label(output.OutputType) (@output.SourceRecordCount records)

+
+ + + + + @foreach (var column in RunPageViewModel.RawColumns) + { + + } + + + + @foreach (var record in output.Records) + { + + @foreach (var column in RunPageViewModel.RawColumns) + { + + } + + } + +
@EgressOutputTypes.Label(output.OutputType) pulled from Zendesk
@column.Header
@column.Value(record)
+
+ + } + + +
+ @if (total > 0 && Model.Run.Status != EgressRunStatus.PreprocessingFailed) + { + Proceed to preprocessing + } + Save and exit +
+ @Html.AntiForgeryToken() + +
+
diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/Summary.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/Summary.cshtml new file mode 100644 index 000000000..45e4ff152 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/Summary.cshtml @@ -0,0 +1,109 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@using DfE.CheckPerformanceData.Application.Egress +@using DfE.CheckPerformanceData.Domain.Enums +@{ + ViewData["Title"] = "Confirm transfer to LDS"; + ViewData["HasError"] = Model.TransferError is not null; + // M3: every output preprocessed to zero saved rows — nothing to send, so no Confirm button. + var nothingToTransfer = Model.Run.Outputs.All(o => (o.OutputRecordCount ?? 0) == 0); + // Nit: a Transferring run already has a transfer in flight (guarded server-side by the CAS + // status flip regardless), so Confirm is not offered a second time here either. + var offerConfirm = !nothingToTransfer && Model.Run.Status != EgressRunStatus.Transferring; +} + +Back to data egress + +

Confirm transfer to LDS

+ +@if (Model.TransferError is not null) +{ +
+
+

The transfer did not complete

+
+

@Model.TransferError

+

No files were left in the target container and the prepared records are intact. You can try again.

+
+
+
+} + +@if (Model.StorageNotConfigured) +{ + @* Review finding (18 Sep): warn before Confirm, not only in the failure banner after it. *@ +
+ + + Warning + Egress storage is not configured for this environment (ConnectionStrings:EgressStorage). Confirm will be refused until it is set; the prepared files can still be previewed and downloaded. + +
+} + +

Review the details below before transferring data to LDS blob storage.

+ +

Transfer details

+
+
+
Checking window
+
@Model.WindowTitle
+
+
+
Preprocessed on
+
@Model.Run.PreprocessedAtUtc?.ToString("d MMMM yyyy 'at' HH:mm") UTC
+
+
+
Target container
+
@Model.TargetDescription
+
+
+
Transferred by
+
@Model.CurrentUserName
+
+
+ +

Files to be transferred

+ + + + + + + + + + + + @foreach (var output in Model.Run.Outputs) + { + + + + + + + } + +
Files to be transferred
Output typeFile nameRecordsDownload
@EgressOutputTypes.Label(output.OutputType)@output.FileName (preview)@output.OutputRecordCountDownload @output.FileName
+ +
Once confirmed, these files will be uploaded to LDS blob storage. This action cannot be undone.
+ +@if (nothingToTransfer) +{ +

Every record was rejected, undecided or discarded before preprocessing, so there is nothing to transfer to LDS. Abandon this run if it is no longer needed.

+} + +
+ @if (offerConfirm) + { +
+ @Html.AntiForgeryToken() + +
+ } + Save and exit +
+ @Html.AntiForgeryToken() + +
+
diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/_RunHeader.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/_RunHeader.cshtml new file mode 100644 index 000000000..0a6277433 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/_RunHeader.cshtml @@ -0,0 +1,6 @@ +@model DfE.CheckPerformanceData.Web.Controllers.Egress.RunPageViewModel +@using DfE.CheckPerformanceData.Application.Egress + +@* Nit: every page using this partial has an

, so the caption must + be govuk-caption-xl to match — GOV.UK pairs caption size to heading size. *@ +@Model.WindowTitle · @string.Join(", ", Model.Run.Outputs.Select(o => EgressOutputTypes.Label(o.OutputType))) diff --git a/src/DfE.CheckPerformanceData.Web/Views/Egress/_ViewStart.cshtml b/src/DfE.CheckPerformanceData.Web/Views/Egress/_ViewStart.cshtml new file mode 100644 index 000000000..65ec09493 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/Views/Egress/_ViewStart.cshtml @@ -0,0 +1,3 @@ +@{ + Layout = "_AdminLayout"; +} diff --git a/src/DfE.CheckPerformanceData.Web/appsettings.json b/src/DfE.CheckPerformanceData.Web/appsettings.json index 0c1723dcb..f081003da 100644 --- a/src/DfE.CheckPerformanceData.Web/appsettings.json +++ b/src/DfE.CheckPerformanceData.Web/appsettings.json @@ -36,6 +36,9 @@ ] }, "AllowedHosts": "*", + "Zendesk": { + "UseFake": false + }, "PollySettings": { "MaxRetryAttempts": 3, "BaseDelayMilliseconds": 1000, diff --git a/src/DfE.CheckPerformanceData.Web/wwwroot/js/egress-preprocess.js b/src/DfE.CheckPerformanceData.Web/wwwroot/js/egress-preprocess.js new file mode 100644 index 000000000..55ed89956 --- /dev/null +++ b/src/DfE.CheckPerformanceData.Web/wwwroot/js/egress-preprocess.js @@ -0,0 +1,87 @@ +(function () { + 'use strict'; + + // Enhances the egress Preprocessing page (AB#294553). Without JS the form posts and the server + // runs every step before redirecting. With JS the Run button opens a server-sent stream and + // each of the eight steps is marked Running / Done / Failed as the server reports it; the + // terminal event carries the URL to move on to (summary, or the failure list). + + function init() { + var root = document.querySelector('[data-module="egress-preprocess"]'); + if (!root) { return; } + + var streamUrl = root.getAttribute('data-stream-url'); + if (!streamUrl || !('EventSource' in window)) { return; } + + var form = root.querySelector('[data-egress-form]'); + var startButton = root.querySelector('[data-egress-start]'); + var wrapper = root.querySelector('[data-egress-progress-wrapper]'); + var status = root.querySelector('[data-egress-status]'); + var bar = root.querySelector('[data-egress-progressbar]'); + if (!startButton) { return; } + + startButton.addEventListener('click', function (e) { + e.preventDefault(); + begin(); + }); + + function begin() { + if (form) { form.classList.add('govuk-!-display-none'); } + if (wrapper) { wrapper.classList.remove('govuk-!-display-none'); } + disableLinks(); + setStatus('Starting…'); + + var es = new EventSource(streamUrl); + es.addEventListener('progress', function (event) { + var data; + try { data = JSON.parse(event.data); } catch (err) { return; } + render(data); + if (data.isComplete) { + es.close(); + if (data.nextUrl) { window.location.href = data.nextUrl; } + } + }); + es.onerror = function () { + setStatus('Connection lost. Refresh the page to see where the run got to.'); + es.close(); + }; + } + + function render(data) { + var li = root.querySelector('[data-egress-step="' + data.step + '"]'); + if (li) { + var tag = li.querySelector('[data-egress-step-state]'); + var message = li.querySelector('[data-egress-step-message]'); + if (tag) { + tag.className = 'govuk-tag govuk-!-margin-left-2 ' + (data.state === 'done' ? 'govuk-tag--green' : data.state === 'failed' ? 'govuk-tag--red' : 'govuk-tag--blue'); + tag.textContent = data.state === 'done' ? 'Done' : data.state === 'failed' ? 'Failed' : 'Running'; + } + if (message && data.state !== 'running') { message.textContent = data.message; } + } + if (bar) { + var completed = data.state === 'running' ? data.step - 1 : data.step; + bar.setAttribute('aria-valuenow', String(completed)); + } + setStatus((data.state === 'running' ? data.stepName + '…' : data.message) + ' (' + (data.state === 'running' ? data.step - 1 : data.step) + ' of ' + data.totalSteps + ' steps complete)'); + } + + function disableLinks() { + var links = root.querySelectorAll('a.govuk-button'); + for (var i = 0; i < links.length; i++) { + links[i].setAttribute('aria-disabled', 'true'); + links[i].classList.add('govuk-button--disabled'); + links[i].addEventListener('click', function (e) { e.preventDefault(); }); + } + } + + function setStatus(text) { + if (status) { status.textContent = text; } + } + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } +})(); diff --git a/terraform/application/application.tf b/terraform/application/application.tf index 144054e69..afecb37b2 100644 --- a/terraform/application/application.tf +++ b/terraform/application/application.tf @@ -24,7 +24,18 @@ module "application_configuration" { AZURE_STORAGE_ACCOUNT_NAME = local.azure_storage_account_name AZURE_STORAGE_ACCESS_KEY = local.azure_storage_access_key AZURE_STORAGE_CONTAINER = local.azure_storage_container - }, local.federated_auth_secrets) + }, local.federated_auth_secrets, local.egress_storage_secrets) +} + +# AB#294553: the LDS egress needs an account to transfer into (ConnectionStrings__EgressStorage); +# without it the app refuses to transfer with a "not configured" message. The real LDS account +# for the long-lived environments is still to be confirmed with LDS (docs/data-egress.md, gaps), +# so only review apps get one here — the E2E suite's egress facts walk the transfer end to end +# against the per-PR "lds" account, whose cypmd/extracts_input the app creates on first upload. +locals { + egress_storage_secrets = var.config == "review" ? { + ConnectionStrings__EgressStorage = module.storage_private.primary_connection_string + } : {} } diff --git a/terraform/application/config/review.yml b/terraform/application/config/review.yml index 463de245b..8bbc222b1 100644 --- a/terraform/application/config/review.yml +++ b/terraform/application/config/review.yml @@ -34,6 +34,14 @@ SeedDevelopmentData: true # hard-guards against Production regardless of this flag. Dev__ToolsEnabled: true +# AB#294553: the E2E egress facts seed their decisions into the DevZendeskTickets outbox +# (/dev/egress/seed), so the review app must read decisions from that outbox rather than +# from esfa-preprod, where those ticket ids do not exist. This is the same opt-in the local +# docker-compose stack makes. Web and worker share this ConfigMap, so the worker's ticket +# writes land in the outbox too — review apps do not create real Zendesk tickets. +# AddCpdEgress refuses this flag in Production regardless of configuration. +Zendesk__UseFake: true + # Rules Engine Worker configuration RulesEngineOptions__QueueName: change-requests RulesEngineOptions__RetryDelayMs: 5000 diff --git a/tests/DfE.CheckPerformanceData.E2ETests/Admin/DataEgressTests.cs b/tests/DfE.CheckPerformanceData.E2ETests/Admin/DataEgressTests.cs new file mode 100644 index 000000000..63e68f458 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.E2ETests/Admin/DataEgressTests.cs @@ -0,0 +1,320 @@ +using System.Net; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using DfE.CheckPerformanceData.E2ETests.Fixtures; +using DfE.CheckPerformanceData.E2ETests.Helpers; +using Microsoft.Playwright; + +namespace DfE.CheckPerformanceData.E2ETests.Admin; + +// AB#294553 acceptance criteria, walked over HTTP (so they run on every platform) with one +// browser fact for the streamed progress. The seeded KS4 June window is the dev seed's; every run +// is cleaned up first because a transferred run blocks the pair forever by design. +[Collection("E2E")] +public sealed class DataEgressTests(PlaywrightFixture fixture) : SeedingPageTest(fixture) +{ + private static readonly Guid WindowId = Guid.Parse("F34D285B-8660-4D12-9C30-787328DEAA0A"); + + private HttpClient Client => Fixture.SeedClient; // the fixture's BaseAddress-bearing client used by SeedHelpers + + // IEnumerable, not Dictionary: ASP.NET Core model-binds a List from + // genuinely repeated identical keys (OutputTypes=A&OutputTypes=B), not from a plain key + // mixed with an indexed one (OutputTypes=A&OutputTypes[1]=B) — verified against the running + // app; the latter silently binds only the first value. A Dictionary cannot + // hold the "OutputTypes" key twice, so callers needing more than one value build a list. + private async Task PostFormAsync(string path, IEnumerable> fields, HttpStatusCode expected = HttpStatusCode.Found) + { + var (token, cookie) = await AntiforgeryHelpers.ScrapeAsync(Client, "/dev/antiforgery-token"); + using var request = new HttpRequestMessage(HttpMethod.Post, $"{Fixture.BaseUrl}{path}") + { + Content = new FormUrlEncodedContent(fields.Append(new KeyValuePair("__RequestVerificationToken", token))) + }; + request.Headers.Add("Cookie", cookie); + var response = await TestHttpClients.SendAsync(request); + Assert.Equal(expected, response.StatusCode); + return response.Headers.Location?.ToString() ?? await response.Content.ReadAsStringAsync(); + } + + private async Task GetAsync(string path, HttpStatusCode expected = HttpStatusCode.OK) + { + using var request = new HttpRequestMessage(HttpMethod.Get, path.StartsWith("http") ? path : $"{Fixture.BaseUrl}{path}"); + var response = await TestHttpClients.SendAsync(request); + Assert.Equal(expected, response.StatusCode); + return await response.Content.ReadAsStringAsync(); + } + + private async Task SeedAsync(string outputType, string decision, int count) + { + using var request = new HttpRequestMessage(HttpMethod.Post, + $"{Fixture.BaseUrl}/dev/egress/seed?windowId={WindowId}&outputType={outputType}&decision={decision}&count={count}&laestab=860/4070&urn=142313&reason=pupil-died"); + (await TestHttpClients.SendAsync(request)).EnsureSuccessStatusCode(); + } + + private async Task CleanupAsync() + { + using var request = new HttpRequestMessage(HttpMethod.Post, $"{Fixture.BaseUrl}/dev/egress/cleanup?windowId={WindowId}"); + (await TestHttpClients.SendAsync(request)).EnsureSuccessStatusCode(); + } + + // S5: the Task 12 duplicate-error-summary fix (moving the output-types error out of the + // fieldset entirely) detached the field error from its fieldset. Confirms live that exactly + // one error summary still renders and the fieldset's aria-describedby now includes the error. + [Fact] + public async Task Pull_with_nothing_selected_shows_one_error_summary_and_associates_the_checkbox_error() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + + var html = await PostFormAsync("/admin/egress", [], HttpStatusCode.OK); + + Assert.Single(Regex.Matches(html, "There is a problem")); + Assert.Contains("govuk-form-group govuk-form-group--error", html); + Assert.Matches("]*aria-describedby=\"OutputTypes-hint OutputTypes-error\"", html); + } + finally { await AuthHelpers.ImpersonateAsEditorAsync(Fixture); } + } + + [Fact] + public async Task A_school_user_gets_404_from_the_egress_section() + { + try + { + await AuthHelpers.ImpersonateAsUnprivilegedUserAsync(Fixture); + await GetAsync("/admin/egress", HttpStatusCode.NotFound); + } + finally { await AuthHelpers.ImpersonateAsEditorAsync(Fixture); } + } + + [Fact] + public async Task The_whole_journey_over_plain_http_pull_preprocess_summary_transfer_complete_then_refused() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "auto_approved", 2); + await SeedAsync("RemoveLearners", "rejected", 1); + await SeedAsync("RemoveLearners", "none", 1); + await SeedAsync("NewLearners", "approved", 1); + + // Pull → Results + var resultsUrl = await PostFormAsync("/admin/egress", + [ + new("WindowId", WindowId.ToString()), + new("OutputTypes", "NewLearners"), + new("OutputTypes", "RemoveLearners") + ]); + Assert.Matches("/admin/egress/runs/[0-9a-f-]{36}/results$", resultsUrl); + var runId = Regex.Match(resultsUrl, "runs/([0-9a-f-]{36})").Groups[1].Value; + var results = await GetAsync(resultsUrl); + Assert.Contains("data-testid=\"egress-results-removelearners\"", results); + Assert.Contains("data-testid=\"egress-results-newlearners\"", results); + Assert.Contains("Auto approved", results); + Assert.Contains("Rejected", results); + Assert.Contains("No Zendesk ticket", results); + Assert.Equal(5, Regex.Matches(results, "DEV-EGRESS-").Count); + + // Preprocessing (no-JS POST) → Summary + var summaryUrl = await PostFormAsync($"/admin/egress/runs/{runId}/preprocessing", []); + Assert.EndsWith($"/admin/egress/runs/{runId}/summary", summaryUrl); + var summary = await GetAsync(summaryUrl); + Assert.Contains("cypmd/extracts_input", summary); + Assert.Matches("CYPMD_LDS_KS4_RemoveLearners_\\d{4}_\\d{2}_\\d{2}\\.csv", summary); + Assert.Matches("CYPMD_LDS_KS4_NewLearners_\\d{4}_\\d{2}_\\d{2}\\.csv", summary); + + // Download: only the two approved removals, in spec shape + var csv = await GetAsync($"/admin/egress/runs/{runId}/download/RemoveLearners"); + var lines = csv.Split("\r\n"); + Assert.Equal("Correction_ID,Correction_Type,Correction_Reason,Key_Stage,Establishment_Number,Surname,Forename,Sex,Date_of_Birth,Cycle_Year,Cycle_Month,Local_Authority,Learner_ID,Year_Group", lines[0]); + Assert.Equal(3, lines.Length); + Assert.Contains(",31,4,KS4,4070,", lines[1]); + Assert.Contains(",860,", lines[1]); + Assert.EndsWith(",", lines[1]); // blank Year_Group: pupil-died is not a year-group change + Assert.DoesNotContain("\n", lines[2]); + + // Transfer → Complete + var completeUrl = await PostFormAsync($"/admin/egress/runs/{runId}/transfer", []); + Assert.EndsWith($"/admin/egress/runs/{runId}/complete", completeUrl); + var complete = await GetAsync(completeUrl); + Assert.Contains("Transfer complete", complete); + Assert.Contains("Files transferred", complete); + + // Refused: the pair has been transferred + var refused = await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }, HttpStatusCode.OK); + Assert.Contains("already been transferred", refused); + + // Listed as completed + var index = await GetAsync("/admin/egress"); + Assert.Contains("data-testid=\"egress-completed-runs\"", index); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + [Fact] + public async Task A_failing_record_fails_the_batch_and_the_failure_page_names_it() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "approved", 1); + using (var bad = new HttpRequestMessage(HttpMethod.Post, + $"{Fixture.BaseUrl}/dev/egress/seed?windowId={WindowId}&outputType=RemoveLearners&decision=approved&count=1&laestab=860/4070&urn=142313&reason=other")) + (await TestHttpClients.SendAsync(bad)).EnsureSuccessStatusCode(); + + var resultsUrl = await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + var runId = Regex.Match(resultsUrl, "runs/([0-9a-f-]{36})").Groups[1].Value; + var failedUrl = await PostFormAsync($"/admin/egress/runs/{runId}/preprocessing", []); + Assert.EndsWith($"/admin/egress/runs/{runId}/failed", failedUrl); + var failed = await GetAsync(failedUrl); + Assert.Contains("No records were transferred", failed); + Assert.Contains("Correction_Reason", failed); + Assert.Contains("other", failed); + + // The pair is free again: a new run can start. + await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + // S10: E2E had no Abandon fact — confirms the pair is genuinely released, not just that the + // banner appears. + [Fact] + public async Task Abandon_releases_the_pair_for_a_fresh_pull() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "auto_approved", 1); + + var resultsUrl = await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + var runId = Regex.Match(resultsUrl, "runs/([0-9a-f-]{36})").Groups[1].Value; + + var homeUrl = await PostFormAsync($"/admin/egress/runs/{runId}/abandon", []); + Assert.EndsWith("/admin/egress", homeUrl); + // The banner itself is TempData-cookie-driven, which this harness's UseCookies=false + // no-redirect client does not carry across the redirect (unlike a real browser or the + // manual curl walk that confirmed it live) — the state change below is what matters. + + // The pair is free again: a new pull for the same window and output type succeeds + // rather than being refused. + var freshResultsUrl = await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + Assert.Matches("/admin/egress/runs/[0-9a-f-]{36}/results$", freshResultsUrl); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + // S10: E2E had no fact for the "in-progress" refusal (only the post-transfer one). + [Fact] + public async Task A_second_pull_for_the_same_pair_is_refused_while_the_first_is_still_open() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "auto_approved", 1); + await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + + var refused = await PostFormAsync("/admin/egress", + new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }, HttpStatusCode.OK); + + Assert.Contains("is already being processed by", refused); + Assert.Contains("Wait for that run to finish or be abandoned", refused); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + // S10: E2E had no fact for resuming a saved (in-progress) run from the Pull page's list. + [Fact] + public async Task A_saved_run_is_listed_on_the_pull_page_and_resume_reopens_it() + { + try + { + await AuthHelpers.ImpersonateAsAdminAsync(Fixture); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "auto_approved", 1); + var resultsUrl = await PostFormAsync("/admin/egress", new Dictionary { ["WindowId"] = WindowId.ToString(), ["OutputTypes"] = "RemoveLearners" }); + var runId = Regex.Match(resultsUrl, "runs/([0-9a-f-]{36})").Groups[1].Value; + + var index = await GetAsync("/admin/egress"); + Assert.Contains("data-testid=\"egress-saved-runs\"", index); + Assert.Contains(runId, index); + Assert.Contains("Data pulled", index); + + using var resumeRequest = new HttpRequestMessage(HttpMethod.Get, $"{Fixture.BaseUrl}/admin/egress/runs/{runId}"); + var resumeResponse = await TestHttpClients.SendAsync(resumeRequest); + Assert.Equal(HttpStatusCode.Found, resumeResponse.StatusCode); + Assert.EndsWith(resultsUrl, resumeResponse.Headers.Location?.ToString()); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + [SkippableFact] + public async Task With_javascript_the_steps_tick_as_the_stream_reports_them() + { + Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux), "Playwright browser test Linux-only"); + try + { + AttachCookieToContext(await AuthHelpers.ImpersonateAsAdminAsync(Fixture)); + await CleanupAsync(); + await SeedAsync("RemoveLearners", "auto_approved", 1); + + await Page.GotoAsync($"{Fixture.BaseUrl}/admin/egress"); + await Page.SelectOptionAsync("select#WindowId", WindowId.ToString()); + await Page.CheckAsync("input[name='OutputTypes'][value='RemoveLearners']"); + await Page.ClickAsync("[data-testid='egress-pull']"); + await Page.ClickAsync("[data-testid='egress-proceed']"); + await Expect(Page.Locator("h1")).ToContainTextAsync("Preprocessing data"); + + await Page.ClickAsync("[data-testid='egress-run-preprocessing']"); + + // A single record against the local database preprocesses in well under a second, so + // the browser can navigate to Summary before ever rendering an intermediate step's + // "Done" state — asserting on step 1 (or even step 8) is a race the pipeline usually + // wins. The only assertion robust to that speed is the terminal outcome the stream + // drives the page to. + await Expect(Page.Locator("h1")).ToContainTextAsync("Confirm transfer to LDS", new() { Timeout = 15_000 }); + } + finally + { + await CleanupAsync(); + await AuthHelpers.ImpersonateAsEditorAsync(Fixture); + } + } + + private void AttachCookieToContext(string? cookieHeader) + { + if (string.IsNullOrEmpty(cookieHeader)) return; + var equalsIndex = cookieHeader.IndexOf('='); + if (equalsIndex <= 0) return; + + Context.AddCookiesAsync([new Microsoft.Playwright.Cookie + { + Name = cookieHeader[..equalsIndex], + Value = cookieHeader[(equalsIndex + 1)..], + Url = Fixture.BaseUrl + }]).GetAwaiter().GetResult(); + } +} diff --git a/tests/DfE.CheckPerformanceData.E2ETests/DfE.CheckPerformanceData.E2ETests.csproj b/tests/DfE.CheckPerformanceData.E2ETests/DfE.CheckPerformanceData.E2ETests.csproj index e31a1746a..2d2897ee2 100644 --- a/tests/DfE.CheckPerformanceData.E2ETests/DfE.CheckPerformanceData.E2ETests.csproj +++ b/tests/DfE.CheckPerformanceData.E2ETests/DfE.CheckPerformanceData.E2ETests.csproj @@ -8,13 +8,13 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/DfE.CheckPerformanceData.IntegrationTests.csproj b/tests/DfE.CheckPerformanceData.IntegrationTests/DfE.CheckPerformanceData.IntegrationTests.csproj index fd745cb38..6386cc74f 100644 --- a/tests/DfE.CheckPerformanceData.IntegrationTests/DfE.CheckPerformanceData.IntegrationTests.csproj +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/DfE.CheckPerformanceData.IntegrationTests.csproj @@ -16,13 +16,13 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevEgressControllerTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevEgressControllerTests.cs new file mode 100644 index 000000000..f35996139 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevEgressControllerTests.cs @@ -0,0 +1,159 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.Journey; +using DfE.CheckPerformanceData.Application.RequestSubmission; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using DfE.CheckPerformanceData.Persistence.Entities; +using DfE.CheckPerformanceData.Web.Controllers; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using NSubstitute; + +namespace DfE.CheckPerformanceData.IntegrationTests.Egress; + +// S10: DevEgressControllerTests (unit project) covers only the two 404 cases, which would pass +// even if IsAllowed were hard-coded false — this class adds the positive Development case. Lives +// here against real Postgres for the same reason as DevOutboxEgressTicketSourceTests: Seed and +// Cleanup are built entirely from EF LINQ queries the unit project has no in-memory pattern for. +[Collection(nameof(PostgresCollection))] +public sealed class DevEgressControllerTests(PostgresFixture fixture) +{ + private static readonly Guid WindowId = Guid.Parse("A0000000-0000-0000-0000-00000000E603"); + + private async Task ResetAsync() + { + await using var db = fixture.CreateContext(); + if (!await db.CheckingWindows.AnyAsync(w => w.Id == WindowId)) + { + db.CheckingWindows.Add(new CheckingWindow + { + Id = WindowId, Title = "Dev egress window", KeyStage = KeyStages.KS4, + CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + await db.SaveChangesAsync(); + } + await db.ChangeRequests.Where(r => r.WindowId == WindowId).ExecuteDeleteAsync(); + await db.EgressRuns.Where(r => r.WindowId == WindowId).ExecuteDeleteAsync(); + } + + private DevEgressController Build(IRequestStateBlobClient journeys, IEgressBlobClient blobs) => + new(new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["Dev:ToolsEnabled"] = "true" }).Build(), + fixture.CreateContext(), journeys, blobs, DevelopmentEnvironment()); + + private static IHostEnvironment DevelopmentEnvironment() + { + var env = Substitute.For(); + env.EnvironmentName.Returns("Development"); + return env; + } + + [Fact] + public async Task Seed_creates_committed_requests_with_dev_zendesk_tickets_when_allowed() + { + await ResetAsync(); + var journeys = Substitute.For(); + var controller = Build(journeys, Substitute.For()); + + var result = await controller.Seed(WindowId, "RemoveLearners", "auto_approved", 2, "860/4070", 142313, "pupil-died", CancellationToken.None); + + Assert.IsType(result); + await using var db = fixture.CreateContext(); + var requests = await db.ChangeRequests.Where(r => r.WindowId == WindowId).ToListAsync(); + Assert.Equal(2, requests.Count); + Assert.All(requests, r => Assert.Equal(RequestStatus.SubmittedCommitted, r.Status)); + Assert.All(requests, r => Assert.NotNull(r.CrmId)); + var references = requests.Select(r => r.ReferenceNumber).ToList(); + var tickets = await db.DevZendeskTickets.Where(t => references.Contains(t.ReferenceNumber)).ToListAsync(); + Assert.Equal(2, tickets.Count); + await journeys.Received(2).SaveAsync(WindowId, Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task Cleanup_removes_everything_the_seeder_wrote_for_the_window() + { + await ResetAsync(); + var journeys = Substitute.For(); + await Build(journeys, Substitute.For()) + .Seed(WindowId, "RemoveLearners", "auto_approved", 1, "860/4070", 142313, "pupil-died", CancellationToken.None); + + var result = await Build(journeys, Substitute.For()).Cleanup(WindowId, CancellationToken.None); + + Assert.IsType(result); + await using var db = fixture.CreateContext(); + Assert.Empty(await db.ChangeRequests.Where(r => r.WindowId == WindowId).ToListAsync()); + await journeys.Received(1).DeleteAsync(WindowId, Arg.Any()); + } + + // Nit: Cleanup previously deleted blobs by file name alone, which could remove another run's + // blob if two windows' files happened to collide on name (Q3). It must delete only a blob this + // run actually owns, via the same egressRunId-metadata check the M1 sweep uses. + [Fact] + public async Task Cleanup_deletes_a_blob_using_its_owning_runs_id_not_just_its_file_name() + { + await ResetAsync(); + var runId = Guid.NewGuid(); + const string fileName = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"; + await using (var db = fixture.CreateContext()) + { + db.EgressRuns.Add(new EgressRun + { + Id = runId, WindowId = WindowId, Status = EgressRunStatus.TransferFailed, + StartedById = Guid.NewGuid(), StartedByName = "Ops One", StartedAtUtc = DateTime.UtcNow + }); + db.EgressRunOutputs.Add(new EgressRunOutput + { + Id = Guid.NewGuid(), RunId = runId, WindowId = WindowId, OutputType = EgressOutputType.RemoveLearners, + IsActive = false, RawRecordsJson = "[]", SourceRecordCount = 0, FileName = fileName + }); + await db.SaveChangesAsync(); + } + var blobs = Substitute.For(); + blobs.IsConfigured.Returns(true); + blobs.DeleteIfOwnedByRunAsync(fileName, runId, Arg.Any()).Returns(true); + + var result = await Build(Substitute.For(), blobs).Cleanup(WindowId, CancellationToken.None); + + Assert.IsType(result); + await blobs.Received(1).DeleteIfOwnedByRunAsync(fileName, runId, Arg.Any()); + await blobs.DidNotReceiveWithAnyArgs().DeleteIfExistsAsync(default!, default); + } + + // The blob sweep ran before the row deletes and threw straight out of the action, so on an + // environment whose egress account was unreachable every cleanup answered 500 and left the + // runs behind — and a leftover run is what stopped the dev seeder (and so the pod) starting on + // the next deploy. A dev reset must reset: the database rows go whatever the blob store did. + [Fact] + public async Task Cleanup_still_removes_the_runs_when_the_blob_sweep_fails() + { + await ResetAsync(); + var runId = Guid.NewGuid(); + await using (var db = fixture.CreateContext()) + { + db.EgressRuns.Add(new EgressRun + { + Id = runId, WindowId = WindowId, Status = EgressRunStatus.Preprocessed, + StartedById = Guid.NewGuid(), StartedByName = "Ops One", StartedAtUtc = DateTime.UtcNow + }); + db.EgressRunOutputs.Add(new EgressRunOutput + { + Id = Guid.NewGuid(), RunId = runId, WindowId = WindowId, OutputType = EgressOutputType.RemoveLearners, + IsActive = true, RawRecordsJson = "[]", SourceRecordCount = 0, + FileName = "CYPMD_LDS_KS4_RemoveLearners_2026_09_17.csv" + }); + await db.SaveChangesAsync(); + } + var blobs = Substitute.For(); + blobs.IsConfigured.Returns(true); + blobs.DeleteIfOwnedByRunAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => throw new HttpRequestException("Connection refused (127.0.0.1:10000)")); + + var result = await Build(Substitute.For(), blobs).Cleanup(WindowId, CancellationToken.None); + + Assert.IsType(result); + await using var after = fixture.CreateContext(); + Assert.False(await after.EgressRuns.AnyAsync(r => r.Id == runId), "the run survived a cleanup whose blob sweep failed"); + } +} diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevOutboxEgressTicketSourceTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevOutboxEgressTicketSourceTests.cs new file mode 100644 index 000000000..bac8567ee --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/DevOutboxEgressTicketSourceTests.cs @@ -0,0 +1,70 @@ +using System.Text.Json; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using DfE.CheckPerformanceData.Persistence.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +namespace DfE.CheckPerformanceData.IntegrationTests.Egress; + +// The fake reads the worker's dev outbox: the decision custom field when the ticket carried one, +// else the subject prefix the ticket builder always writes. Without this the dev/E2E stack, where +// Zendesk:UseFake defaults to true, could never pull. +// +// Deviation from the plan: the unit-test project has no in-memory-EF pattern and must not gain a +// MockQueryable dependency, so — per the plan's own fallback — this class lives here against real +// Postgres via PostgresFixture rather than in the unit-test project against a substituted context. +[Collection(nameof(PostgresCollection))] +public sealed class DevOutboxEgressTicketSourceTests(PostgresFixture fixture) +{ + private const long DecisionFieldId = 19056253670034; + + private static DevZendeskTicket Ticket(long id, string subject, params (long FieldId, object? Value)[] fields) => new() + { + Id = Guid.NewGuid(), TicketId = id, Subject = subject, CreatedAtUtc = DateTime.UtcNow, + RawJson = JsonSerializer.Serialize(new CreateTicketRequestDto + { + Ticket = new CreateTicketDto { Subject = subject, CustomFields = fields.Select(f => new CustomFieldDto { Id = f.FieldId, Value = f.Value }).ToList() } + }) + }; + + private async Task SeedAsync(params DevZendeskTicket[] rows) + { + await using var db = fixture.CreateContext(); + var ids = rows.Select(r => r.TicketId).ToList(); + await db.DevZendeskTickets.Where(t => ids.Contains(t.TicketId)).ExecuteDeleteAsync(); + db.DevZendeskTickets.AddRange(rows); + await db.SaveChangesAsync(); + } + + [Fact] + public async Task Prefers_the_decision_field_and_falls_back_to_the_subject() + { + await SeedAsync( + Ticket(10, "CPMD Requires Scrutiny: NotOnRoll (REF-A)", (DecisionFieldId, "approved")), + Ticket(11, "CPMD Auto-Approved: Deceased (REF-B)"), + Ticket(12, "CPMD Auto-Rejected: Inclusion (REF-C)"), + Ticket(13, "CPMD Requires Scrutiny: Other (REF-D)")); + var sut = new DevOutboxEgressTicketSource(fixture.CreateContext(), Options.Create(new ZendeskTicketFieldSettings { DecisionStatusId = DecisionFieldId })); + + var result = await sut.GetDecisionStatusesAsync([10, 11, 12, 13, 14], CancellationToken.None); + + Assert.Equal("approved", result[10]); + Assert.Equal("auto_approved", result[11]); + Assert.Equal("auto_rejected", result[12]); + Assert.Equal("scrutiny", result[13]); + Assert.False(result.ContainsKey(14)); + } + + [Fact] + public async Task Uses_the_well_known_dev_field_id_when_none_is_configured() + { + await SeedAsync(Ticket(20, "CPMD Requires Scrutiny: NotOnRoll (REF-E)", (DecisionFieldId, "rejected"))); + var sut = new DevOutboxEgressTicketSource(fixture.CreateContext(), Options.Create(new ZendeskTicketFieldSettings())); + + var result = await sut.GetDecisionStatusesAsync([20], CancellationToken.None); + + Assert.Equal("rejected", result[20]); + } +} diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressBlobClientTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressBlobClientTests.cs new file mode 100644 index 000000000..5944083a8 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressBlobClientTests.cs @@ -0,0 +1,113 @@ +using System.Text; +using Azure.Storage.Blobs; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using Microsoft.Extensions.Options; + +namespace DfE.CheckPerformanceData.IntegrationTests.Egress; + +[Collection(nameof(AzuriteCollection))] +public sealed class EgressBlobClientTests(AzuriteFixture fixture) +{ + private readonly BlobServiceClient _blobs = new(fixture.ConnectionString); + private static readonly EgressStorageOptions Options = new() { Container = "cypmd", Prefix = "extracts_input/" }; + + private EgressBlobClient Sut(bool configured = true) => new( + configured ? new Dictionary { ["egress"] = _blobs } : new Dictionary(), + Microsoft.Extensions.Options.Options.Create(Options)); + + [Fact] + public async Task Uploads_under_the_prefix_with_csv_content_type_and_hash_metadata_and_never_overwrites() + { + var name = $"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv"; + var sut = Sut(); + + await sut.UploadAsync(name, Encoding.UTF8.GetBytes("A,B\r\n1,2"), "ABC", Guid.NewGuid(), CancellationToken.None); + + var blob = _blobs.GetBlobContainerClient("cypmd").GetBlobClient($"extracts_input/{name}"); + var props = await blob.GetPropertiesAsync(); + Assert.Equal("text/csv", props.Value.ContentType); + Assert.Equal("ABC", props.Value.Metadata["sha256"]); + await Assert.ThrowsAsync(() => sut.UploadAsync(name, [1], "X", Guid.NewGuid(), CancellationToken.None)); + + await sut.DeleteIfExistsAsync(name, CancellationToken.None); + Assert.False(await blob.ExistsAsync()); + await sut.DeleteIfExistsAsync(name, CancellationToken.None); // idempotent + } + + [Fact] + public void Reports_configuration_and_the_target_description() + { + Assert.True(Sut().IsConfigured); + Assert.False(Sut(configured: false).IsConfigured); + Assert.Equal("cypmd/extracts_input", Sut().TargetDescription); + } + + // S3/M1: the metadata-checked delete used by transfer compensation and the Abandon sweep. + [Fact] + public async Task DeleteIfOwnedByRun_deletes_a_blob_stamped_with_that_run_and_returns_true() + { + var runId = Guid.NewGuid(); + var name = $"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv"; + var sut = Sut(); + await sut.UploadAsync(name, Encoding.UTF8.GetBytes("A,B\r\n1,2"), "ABC", runId, CancellationToken.None); + + var removed = await sut.DeleteIfOwnedByRunAsync(name, runId, CancellationToken.None); + + Assert.True(removed); + Assert.False(await _blobs.GetBlobContainerClient("cypmd").GetBlobClient($"extracts_input/{name}").ExistsAsync()); + } + + [Fact] + public async Task DeleteIfOwnedByRun_never_touches_a_blob_stamped_with_a_different_run() + { + var owningRun = Guid.NewGuid(); + var name = $"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv"; + var sut = Sut(); + await sut.UploadAsync(name, Encoding.UTF8.GetBytes("A,B\r\n1,2"), "ABC", owningRun, CancellationToken.None); + + var removed = await sut.DeleteIfOwnedByRunAsync(name, Guid.NewGuid(), CancellationToken.None); + + Assert.False(removed); + Assert.True(await _blobs.GetBlobContainerClient("cypmd").GetBlobClient($"extracts_input/{name}").ExistsAsync()); + } + + [Fact] + public async Task DeleteIfOwnedByRun_returns_false_for_a_blob_that_does_not_exist() + { + var removed = await Sut().DeleteIfOwnedByRunAsync($"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv", Guid.NewGuid(), CancellationToken.None); + Assert.False(removed); + } + + // Follow-up (Abandon crash-window orphan): transfer needs to know WHO stamped a colliding + // file before deciding whether it may be reclaimed. + [Fact] + public async Task GetOwnerRunId_returns_the_run_stamped_on_the_blob() + { + var runId = Guid.NewGuid(); + var name = $"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv"; + var sut = Sut(); + await sut.UploadAsync(name, Encoding.UTF8.GetBytes("A,B\r\n1,2"), "ABC", runId, CancellationToken.None); + + Assert.Equal(runId, await sut.GetOwnerRunIdAsync(name, CancellationToken.None)); + } + + [Fact] + public async Task GetOwnerRunId_is_null_for_a_blob_that_does_not_exist() + { + Assert.Null(await Sut().GetOwnerRunIdAsync($"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv", CancellationToken.None)); + } + + [Fact] + public async Task GetOwnerRunId_is_null_for_a_blob_this_service_never_stamped() + { + var name = $"CYPMD_LDS_KS4_RemoveLearners_{Guid.NewGuid():N}.csv"; + var container = _blobs.GetBlobContainerClient("cypmd"); + await container.CreateIfNotExistsAsync(); + await container.GetBlobClient($"extracts_input/{name}").UploadAsync(new BinaryData("A,B\r\n1,2")); + + Assert.Null(await Sut().GetOwnerRunIdAsync(name, CancellationToken.None)); + Assert.False(await Sut().DeleteIfOwnedByRunAsync(name, Guid.NewGuid(), CancellationToken.None)); + } +} diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressRunRepositoryTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressRunRepositoryTests.cs new file mode 100644 index 000000000..8c0f28803 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressRunRepositoryTests.cs @@ -0,0 +1,405 @@ +using DfE.CheckPerformance.Persistence.Entities; +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using DfE.CheckPerformanceData.Persistence.Entities; +using DfE.CheckPerformanceData.Persistence.Repositories; +using Microsoft.EntityFrameworkCore; + +namespace DfE.CheckPerformanceData.IntegrationTests.Egress; + +[Collection(nameof(PostgresCollection))] +public sealed class EgressRunRepositoryTests(PostgresFixture fixture) +{ + private static readonly Guid WindowId = Guid.Parse("A0000000-0000-0000-0000-00000000E602"); + private static readonly Guid UserId = Guid.Parse("A0000000-0000-0000-0000-00000000E6AA"); + + private EgressRunRepository Repository() => new(fixture.CreateContext()); + + private async Task ResetAsync() + { + await using var db = fixture.CreateContext(); + if (!await db.CheckingWindows.AnyAsync(w => w.Id == WindowId)) + { + db.CheckingWindows.Add(new CheckingWindow + { + Id = WindowId, Title = "Egress repo window", KeyStage = KeyStages.KS4, + CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + await db.SaveChangesAsync(); + } + await db.EgressRuns.Where(r => r.WindowId == WindowId).ExecuteDeleteAsync(); + await db.ChangeRequests.Where(r => r.WindowId == WindowId).ExecuteDeleteAsync(); + } + + private static EgressSourceRecord Record(string reference, long? ticket, string decision) => new() + { + ChangeRequestId = Guid.NewGuid(), ReferenceNumber = reference, TicketId = ticket, Decision = decision, + OutputType = EgressOutputType.RemoveLearners, WindowType = CheckingWindowType.KS4June, + SubmittedAtUtc = new DateTime(2026, 6, 5, 9, 0, 0, DateTimeKind.Utc), OrganisationUrn = 142313, + OrganisationLaestab = "860/4070", PupilSurname = "Smith", PupilFirstname = "Alice", JourneyFound = true + }; + + private static EgressRunCreate Create(params EgressOutputType[] types) => new( + WindowId, UserId, "Ops One", "ops.one@education.gov.uk", + types.Select(t => new EgressRunOutputCreate(t, [Record("REF-1", 1001, "auto_approved"), Record("REF-2", 1002, "rejected")])).ToList()); + + [Fact] + public async Task Create_then_get_round_trips_the_raw_records() + { + await ResetAsync(); + var repo = Repository(); + + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + var run = await repo.GetRunAsync(id, CancellationToken.None); + + Assert.NotNull(run); + Assert.Equal(EgressRunStatus.Pulled, run!.Status); + var output = Assert.Single(run.Outputs); + Assert.True(output.IsActive); + Assert.Equal(2, output.SourceRecordCount); + Assert.Equal(["REF-1", "REF-2"], output.Records.Select(r => r.ReferenceNumber)); + Assert.Equal("auto_approved", output.Records[0].Decision); + } + + [Fact] + public async Task A_second_run_for_the_same_pair_is_refused_and_names_the_blocker() + { + await ResetAsync(); + var repo = Repository(); + var first = await repo.CreateRunAsync(Create(EgressOutputType.NewLearners), CancellationToken.None); + + var blocker = await repo.FindBlockerAsync(WindowId, EgressOutputType.NewLearners, CancellationToken.None); + Assert.NotNull(blocker); + Assert.Equal(first, blocker!.RunId); + Assert.Equal("Ops One", blocker.StartedByName); + Assert.Equal(EgressRunStatus.Pulled, blocker.Status); + + await Assert.ThrowsAsync(() => + Repository().CreateRunAsync(Create(EgressOutputType.NewLearners), CancellationToken.None)); + Assert.Null(await repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, CancellationToken.None)); + } + + [Fact] + public async Task Preprocessing_failure_releases_the_pair_and_keeps_the_failures() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + + await repo.MarkPreprocessingFailedAsync(id, EgressRunStatus.Pulled, + [new EgressRecordFailure("Split DfE establishment number", 1001, "REF-1", "Local_Authority", "must be 3 digits")], + CancellationToken.None); + + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.PreprocessingFailed, run!.Status); + Assert.Single(run.Failures); + Assert.Equal("Local_Authority", run.Failures[0].Field); + Assert.Null(await repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, CancellationToken.None)); + } + + // M2: the repository side of "no re-run" — a failed run's own outputs stay inactive forever + // (EgressPreprocessor now refuses to touch them again), and a fresh run for the same pair is + // admitted rather than blocked, exactly as the Failed page's "start a new run" copy promises. + [Fact] + public async Task A_fresh_run_for_the_same_pair_is_admitted_after_a_preprocessing_failure() + { + await ResetAsync(); + var repo = Repository(); + var failed = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + await repo.MarkPreprocessingFailedAsync(failed, EgressRunStatus.Pulled, + [new EgressRecordFailure("Split DfE establishment number", 1001, "REF-1", "Local_Authority", "must be 3 digits")], + CancellationToken.None); + + var retry = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + + var failedRun = await repo.GetRunAsync(failed, CancellationToken.None); + Assert.All(failedRun!.Outputs, o => Assert.False(o.IsActive)); + var blocker = await repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, CancellationToken.None); + Assert.Equal(retry, blocker!.RunId); + } + + // S2: EnableRetryOnFailure re-runs the whole execution-strategy delegate on a transient fault. + // A prior attempt's AddRange calls leave their (never-persisted) entities tracked as Added; if + // the retry's AddRange runs again without clearing the tracker first, both sets of entities get + // saved — 2N learner rows for an N-row preprocessing run, which is what LDS receives as the + // file. Reproduced here by tracking a stale entity by hand rather than forcing a real transient + // Postgres fault, which the same ChangeTracker.Clear() at the top of the delegate must discard. + [Fact] + public async Task A_stale_tracked_entity_left_by_an_earlier_attempt_is_not_saved_alongside_the_real_one() + { + await ResetAsync(); + var context = fixture.CreateContext(); + var repo = new EgressRunRepository(context); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + context.EgressRemoveLearners.Add(new EgressRemoveLearner + { + Id = Guid.NewGuid(), RunId = id, ChangeRequestId = Guid.NewGuid(), TicketId = 9999, ReferenceNumber = "STALE", + CorrectionId = "9999", CorrectionType = "31", CorrectionReason = "4", KeyStage = "KS4", EstablishmentNumber = "4070", + Surname = "Stale", Forename = "Entity", Sex = "F", DateOfBirth = "2010-01-01", CycleYear = "2026", CycleMonth = "6", + LocalAuthority = "860", LearnerId = "555" + }); + var remove = new RemoveLearnerRow("1001", "31", "4", "KS4", "4070", "Smith", "Alice", "F", "2010-09-07", "2026", "6", "860", "555", Guid.NewGuid(), 1001, "REF-1"); + var names = new Dictionary { [EgressOutputType.RemoveLearners] = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv" }; + + await repo.SavePreprocessedAsync(id, EgressRunStatus.Pulled, [], [remove], new DateOnly(2026, 6, 8), names, CancellationToken.None); + + var saved = await repo.GetRemoveLearnersAsync(id, CancellationToken.None); + var only = Assert.Single(saved); + Assert.Equal("REF-1", only.ReferenceNumber); + } + + // S2 (second-pass nit): MarkTransferFailedAsync also writes an AuditEntry inside its retried + // delegate but was missing the ChangeTracker.Clear() that SavePreprocessedAsync and + // MarkTransferredAsync already have — the same retry-duplication hazard as the fact above, + // reproduced the same way: a stale tracked AuditEntry left by an earlier attempt must not be + // saved alongside the real one. + [Fact] + public async Task A_stale_tracked_audit_entry_left_by_an_earlier_attempt_is_not_saved_alongside_the_real_one() + { + await ResetAsync(); + var context = fixture.CreateContext(); + var repo = new EgressRunRepository(context); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + context.AuditEntries.Add(new AuditEntry + { + EntityType = "EgressRun", EntityId = id.ToString(), Action = "TransferFailed", + Timestamp = DateTime.UtcNow, UserId = UserId.ToString(), NewValues = "{}" + }); + + await repo.MarkTransferFailedAsync(id, EgressRunStatus.Pulled, "Blob upload refused", UserId.ToString(), CancellationToken.None); + + await using var db = fixture.CreateContext(); + Assert.Equal(1, await db.AuditEntries.CountAsync(a => a.EntityType == "EgressRun" && a.EntityId == id.ToString() && a.Action == "TransferFailed")); + } + + [Fact] + public async Task Save_preprocessed_writes_rows_file_names_and_export_date_in_one_go() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners, EgressOutputType.NewLearners), CancellationToken.None); + var remove = new RemoveLearnerRow("1001", "31", "4", "KS4", "4070", "Smith", "Alice", "F", "2010-09-07", "2026", "6", "860", "555", Guid.NewGuid(), 1001, "REF-1") + { YearGroup = "12", RemovalYear0 = "TRUE", RemovalYear1 = "FALSE", RemovalYear2 = "" }; + var add = new NewLearnerRow("1003", "10", "KS4", "860", "4070", "Jones", "Bob", "M", "2010-01-02", "2018-09-04", "", "2026", "6", "142313", "", "A860407000011", "", "10", Guid.NewGuid(), 1003, "REF-3"); + var names = new Dictionary + { + [EgressOutputType.RemoveLearners] = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", + [EgressOutputType.NewLearners] = "CYPMD_LDS_KS4_NewLearners_2026_06_08.csv" + }; + + await repo.SavePreprocessedAsync(id, EgressRunStatus.Pulled, [add], [remove], new DateOnly(2026, 6, 8), names, CancellationToken.None); + + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.Preprocessed, run!.Status); + Assert.Equal(new DateOnly(2026, 6, 8), run.ExportDate); + Assert.Equal("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", run.Outputs.Single(o => o.OutputType == EgressOutputType.RemoveLearners).FileName); + Assert.Equal(1, run.Outputs.Single(o => o.OutputType == EgressOutputType.NewLearners).OutputRecordCount); + Assert.Equal(remove, Assert.Single(await repo.GetRemoveLearnersAsync(id, CancellationToken.None))); + Assert.Equal(add, Assert.Single(await repo.GetNewLearnersAsync(id, CancellationToken.None))); + + // Saving again (a re-run of preprocessing) replaces rather than duplicates. + await repo.SavePreprocessedAsync(id, EgressRunStatus.Preprocessed, [add], [remove], new DateOnly(2026, 6, 8), names, CancellationToken.None); + Assert.Single(await repo.GetRemoveLearnersAsync(id, CancellationToken.None)); + } + + [Fact] + public async Task Transferred_writes_the_audit_row_and_keeps_the_pair_blocked_forever() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + var audit = new EgressTransferAudit(UserId.ToString(), "Ops One", "cypmd/extracts_input", + new Dictionary { [EgressOutputType.RemoveLearners] = ("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", 1, "ABC") }); + + await repo.MarkTransferredAsync(id, EgressRunStatus.Pulled, audit, new DateTime(2026, 6, 8, 14, 38, 0, DateTimeKind.Utc), CancellationToken.None); + + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.Transferred, run!.Status); + Assert.Equal("Ops One", run.TransferredByName); + Assert.Equal("ABC", run.Outputs[0].Sha256); + var blocker = await repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, CancellationToken.None); + Assert.Equal(EgressRunStatus.Transferred, blocker!.Status); + Assert.NotNull(blocker.TransferredAtUtc); + + await using var db = fixture.CreateContext(); + var entry = await db.AuditEntries.SingleAsync(a => a.EntityType == "EgressRun" && a.EntityId == id.ToString() && a.Action == "Transfer"); + Assert.Contains("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", entry.NewValues); + Assert.Equal(UserId.ToString(), entry.UserId); + } + + [Fact] + public async Task Transfer_failure_releases_the_pair_and_a_retry_can_reactivate_it_unless_taken() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + + await repo.MarkTransferFailedAsync(id, EgressRunStatus.Pulled, "Blob upload refused", UserId.ToString(), CancellationToken.None); + Assert.Null(await repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, CancellationToken.None)); + Assert.Equal("Blob upload refused", (await repo.GetRunAsync(id, CancellationToken.None))!.TransferFailureReason); + + Assert.Null(await repo.TryReactivateAsync(id, CancellationToken.None)); // retry allowed + await repo.MarkTransferFailedAsync(id, EgressRunStatus.TransferFailed, "again", UserId.ToString(), CancellationToken.None); + + var newer = await Repository().CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + var blocked = await repo.TryReactivateAsync(id, CancellationToken.None); // pair taken + Assert.Equal(EgressOutputType.RemoveLearners, blocked!.Value.OutputType); + Assert.Equal(newer, blocked.Value.Blocker.RunId); + + await using var db = fixture.CreateContext(); + Assert.Equal(2, await db.AuditEntries.CountAsync(a => a.EntityType == "EgressRun" && a.EntityId == id.ToString() && a.Action == "TransferFailed")); + } + + // S1: RawRecordsJson (the pulled payload — names, DOB, sex, UPN, every journey answer) must not + // be copied into AuditEntries.NewValues on insert; only the run-level Transfer/TransferFailed + // audit row (already asserted above) is the audit record this feature writes. Same rationale as + // the existing EgressNewLearner/EgressRemoveLearner exemption, extended to the row that carries + // the raw pull. + [Fact] + public async Task Creating_a_run_and_saving_preprocessed_rows_writes_no_audit_entry_for_any_egress_row() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + var remove = new RemoveLearnerRow("1001", "31", "4", "KS4", "4070", "Smith", "Alice", "F", "2010-09-07", "2026", "6", "860", "555", Guid.NewGuid(), 1001, "REF-1"); + var names = new Dictionary { [EgressOutputType.RemoveLearners] = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv" }; + await repo.SavePreprocessedAsync(id, EgressRunStatus.Pulled, [], [remove], new DateOnly(2026, 6, 8), names, CancellationToken.None); + + await using var db = fixture.CreateContext(); + Assert.False(await db.AuditEntries.AnyAsync(a => + a.EntityType == "EgressRunOutput" || a.EntityType == "EgressNewLearner" || a.EntityType == "EgressRemoveLearner")); + } + + // M4: every terminal write is guarded by the expected status it requires, so a run that has + // moved on (e.g. Abandoned) since the caller last read it can never be silently overwritten. + [Fact] + public async Task MarkTransferred_after_abandon_is_a_no_op() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + Assert.Equal(1, await repo.AbandonAsync(id, CancellationToken.None)); + var audit = new EgressTransferAudit(UserId.ToString(), "Ops One", "cypmd/extracts_input", + new Dictionary { [EgressOutputType.RemoveLearners] = ("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", 1, "ABC") }); + + var rows = await repo.MarkTransferredAsync(id, EgressRunStatus.Transferring, audit, DateTime.UtcNow, CancellationToken.None); + + Assert.Equal(0, rows); + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.Abandoned, run!.Status); + Assert.Null(run.Outputs[0].FileName); + await using var db = fixture.CreateContext(); + Assert.False(await db.AuditEntries.AnyAsync(a => a.EntityType == "EgressRun" && a.EntityId == id.ToString() && a.Action == "Transfer")); + } + + [Fact] + public async Task Abandon_after_transferred_is_a_no_op() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + var audit = new EgressTransferAudit(UserId.ToString(), "Ops One", "cypmd/extracts_input", + new Dictionary { [EgressOutputType.RemoveLearners] = ("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", 1, "ABC") }); + Assert.Equal(1, await repo.MarkTransferredAsync(id, EgressRunStatus.Pulled, audit, DateTime.UtcNow, CancellationToken.None)); + + var rows = await repo.AbandonAsync(id, CancellationToken.None); + + Assert.Equal(0, rows); + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.Transferred, run!.Status); + Assert.True(run.Outputs[0].IsActive); + } + + // M4: a run stuck in Preprocessing (a pod restart mid-pipeline) must always be releasable — + // the lock has no expiry and no other override. + [Fact] + public async Task Abandon_admits_a_run_stuck_in_preprocessing() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + Assert.True(await repo.TrySetStatusAsync(id, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, CancellationToken.None)); + + var rows = await repo.AbandonAsync(id, CancellationToken.None); + + Assert.Equal(1, rows); + Assert.Equal(EgressRunStatus.Abandoned, (await repo.GetRunAsync(id, CancellationToken.None))!.Status); + } + + [Fact] + public async Task SavePreprocessed_after_abandon_is_a_no_op() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + Assert.Equal(1, await repo.AbandonAsync(id, CancellationToken.None)); + var remove = new RemoveLearnerRow("1001", "31", "4", "KS4", "4070", "Smith", "Alice", "F", "2010-09-07", "2026", "6", "860", "555", Guid.NewGuid(), 1001, "REF-1"); + var names = new Dictionary { [EgressOutputType.RemoveLearners] = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv" }; + + var rows = await repo.SavePreprocessedAsync(id, EgressRunStatus.Preprocessing, [], [remove], new DateOnly(2026, 6, 8), names, CancellationToken.None); + + Assert.Equal(0, rows); + var run = await repo.GetRunAsync(id, CancellationToken.None); + Assert.Equal(EgressRunStatus.Abandoned, run!.Status); + Assert.Empty(await repo.GetRemoveLearnersAsync(id, CancellationToken.None)); + } + + [Fact] + public async Task Candidates_are_the_windows_amendments_of_that_type_whatever_their_ticket_state() + { + await ResetAsync(); + await using (var db = fixture.CreateContext()) + { + ChangeRequest Row(string reference, WhatToChange type, RequestStatus status, string? crm) => new() + { + Id = Guid.NewGuid(), WindowId = WindowId, OrganisationUrn = 142313, OrganisationLaestab = "860/4070", + Submitted = new DateTime(2026, 6, 5, 9, 0, 0), SubmittedById = UserId, SubmittedByName = "School User", + Status = status, ReferenceNumber = reference, RequestType = RequestType.Amendment, + RequestTypeDescription = type.ToString(), AmendmentType = type, CrmId = crm + }; + db.ChangeRequests.AddRange( + Row("C-REMOVE-COMMITTED", WhatToChange.Remove, RequestStatus.SubmittedCommitted, "2001"), + Row("C-REMOVE-UNCOMMITTED", WhatToChange.Remove, RequestStatus.SubmittedUnCommitted, null), + Row("C-REMOVE-WITHDRAWN", WhatToChange.Remove, RequestStatus.Withdrawn, null), + Row("C-ADD", WhatToChange.Add, RequestStatus.SubmittedCommitted, "2002")); + await db.SaveChangesAsync(); + } + + var removes = await Repository().GetCandidateRequestsAsync(WindowId, WhatToChange.Remove, CancellationToken.None); + + Assert.Equal(["C-REMOVE-COMMITTED", "C-REMOVE-UNCOMMITTED"], removes.Select(r => r.ReferenceNumber).Order()); + Assert.Equal("860/4070", removes[0].OrganisationLaestab); + } + + [Fact] + public async Task List_returns_newest_first_with_window_title_and_output_types() + { + await ResetAsync(); + var repo = Repository(); + var older = await repo.CreateRunAsync(Create(EgressOutputType.NewLearners), CancellationToken.None); + await repo.AbandonAsync(older, CancellationToken.None); + var newer = await repo.CreateRunAsync(Create(EgressOutputType.NewLearners, EgressOutputType.RemoveLearners), CancellationToken.None); + + var list = (await repo.ListRunsAsync(CancellationToken.None)).Where(r => r.WindowId == WindowId).ToList(); + + Assert.Equal([newer, older], list.Select(r => r.Id)); + Assert.Equal("Egress repo window", list[0].WindowTitle); + Assert.Equal([EgressOutputType.NewLearners, EgressOutputType.RemoveLearners], list[0].OutputTypes); + Assert.Equal(EgressRunStatus.Abandoned, list[1].Status); + } + + [Fact] + public async Task Status_transitions_are_conditional_on_the_expected_current_state() + { + await ResetAsync(); + var repo = Repository(); + var id = await repo.CreateRunAsync(Create(EgressOutputType.RemoveLearners), CancellationToken.None); + + Assert.True(await repo.TrySetStatusAsync(id, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, CancellationToken.None)); + Assert.False(await repo.TrySetStatusAsync(id, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, CancellationToken.None)); + Assert.Equal(EgressRunStatus.Preprocessing, (await repo.GetRunAsync(id, CancellationToken.None))!.Status); + } +} diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressSchemaTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressSchemaTests.cs new file mode 100644 index 000000000..c0a679660 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Egress/EgressSchemaTests.cs @@ -0,0 +1,96 @@ +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using DfE.CheckPerformanceData.Persistence.Entities; +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace DfE.CheckPerformanceData.IntegrationTests.Egress; + +// The concurrency rule of AB#294553 — one egress run per window and output type at a time, and +// never a second one after a successful transfer — is a database constraint, not a service +// check. These pin the constraint itself, so a race the service check misses is still refused. +[Collection(nameof(PostgresCollection))] +public sealed class EgressSchemaTests(PostgresFixture fixture) +{ + private static readonly Guid WindowId = Guid.Parse("A0000000-0000-0000-0000-00000000E601"); + + private async Task EnsureWindowAsync() + { + await using var db = fixture.CreateContext(); + if (await db.CheckingWindows.AnyAsync(w => w.Id == WindowId)) return; + db.CheckingWindows.Add(new CheckingWindow + { + Id = WindowId, Title = "Egress schema window", KeyStage = KeyStages.KS4, + CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + await db.SaveChangesAsync(); + } + + private static EgressRun Run(EgressRunStatus status, bool active, EgressOutputType type) => new() + { + Id = Guid.NewGuid(), WindowId = WindowId, Status = status, + StartedById = Guid.NewGuid(), StartedByName = "Ops One", StartedAtUtc = DateTime.UtcNow, + Outputs = + { + new EgressRunOutput + { + Id = Guid.NewGuid(), WindowId = WindowId, OutputType = type, IsActive = active, + RawRecordsJson = "[]", SourceRecordCount = 0 + } + } + }; + + [Fact] + public async Task A_second_active_output_for_the_same_window_and_type_is_refused_by_the_database() + { + await EnsureWindowAsync(); + var type = EgressOutputType.RemoveLearners; + await using (var db = fixture.CreateContext()) + { + await db.EgressRunOutputs.Where(o => o.WindowId == WindowId && o.OutputType == type).ExecuteDeleteAsync(); + db.EgressRuns.Add(Run(EgressRunStatus.Pulled, active: true, type)); + await db.SaveChangesAsync(); + } + + await using var second = fixture.CreateContext(); + second.EgressRuns.Add(Run(EgressRunStatus.Pulled, active: true, type)); + var ex = await Assert.ThrowsAsync(() => second.SaveChangesAsync()); + var pg = Assert.IsType(ex.InnerException); + Assert.Equal("23505", pg.SqlState); + Assert.Equal("ix_egress_run_outputs_active_window_output", pg.ConstraintName); + } + + [Fact] + public async Task An_inactive_output_does_not_block_a_new_active_one() + { + await EnsureWindowAsync(); + var type = EgressOutputType.NewLearners; + await using var db = fixture.CreateContext(); + await db.EgressRunOutputs.Where(o => o.WindowId == WindowId && o.OutputType == type).ExecuteDeleteAsync(); + db.EgressRuns.Add(Run(EgressRunStatus.PreprocessingFailed, active: false, type)); + db.EgressRuns.Add(Run(EgressRunStatus.Abandoned, active: false, type)); + db.EgressRuns.Add(Run(EgressRunStatus.Pulled, active: true, type)); + await db.SaveChangesAsync(); // no throw + + Assert.Equal(3, await db.EgressRunOutputs.CountAsync(o => o.WindowId == WindowId && o.OutputType == type)); + } + + [Fact] + public async Task Learner_tables_and_the_request_laestab_column_exist() + { + await using var db = fixture.CreateContext(); + await using var conn = new NpgsqlConnection(fixture.ConnectionString); + await conn.OpenAsync(); + await using var cmd = new NpgsqlCommand( + """ + select count(*) from information_schema.tables + where table_name in ('egress_runs','egress_run_outputs','new_learners','remove_learners') + """, conn); + Assert.Equal(4L, (long)(await cmd.ExecuteScalarAsync())!); + + await using var col = new NpgsqlCommand( + """select character_maximum_length from information_schema.columns where table_name = 'ChangeRequests' and column_name = 'OrganisationLaestab'""", conn); + Assert.Equal(20, (int)(await col.ExecuteScalarAsync())!); + } +} diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/Seeding/CheckingWindowSeedWithEgressHistoryTests.cs b/tests/DfE.CheckPerformanceData.IntegrationTests/Seeding/CheckingWindowSeedWithEgressHistoryTests.cs new file mode 100644 index 000000000..4a9375246 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/Seeding/CheckingWindowSeedWithEgressHistoryTests.cs @@ -0,0 +1,69 @@ +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.IntegrationTests.Fixtures; +using DfE.CheckPerformanceData.Persistence.Entities; +using DfE.CheckPerformanceData.Persistence.Seeding; +using Microsoft.EntityFrameworkCore; + +namespace DfE.CheckPerformanceData.IntegrationTests.Seeding; + +// The dev seeder wipes CheckingWindows at start-up wherever SeedDevelopmentData is on — local, +// the deployed DEV app and every review app. egress_runs references CheckingWindows with a +// RESTRICT foreign key, so a single egress run left behind (the E2E cleanup failing part-way is +// enough) made that wipe throw, the host terminated before it listened, and the pod never became +// Ready. On a review app that is invisible: the previous pod keeps serving, terraform times out +// waiting for the rollout, a re-run reports "No changes", and the health check passes against the +// old image. PR #441's review app served the 15 Sep image for two days this way. +// +// So the seed must take the egress history down with the windows it replaces — the same way it +// already takes the change requests. Against real Postgres because the failure is the database's +// referential action, which nothing in-memory would reproduce. +[Collection(nameof(PostgresCollection))] +public sealed class CheckingWindowSeedWithEgressHistoryTests(PostgresFixture fixture) +{ + private static readonly Guid WindowId = Guid.Parse("C1E5D9A2-7B3F-4E60-9D84-2A6F1B7C3E95"); + + [Fact] + public async Task Reseeding_the_windows_succeeds_when_an_egress_run_references_one_of_them() + { + var runId = Guid.NewGuid(); + await using (var db = fixture.CreateContext()) + { + if (!await db.CheckingWindows.AnyAsync(w => w.Id == WindowId)) + { + db.CheckingWindows.Add(new CheckingWindow + { + Id = WindowId, Title = "Window with egress history", KeyStage = KeyStages.KS4, + CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + await db.SaveChangesAsync(); + } + db.EgressRuns.Add(new EgressRun + { + Id = runId, WindowId = WindowId, Status = EgressRunStatus.Preprocessed, + StartedById = Guid.NewGuid(), StartedByName = "Left behind", StartedAtUtc = DateTime.UtcNow + }); + db.EgressRunOutputs.Add(new EgressRunOutput + { + Id = Guid.NewGuid(), RunId = runId, WindowId = WindowId, OutputType = EgressOutputType.RemoveLearners, + IsActive = true, RawRecordsJson = "[]", SourceRecordCount = 0, + FileName = "CYPMD_LDS_KS4_RemoveLearners_2026_09_17.csv" + }); + await db.SaveChangesAsync(); + } + + await using (var db = fixture.CreateContext()) + { + await SeedCheckingWindows.ExecuteSeed(db, + DevDataSeeder.KeyStage4JuneCheckingWindowId, DevDataSeeder.ClosedKeyStage4JuneCheckingWindowId, + DevDataSeeder.Post16CheckingWindowId, DevDataSeeder.ClosedPupilDataPost16CheckingWindowId); + } + + await using (var db = fixture.CreateContext()) + { + Assert.False(await db.EgressRuns.AnyAsync(r => r.Id == runId), "the egress run survived the reseed"); + Assert.False(await db.CheckingWindows.AnyAsync(w => w.Id == WindowId), "the old window survived the reseed"); + Assert.True(await db.CheckingWindows.AnyAsync(w => w.Id == DevDataSeeder.KeyStage4JuneCheckingWindowId)); + } + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Admin/DefaultAdminAccessSeederTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Admin/DefaultAdminAccessSeederTests.cs index fc0712731..d44e353e7 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/Admin/DefaultAdminAccessSeederTests.cs +++ b/tests/DfE.CheckPerformanceData.UnitTests/Admin/DefaultAdminAccessSeederTests.cs @@ -34,6 +34,24 @@ public void AllSections_ContainsMessagesInbox() Assert.Contains("messages-inbox", DefaultAdminAccessSeeder.AllSections); } + // The egress controller is gated on AdminNavKeys.Egress; without a matching grant a fresh-DB + // admin would 404 on /admin/egress (AB#294553). + [Fact] + public void AllSections_ContainsEgress() + { + Assert.Contains(AdminNavKeys.Egress, DefaultAdminAccessSeeder.AllSections); + Assert.Equal("egress", AdminNavKeys.Egress); + } + + // The group is a container only: access is implied by the tile, so it must NOT be a section + // (the same rule as cms-admin / system-admin). + [Fact] + public void AllSections_DoesNotContainTheEgressGroup() + { + Assert.Equal("egress-group", AdminNavKeys.EgressGroup); + Assert.DoesNotContain(AdminNavKeys.EgressGroup, DefaultAdminAccessSeeder.AllSections); + } + // The two lists MUST stay in sync — the seeder's comment above the list literally // says so, and the [RequireAdminSection(AdminNavKeys.X)] gate only lets a user // through if AdminSectionAccess has a matching row for their role. Guards against diff --git a/tests/DfE.CheckPerformanceData.UnitTests/DfE.CheckPerformanceData.Application.UnitTests.csproj b/tests/DfE.CheckPerformanceData.UnitTests/DfE.CheckPerformanceData.Application.UnitTests.csproj index 976386ff2..501a93ee8 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/DfE.CheckPerformanceData.Application.UnitTests.csproj +++ b/tests/DfE.CheckPerformanceData.UnitTests/DfE.CheckPerformanceData.Application.UnitTests.csproj @@ -18,11 +18,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressBlobClientTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressBlobClientTests.cs new file mode 100644 index 000000000..79eda6e68 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressBlobClientTests.cs @@ -0,0 +1,54 @@ +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using Microsoft.Extensions.Options; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// S7: a create-only upload's 409 must be interpreted narrowly — only the conflict the code +// actually handles (a blob already exists) is worth its own exception; anything else (a lease, a +// container mid-delete) must propagate as-is rather than being misreported as "remove it by hand". +public sealed class EgressBlobClientTests +{ + private static readonly EgressStorageOptions Options = new() { Container = "cypmd", Prefix = "extracts_input/" }; + + private static (EgressBlobClient Sut, BlobClient Blob) Build() + { + var service = Substitute.For(); + var container = Substitute.For(); + var blob = Substitute.For(); + service.GetBlobContainerClient("cypmd").Returns(container); + container.CreateIfNotExistsAsync( + Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns((Response?)null); + container.GetBlobClient("extracts_input/CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv").Returns(blob); + var sut = new EgressBlobClient(new Dictionary { ["egress"] = service }, Microsoft.Extensions.Options.Options.Create(Options)); + return (sut, blob); + } + + [Fact] + public async Task A_409_for_an_existing_blob_becomes_the_named_exception() + { + var (sut, blob) = Build(); + blob.UploadAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException>(new RequestFailedException(409, "already exists", nameof(BlobErrorCode.BlobAlreadyExists), null))); + + await Assert.ThrowsAsync( + () => sut.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", [1, 2, 3], "sha", Guid.NewGuid(), CancellationToken.None)); + } + + [Fact] + public async Task A_409_for_any_other_reason_propagates_unchanged() + { + var (sut, blob) = Build(); + blob.UploadAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException>(new RequestFailedException(409, "container is being deleted", "ContainerBeingDeleted", null))); + + var ex = await Assert.ThrowsAsync( + () => sut.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", [1, 2, 3], "sha", Guid.NewGuid(), CancellationToken.None)); + Assert.Equal("ContainerBeingDeleted", ex.ErrorCode); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressColumnSetsTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressColumnSetsTests.cs new file mode 100644 index 000000000..bb9a5a570 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressColumnSetsTests.cs @@ -0,0 +1,124 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// Headings are a contract with LDS: exact text, order and count (AB#292610 "Field headings match +// the spec exactly"). Both sets are the LDS_CYPMD_Data specification v2.4 sheets ("New Learner", +// "Remove Learner") read top to bottom, keeping only rows marked X for the key stage in question. +// Middle_Name is struck through in v2.4 ("CYPMD will not be sending this field from June 2026") +// and is therefore absent. Edit EgressColumnSets and these expectations together, nothing else. +public sealed class EgressColumnSetsTests +{ + private static readonly string[] RemoveBase = + [ + "Correction_ID", "Correction_Type", "Correction_Reason", "Key_Stage", "Establishment_Number", + "Surname", "Forename", "Sex", "Date_of_Birth", "Cycle_Year", "Cycle_Month", "Local_Authority", "Learner_ID" + ]; + + private static readonly string[] NewBase = + [ + "Correction_ID", "Correction_Type", "Key_Stage", "Establishment_Number", "Surname", "Forename", "Sex", + "Date_of_Birth", "Admission_Date", "Post_Code", "Cycle_Year", "Cycle_Month", "Local_Authority", "URN", + "ULN", "UPN", "Learner_ID", "Year_Group" + ]; + + [Fact] + public void Remove_learners_base_headings_are_the_spec_sheet_in_order() + => Assert.Equal(RemoveBase, EgressColumnSets.RemoveLearners.Select(c => c.Header).ToArray()); + + [Fact] + public void New_learners_base_headings_are_the_spec_sheet_in_order_without_the_struck_Middle_Name() + { + var headers = EgressColumnSets.NewLearners.Select(c => c.Header).ToArray(); + Assert.Equal(NewBase, headers); + Assert.Equal(Array.IndexOf(headers, "ULN") + 1, Array.IndexOf(headers, "UPN")); + Assert.Equal(Array.IndexOf(headers, "UPN") + 1, Array.IndexOf(headers, "Learner_ID")); + Assert.DoesNotContain("Middle_Name", headers); + Assert.DoesNotContain("SEN_Status", headers); + } + + [Theory] + [InlineData(CheckingWindowType.KS2)] + [InlineData(CheckingWindowType.KS4June)] + [InlineData(CheckingWindowType.KS4Autumn)] + public void New_learners_for_KS2_and_KS4_is_the_base_set(CheckingWindowType windowType) + => Assert.Equal(NewBase, EgressColumnSets.NewLearnersFor(windowType).Select(c => c.Header).ToArray()); + + [Fact] + public void New_learners_for_16_19_appends_the_attendance_years_and_KS4_year() + { + var headers = EgressColumnSets.NewLearnersFor(CheckingWindowType.Post16).Select(c => c.Header).ToArray(); + Assert.Equal([.. NewBase, "Attendance_Year_0", "Attendance_Year_1", "Attendance_Year_2", "KS4_Year"], headers); + // No Post16 Add journey exists yet, so the four extra cells are blank (spec: NULL allowed). + var values = EgressColumnSets.NewLearnersFor(CheckingWindowType.Post16).Select(c => c.Value(SampleRows.New())).ToArray(); + Assert.Equal(["", "", "", ""], values[^4..]); + } + + // v2.4 "Remove Learner": Year_Group is X for KS4 only ("for year group change requests only"); + // Removal_Year_0..2 are X for 16-18 only; KS2 gets neither. + [Fact] + public void Remove_learners_for_KS2_is_the_base_set() + => Assert.Equal(RemoveBase, EgressColumnSets.RemoveLearnersFor(CheckingWindowType.KS2).Select(c => c.Header).ToArray()); + + [Theory] + [InlineData(CheckingWindowType.KS4June)] + [InlineData(CheckingWindowType.KS4Autumn)] + public void Remove_learners_for_KS4_appends_Year_Group(CheckingWindowType windowType) + { + var columns = EgressColumnSets.RemoveLearnersFor(windowType); + Assert.Equal([.. RemoveBase, "Year_Group"], columns.Select(c => c.Header).ToArray()); + Assert.Equal("12", columns[^1].Value(SampleRows.Remove() with { YearGroup = "12" })); + } + + [Fact] + public void Remove_learners_for_16_19_appends_the_three_removal_years() + { + var columns = EgressColumnSets.RemoveLearnersFor(CheckingWindowType.Post16); + Assert.Equal([.. RemoveBase, "Removal_Year_0", "Removal_Year_1", "Removal_Year_2"], columns.Select(c => c.Header).ToArray()); + var row = SampleRows.Remove() with { RemovalYear0 = "TRUE", RemovalYear1 = "FALSE", RemovalYear2 = "TRUE" }; + Assert.Equal(["TRUE", "FALSE", "TRUE"], columns.Skip(RemoveBase.Length).Select(c => c.Value(row)).ToArray()); + } + + [Fact] + public void No_heading_carries_a_trailing_underscore_or_whitespace() + { + foreach (var header in EgressColumnSets.RemoveLearners.Select(c => c.Header) + .Concat(EgressColumnSets.NewLearnersFor(CheckingWindowType.Post16).Select(c => c.Header))) + { + Assert.Equal(header.Trim(), header); + Assert.False(header.EndsWith('_'), header); + } + } + + [Fact] + public void Remove_row_projects_every_column_from_the_row() + { + var values = EgressColumnSets.RemoveLearners.Select(c => c.Value(SampleRows.Remove())).ToArray(); + Assert.Equal(["88856", "31", "4", "KS4", "4603", "Bellingham", "Jude", "M", "2007-06-01", "2026", "6", "873", "10000011"], values); + } + + [Fact] + public void New_row_projects_every_column_from_the_row_in_spec_order() + { + var values = EgressColumnSets.NewLearners.Select(c => c.Value(SampleRows.New())).ToArray(); + Assert.Equal(["69390", "10", "KS4", "5412", "Lennox", "Annie", "F", "2010-09-07", "2018-09-04", "", + "2026", "6", "881", "136412", "", "A881541200011", "", "10"], values); + } +} + +internal static class SampleRows +{ + public static RemoveLearnerRow Remove() => new( + CorrectionId: "88856", CorrectionType: "31", CorrectionReason: "4", KeyStage: "KS4", + EstablishmentNumber: "4603", Surname: "Bellingham", Forename: "Jude", Sex: "M", DateOfBirth: "2007-06-01", + CycleYear: "2026", CycleMonth: "6", LocalAuthority: "873", LearnerId: "10000011", + ChangeRequestId: Guid.Parse("11111111-1111-1111-1111-111111111111"), TicketId: 88856, ReferenceNumber: "CYPMD_KS4June_AAA0001"); + + public static NewLearnerRow New() => new( + CorrectionId: "69390", CorrectionType: "10", KeyStage: "KS4", LocalAuthority: "881", EstablishmentNumber: "5412", + Surname: "Lennox", Forename: "Annie", Sex: "F", DateOfBirth: "2010-09-07", AdmissionDate: "2018-09-04", + Postcode: "", CycleYear: "2026", CycleMonth: "6", SchoolUrn: "136412", Uln: "", Upn: "A881541200011", LearnerId: "", + YearGroup: "10", + ChangeRequestId: Guid.Parse("22222222-2222-2222-2222-222222222222"), TicketId: 69390, ReferenceNumber: "CYPMD_KS4June_AAA0002"); +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressCsvWriterTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressCsvWriterTests.cs new file mode 100644 index 000000000..e65ddbfac --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressCsvWriterTests.cs @@ -0,0 +1,44 @@ +using System.Text; +using DfE.CheckPerformanceData.Application.Egress; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// AB#292610 file rules: CSV; no trailing blank characters; no rows below the final data row; no +// trailing comma after the final heading; no extra columns. The writer is the one place those hold. +public sealed class EgressCsvWriterTests +{ + private static readonly IReadOnlyList> Columns = + [ + new("A", r => r[0]), + new("B", r => r[1]) + ]; + + [Fact] + public void Header_then_rows_joined_with_CRLF_and_no_trailing_line_break() + { + var bytes = EgressCsvWriter.Write(Columns, [["1", "2"], ["3", "4"]]); + Assert.Equal("A,B\r\n1,2\r\n3,4", Encoding.UTF8.GetString(bytes)); + } + + [Fact] + public void Header_only_when_there_are_no_rows() + { + var bytes = EgressCsvWriter.Write(Columns, []); + Assert.Equal("A,B", Encoding.UTF8.GetString(bytes)); + } + + [Fact] + public void Values_are_trimmed_and_quoted_only_when_they_need_it() + { + var bytes = EgressCsvWriter.Write(Columns, [[" plain ", "has,comma"], ["has \"quote\"", "line\nbreak"]]); + Assert.Equal("A,B\r\nplain,\"has,comma\"\r\n\"has \"\"quote\"\"\",\"line\nbreak\"", Encoding.UTF8.GetString(bytes)); + } + + [Fact] + public void Utf8_without_a_byte_order_mark() + { + var bytes = EgressCsvWriter.Write(Columns, [["é", "x"]]); + Assert.NotEqual(0xEF, bytes[0]); + Assert.Equal("A,B\r\né,x", Encoding.UTF8.GetString(bytes)); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressOutputTypesTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressOutputTypesTests.cs new file mode 100644 index 000000000..631600048 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressOutputTypesTests.cs @@ -0,0 +1,75 @@ +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// The file name is a contract with LDS (AB#292610): CYPMD_LDS_{stage}_{type}_YYYY_MM_DD. Pinned +// literally because a wrong stage token or separator is a file LDS silently ignores. +public sealed class EgressOutputTypesTests +{ + [Theory] + [InlineData(EgressOutputType.NewLearners, WhatToChange.Add)] + [InlineData(EgressOutputType.RemoveLearners, WhatToChange.Remove)] + public void Each_output_type_maps_to_exactly_one_amendment_journey(EgressOutputType type, WhatToChange expected) + => Assert.Equal(expected, EgressOutputTypes.WhatToChangeFor(type)); + + [Theory] + [InlineData(CheckingWindowType.KS4June, "KS4")] + [InlineData(CheckingWindowType.KS4Autumn, "KS4")] + [InlineData(CheckingWindowType.KS2, "KS2")] + [InlineData(CheckingWindowType.Post16, "KS5")] + public void Stage_token_follows_the_LDS_naming_convention(CheckingWindowType windowType, string expected) + => Assert.Equal(expected, EgressOutputTypes.StageToken(windowType)); + + // The Key_Stage cell is the spec's value (LDS_CYPMD_Data specification v2.4, Remove Learner J12: + // "KS2, KS4, 16-19"); only the FILE NAME uses AB#292610's KS5 token. + [Theory] + [InlineData(CheckingWindowType.KS4June, "KS4")] + [InlineData(CheckingWindowType.KS4Autumn, "KS4")] + [InlineData(CheckingWindowType.KS2, "KS2")] + [InlineData(CheckingWindowType.Post16, "16-19")] + public void Key_stage_cell_value_is_the_spec_value_not_the_file_token(CheckingWindowType windowType, string expected) + => Assert.Equal(expected, EgressOutputTypes.KeyStageValue(windowType)); + + [Fact] + public void File_name_is_the_agreed_convention_with_the_export_date() + { + var name = EgressOutputTypes.FileName(CheckingWindowType.KS4June, EgressOutputType.RemoveLearners, new DateOnly(2026, 6, 8)); + Assert.Equal("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", name); + } + + [Fact] + public void New_learners_file_token_and_correction_type() + { + Assert.Equal("CYPMD_LDS_KS2_NewLearners_2026_10_07.csv", + EgressOutputTypes.FileName(CheckingWindowType.KS2, EgressOutputType.NewLearners, new DateOnly(2026, 10, 7))); + Assert.Equal("10", EgressOutputTypes.CorrectionType(EgressOutputType.NewLearners)); + Assert.Equal("31", EgressOutputTypes.CorrectionType(EgressOutputType.RemoveLearners)); + } + + [Theory] + [InlineData("approved", true)] + [InlineData("auto_approved", true)] + [InlineData("AUTO_APPROVED", true)] + [InlineData("rejected", false)] + [InlineData("auto_rejected", false)] + [InlineData("scrutiny", false)] + [InlineData("no-ticket", false)] + [InlineData(null, false)] + public void Only_approved_and_auto_approved_pass_the_filter(string? decision, bool expected) + => Assert.Equal(expected, EgressDecisions.IsApproved(decision)); + + [Fact] + public void Labels_are_human_readable() + { + Assert.Equal("Auto approved", EgressDecisions.Label("auto_approved")); + Assert.Equal("No Zendesk ticket", EgressDecisions.Label(EgressDecisions.NoTicket)); + Assert.Equal("Ticket not found", EgressDecisions.Label(EgressDecisions.NotFound)); + Assert.Equal("Unknown", EgressDecisions.Label(null)); + } + + [Fact] + public void All_lists_both_supported_types_in_display_order() + => Assert.Equal([EgressOutputType.NewLearners, EgressOutputType.RemoveLearners], EgressOutputTypes.All); +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressPreprocessorTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressPreprocessorTests.cs new file mode 100644 index 000000000..b9eda7999 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressPreprocessorTests.cs @@ -0,0 +1,248 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// The pipeline reports eight steps in a fixed order, keeps only approved records, and is +// all-or-nothing: one failing record means nothing is saved and every failure is listed. +public sealed class EgressPreprocessorTests +{ + private static readonly Guid RunId = Guid.Parse("33333333-3333-3333-3333-333333333333"); + private static readonly Guid WindowId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + private readonly IEgressRunRepository _repo = Substitute.For(); + private readonly IWindowService _windows = Substitute.For(); + private readonly FakeTimeProvider _clock = new(new DateTimeOffset(2026, 6, 8, 13, 35, 0, TimeSpan.Zero)); + + private EgressPreprocessor Sut() => new(_repo, _windows, _clock, Substitute.For>()); + + private static CheckingWindowDto Window(CheckingWindowType type = CheckingWindowType.KS4June) => new() + { + Id = WindowId, Title = "Window", KeyStage = KeyStages.KS4, CheckingWindowType = type, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }; + + private static EgressSourceRecord Remove(string reference, string decision, string reason = "pupil-died") => new() + { + ChangeRequestId = Guid.NewGuid(), ReferenceNumber = reference, TicketId = reference.GetHashCode() & 0xffff, Decision = decision, + OutputType = EgressOutputType.RemoveLearners, WindowType = CheckingWindowType.KS4June, + SubmittedAtUtc = new DateTime(2026, 6, 5, 9, 0, 0, DateTimeKind.Utc), OrganisationUrn = 142313, OrganisationLaestab = "860/4070", + PupilFirstname = "Alice", PupilSurname = "Smith", PupilSex = "F", PupilDateOfBirth = "07/09/2010", PupilIdentifier = "A860407000011", + PupilMatchRef = 555, PupilLaestab = "8604070", JourneyFound = true, Answers = new Dictionary { ["reason"] = reason } + }; + + private void RunIs(EgressRunStatus status, params EgressSourceRecord[] records) + { + _repo.GetRunAsync(RunId, Arg.Any()).Returns(new EgressRunDto(RunId, WindowId, status, Guid.NewGuid(), "Ops One", + DateTime.UtcNow, null, null, null, null, [], null, + [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, records, records.Length, null, null, null)])); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + // M4: the guarded writes now return rows affected — default to "won the race" (1) so + // existing success/failure-path tests are unaffected; the specific lost-the-race test + // overrides this explicitly. + _repo.SavePreprocessedAsync(Arg.Any(), Arg.Any(), Arg.Any>(), + Arg.Any>(), Arg.Any(), Arg.Any>(), Arg.Any()).Returns(1); + _repo.MarkPreprocessingFailedAsync(Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()).Returns(1); + } + + private async Task> Collect() + { + var events = new List(); + await foreach (var e in Sut().RunAsync(RunId, CancellationToken.None)) events.Add(e); + return events; + } + + [Fact] + public async Task Reports_the_eight_steps_in_order_and_saves_only_approved_records() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved"), Remove("R2", "auto_approved"), Remove("R3", "rejected"), Remove("R4", "scrutiny"), Remove("R5", EgressDecisions.NoTicket)); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + + var events = await Collect(); + + Assert.Equal(EgressPreprocessor.StepNames, events.Where(e => e.State == "done").Select(e => e.StepName).ToArray()); + Assert.Equal(8, events.Last().TotalSteps); + var filter = events.Single(e => e.StepName == "Filter records" && e.State == "done"); + Assert.Equal(5, filter.RecordsIn); + Assert.Equal(2, filter.RecordsOut); + var last = events.Last(); + Assert.True(last.IsComplete); + Assert.False(last.IsError); + Assert.Equal(EgressRunStatus.Preprocessed, last.FinalStatus); + await _repo.Received(1).SavePreprocessedAsync(RunId, EgressRunStatus.Preprocessing, Arg.Is>(l => l.Count == 0), + Arg.Is>(l => l.Count == 2 && l.All(r => r.CorrectionReason == "4")), + new DateOnly(2026, 6, 8), + Arg.Is>(d => d[EgressOutputType.RemoveLearners] == "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"), + Arg.Any()); + } + + [Fact] + public async Task Saved_rows_carry_the_windows_cycle_year_and_month() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved") with { SubmittedAtUtc = new DateTime(2026, 7, 5, 9, 0, 0, DateTimeKind.Utc) }); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + IReadOnlyList? saved = null; + _repo.SavePreprocessedAsync(RunId, EgressRunStatus.Preprocessing, Arg.Any>(), + Arg.Do>(r => saved = r), Arg.Any(), Arg.Any>(), Arg.Any()).Returns(1); + + await Collect(); + + var row = Assert.Single(saved!); + Assert.Equal("2026", row.CycleYear); // Window() starts 2026-06-01 + Assert.Equal("6", row.CycleMonth); + } + + [Fact] + public async Task One_failing_record_fails_the_batch_and_saves_nothing() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved"), Remove("R2", "approved", reason: "other")); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + + var events = await Collect(); + + var last = events.Last(); + Assert.True(last.IsComplete); + Assert.True(last.IsError); + Assert.Equal(EgressRunStatus.PreprocessingFailed, last.FinalStatus); + Assert.Equal(1, last.FailureCount); + Assert.Equal("failed", events.Single(e => e.StepName == "Save to database" && e.IsComplete).State); + await _repo.DidNotReceiveWithAnyArgs().SavePreprocessedAsync(default, default, default!, default!, default, default!, default); + await _repo.Received(1).MarkPreprocessingFailedAsync(RunId, EgressRunStatus.Preprocessing, + Arg.Is>(f => f.Count == 1 && f[0].ReferenceNumber == "R2" && f[0].Field == "Correction_Reason"), + Arg.Any()); + } + + // Nit (folded into M3): the file-name stage must come from the window, not be guessed from + // records — a run with zero pulled records (a window with no candidate requests at all) must + // still name its file after the window's own stage, never a hard-coded KS4June default. + [Fact] + public async Task File_name_stage_comes_from_the_window_even_when_the_run_has_no_records() + { + RunIs(EgressRunStatus.Pulled); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window(CheckingWindowType.KS2)); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + + await Collect(); + + await _repo.Received(1).SavePreprocessedAsync(RunId, EgressRunStatus.Preprocessing, Arg.Any>(), Arg.Any>(), + Arg.Any(), Arg.Is>(d => d[EgressOutputType.RemoveLearners].Contains("_KS2_")), + Arg.Any()); + } + + [Fact] + public async Task Export_date_is_the_London_calendar_date() + { + // 23:30 UTC on 8 June is 00:30 BST on 9 June — the file must say 9 June. + _clock.SetUtcNow(new DateTimeOffset(2026, 6, 8, 23, 30, 0, TimeSpan.Zero)); + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved")); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + + await Collect(); + + await _repo.Received(1).SavePreprocessedAsync(RunId, EgressRunStatus.Preprocessing, Arg.Any>(), Arg.Any>(), + new DateOnly(2026, 6, 9), Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task A_run_that_is_not_in_a_runnable_state_ends_with_an_error_event_and_touches_nothing() + { + RunIs(EgressRunStatus.Transferred, Remove("R1", "approved")); + + var events = await Collect(); + + var only = Assert.Single(events); + Assert.True(only.IsError); + Assert.True(only.IsComplete); + await _repo.DidNotReceiveWithAnyArgs().TrySetStatusAsync(default, default, default, default); + } + + // M2: re-running a PreprocessingFailed run bypassed the lock — the failed run releases its + // pair (Failed.cshtml says "start a new run"), so a re-run reaching Preprocessed while a + // colleague's fresh run also holds the pair would let both transfer it. Same terminal refusal + // shape as any other non-runnable status. + // + // Second-pass nit: the refusal text used to interpolate the raw enum member name + // ("PreprocessingFailed"), leaking C# to the end user. It must use the same human label + // Index.cshtml's StageLabel helper shows for this status ("Preprocessing failed") instead. + // + // Final-review nit: interpolating that label straight into "This run is {label} and cannot + // be preprocessed" reads as ungrammatical English for several labels (e.g. "This run is + // Transfer failed and cannot be preprocessed."). Phrased as a stage statement instead, which + // reads correctly for every label EgressRunStatuses.Label can return. + [Fact] + public async Task A_preprocessing_failed_run_is_refused_not_re_run() + { + RunIs(EgressRunStatus.PreprocessingFailed, Remove("R1", "approved")); + + var events = await Collect(); + + var only = Assert.Single(events); + Assert.True(only.IsError); + Assert.True(only.IsComplete); + Assert.Equal("This run cannot be preprocessed because its stage is Preprocessing failed.", only.Message); + await _repo.DidNotReceiveWithAnyArgs().TrySetStatusAsync(default, default, default, default); + } + + // M4: SavePreprocessedAsync's own guard can lose the race (an Abandon landed while this + // pipeline ran) without throwing — 0 rows means the run must not be reported as Preprocessed. + [Fact] + public async Task Zero_rows_from_SavePreprocessed_reports_the_run_was_abandoned_not_preprocessed() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved")); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + _repo.SavePreprocessedAsync(Arg.Any(), Arg.Any(), Arg.Any>(), + Arg.Any>(), Arg.Any(), Arg.Any>(), Arg.Any()).Returns(0); + + var events = await Collect(); + + var last = events.Last(); + Assert.True(last.IsComplete); + Assert.True(last.IsError); + Assert.Null(last.FinalStatus); + Assert.Contains("abandoned", last.Message); + } + + // M4: same race for the failure path — the run must not be reported PreprocessingFailed (with + // its failures pinned) when it is actually Abandoned. + [Fact] + public async Task Zero_rows_from_MarkPreprocessingFailed_reports_the_run_was_abandoned_not_failed() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved"), Remove("R2", "approved", reason: "other")); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + _repo.MarkPreprocessingFailedAsync(Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()).Returns(0); + + var events = await Collect(); + + var last = events.Last(); + Assert.True(last.IsComplete); + Assert.True(last.IsError); + Assert.Null(last.FinalStatus); + Assert.Contains("abandoned", last.Message); + } + + [Fact] + public async Task Cancellation_mid_run_puts_the_run_back_to_its_previous_state() + { + RunIs(EgressRunStatus.Pulled, Remove("R1", "approved")); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Pulled, EgressRunStatus.Preprocessing, Arg.Any()).Returns(true); + using var cts = new CancellationTokenSource(); + + var enumerator = Sut().RunAsync(RunId, cts.Token).GetAsyncEnumerator(cts.Token); + Assert.True(await enumerator.MoveNextAsync()); // step 1 running + cts.Cancel(); + await Assert.ThrowsAnyAsync(async () => { while (await enumerator.MoveNextAsync()) { } }); + await enumerator.DisposeAsync(); + + await _repo.Received(1).TrySetStatusAsync(RunId, EgressRunStatus.Preprocessing, EgressRunStatus.Pulled, CancellationToken.None); + await _repo.DidNotReceiveWithAnyArgs().SavePreprocessedAsync(default, default, default!, default!, default, default!, default); + } + + private sealed class FakeTimeProvider(DateTimeOffset now) : TimeProvider + { + private DateTimeOffset _now = now; + public override DateTimeOffset GetUtcNow() => _now; + public void SetUtcNow(DateTimeOffset value) => _now = value; + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRecordBuilderTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRecordBuilderTests.cs new file mode 100644 index 000000000..7ddf38369 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRecordBuilderTests.cs @@ -0,0 +1,204 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// Steps 2-6 of the pipeline on one record. A failure is recorded against the step and field that +// found it and the record carries on, so the ops user sees EVERY problem in one run. +public sealed class EgressRecordBuilderTests +{ + private static EgressSourceRecord Remove(string? reason = "pupil-died", string? pupilLaestab = "8604070", string? orgLaestab = "860/4070", string dob = "07/09/2010", int matchRef = 555) => new() + { + ChangeRequestId = Guid.Parse("11111111-1111-1111-1111-111111111111"), ReferenceNumber = "REF-R", TicketId = 88856, Decision = "approved", + OutputType = EgressOutputType.RemoveLearners, WindowType = CheckingWindowType.KS4June, + SubmittedAtUtc = new DateTime(2026, 6, 5, 9, 0, 0, DateTimeKind.Utc), OrganisationUrn = 142313, OrganisationLaestab = orgLaestab, + PupilFirstname = " Jude ", PupilSurname = "Bellingham", PupilSex = "M", PupilDateOfBirth = dob, PupilIdentifier = "A860407000011", + PupilMatchRef = matchRef, PupilLaestab = pupilLaestab, JourneyFound = true, + Answers = reason is null ? new Dictionary() : new Dictionary { ["reason"] = reason } + }; + + private static EgressSourceRecord Add(string? orgLaestab = "860/4070") => new() + { + ChangeRequestId = Guid.Parse("22222222-2222-2222-2222-222222222222"), ReferenceNumber = "REF-A", TicketId = 69390, Decision = "approved", + OutputType = EgressOutputType.NewLearners, WindowType = CheckingWindowType.KS2, + SubmittedAtUtc = new DateTime(2026, 10, 7, 9, 0, 0, DateTimeKind.Utc), OrganisationUrn = 136412, OrganisationLaestab = orgLaestab, + PupilFirstname = "Annie", PupilSurname = "Lennox", PupilSex = "F", PupilDateOfBirth = "07/09/2010", PupilIdentifier = "", PupilLaestab = "", JourneyFound = true, + Answers = new Dictionary + { + ["first-name"] = "Annie", ["last-name"] = "Lennox", ["date-of-birth"] = "2010-09-07", ["sex"] = "F", ["upn"] = "A881541200011", + ["admission-date"] = "2018-09-04", ["year-group"] = "6", ["sen-status"] = "N" + } + }; + + // Cycle_Year/Cycle_Month come from the checking window (spec: "the month number in which the + // cycle ... takes place", "For June checking exercise this will hold 6"), never from each + // record's submission date — a June window's July submissions still say 6. + private static EgressWorkItem Run(EgressSourceRecord source, string cycleYear = "2026", string cycleMonth = "6") + { + var item = new EgressWorkItem(source, cycleYear, cycleMonth); + EgressRecordBuilder.DeriveCodes(item); + EgressRecordBuilder.SplitEstablishment(item); + EgressRecordBuilder.StandardiseDates(item); + EgressRecordBuilder.Build(item); + EgressRecordBuilder.Trim(item); + return item; + } + + [Fact] + public void A_remove_record_becomes_a_spec_row() + { + var item = Run(Remove()); + Assert.Empty(item.Failures); + Assert.Equal(new RemoveLearnerRow("88856", "31", "4", "KS4", "4070", "Bellingham", "Jude", "M", "2010-09-07", "2026", "6", "860", "555", + Guid.Parse("11111111-1111-1111-1111-111111111111"), 88856, "REF-R"), item.RemoveRow); + } + + [Fact] + public void A_new_learner_record_becomes_a_spec_row_from_the_journey_answers_and_the_schools_laestab() + { + var item = Run(Add(), cycleMonth: "10"); + Assert.Empty(item.Failures); + Assert.Equal(new NewLearnerRow("69390", "10", "KS2", "860", "4070", "Lennox", "Annie", "F", "2010-09-07", "2018-09-04", "", "2026", "10", + "136412", "", "A881541200011", "", "6", Guid.Parse("22222222-2222-2222-2222-222222222222"), 69390, "REF-A"), item.NewRow); + } + + [Fact] + public void Pupil_laestab_wins_over_the_request_rows_when_both_are_present() + { + var item = Run(Remove(pupilLaestab: "3735401", orgLaestab: "860/4070")); + Assert.Equal("373", item.RemoveRow!.LocalAuthority); + Assert.Equal("5401", item.RemoveRow.EstablishmentNumber); + } + + [Fact] + public void No_laestab_anywhere_fails_the_split_step_with_the_field_named() + { + var item = Run(Remove(pupilLaestab: "", orgLaestab: null)); + var failure = Assert.Single(item.Failures); + Assert.Equal(EgressRecordBuilder.StepSplit, failure.Step); + Assert.Equal("Establishment_Number", failure.Field); + Assert.Equal(88856, failure.TicketId); + Assert.Null(item.RemoveRow); + } + + [Fact] + public void An_unmapped_remove_reason_fails_the_codes_step() + { + var item = Run(Remove(reason: "other")); + var failure = Assert.Single(item.Failures); + Assert.Equal(EgressRecordBuilder.StepCodes, failure.Step); + Assert.Equal("Correction_Reason", failure.Field); + Assert.Contains("other", failure.Reason); + } + + [Fact] + public void A_missing_journey_fails_the_build_step() + { + var item = Run(Remove() with { JourneyFound = false }); + Assert.Contains(item.Failures, f => f.Step == EgressRecordBuilder.StepBuild && f.Reason.Contains("journey")); + } + + [Fact] + public void An_unparseable_date_fails_the_dates_step_and_later_steps_do_not_pile_on() + { + var item = Run(Remove(dob: "31/02/2010")); + var failure = Assert.Single(item.Failures); + Assert.Equal(EgressRecordBuilder.StepDates, failure.Step); + Assert.Equal("Date_of_Birth", failure.Field); + } + + // S4: steps 2-4 are independent of one another, so a fault in one must not hide a fault in + // another — ops must see everything wrong with a record in one run, not discover the second + // problem only after fixing the first and re-running. + [Fact] + public void Independent_faults_in_different_steps_are_all_listed_not_just_the_first() + { + var item = Run(Remove(reason: "other", pupilLaestab: "", orgLaestab: null)); + Assert.Equal(2, item.Failures.Count); + Assert.Contains(item.Failures, f => f.Step == EgressRecordBuilder.StepCodes && f.Field == "Correction_Reason"); + Assert.Contains(item.Failures, f => f.Step == EgressRecordBuilder.StepSplit && f.Field == "Establishment_Number"); + Assert.Null(item.RemoveRow); + } + + [Fact] + public void Post16_reasons_and_stage_are_handled() + { + var item = Run(Remove(reason: "student-died") with { WindowType = CheckingWindowType.Post16 }); + Assert.Equal("16-19", item.RemoveRow!.KeyStage); + Assert.Equal("4", item.RemoveRow.CorrectionReason); + } + + [Fact] + public void Cycle_month_is_not_zero_padded_and_cycle_year_is_four_digits() + { + var item = Run(Remove()); + Assert.Equal("6", item.RemoveRow!.CycleMonth); + Assert.Equal("2026", item.RemoveRow.CycleYear); + } + + [Fact] + public void Cycle_values_come_from_the_window_not_the_submission_date() + { + // Submitted 5 July, but the KS4 June exercise is month 6. + var item = Run(Remove() with { SubmittedAtUtc = new DateTime(2026, 7, 5, 9, 0, 0, DateTimeKind.Utc) }, cycleYear: "2026", cycleMonth: "6"); + Assert.Equal("6", item.RemoveRow!.CycleMonth); + Assert.Equal("2026", item.RemoveRow.CycleYear); + } + + [Fact] + public void A_KS4_year_group_change_carries_the_year_group_moved_to() + { + var source = Remove(reason: "year-group-change"); + source = source with { Answers = new Dictionary(source.Answers) { ["higher-lower"] = "lower", ["year-group-lower-moved-to"] = "9", ["year-group-higher-moved-to"] = "12" } }; + var item = Run(source); + Assert.Empty(item.Failures); + Assert.Equal("17", item.RemoveRow!.CorrectionReason); + Assert.Equal("9", item.RemoveRow.YearGroup); + } + + [Fact] + public void A_KS4_removal_that_is_not_a_year_group_change_leaves_Year_Group_blank() + => Assert.Equal("", Run(Remove()).RemoveRow!.YearGroup); + + [Fact] + public void A_16_19_removal_maps_the_chosen_academic_years_onto_Removal_Year_0_to_2() + { + var source = Remove(reason: "other") with { WindowType = CheckingWindowType.Post16 }; + source = source with { Answers = new Dictionary { ["reason"] = "other", ["years-to-remove"] = "2025-2026|2023-2024" } }; + var item = Run(source, cycleYear: "2026", cycleMonth: "10"); + Assert.Empty(item.Failures); + Assert.Equal("16-19", item.RemoveRow!.KeyStage); + Assert.Equal("TRUE", item.RemoveRow.RemovalYear0); // 2025-2026 + Assert.Equal("FALSE", item.RemoveRow.RemovalYear1); // 2024-2025 + Assert.Equal("TRUE", item.RemoveRow.RemovalYear2); // 2023-2024 + Assert.Equal("", item.RemoveRow.YearGroup); + Assert.Equal("329", item.RemoveRow.CorrectionReason); + } + + [Fact] + public void A_16_19_removal_whose_journey_did_not_ask_about_years_leaves_them_blank() + { + var item = Run(Remove(reason: "student-died") with { WindowType = CheckingWindowType.Post16 }, cycleMonth: "10"); + Assert.Equal(("", "", ""), (item.RemoveRow!.RemovalYear0, item.RemoveRow.RemovalYear1, item.RemoveRow.RemovalYear2)); + } + + [Fact] + public void A_16_19_not_on_roll_removal_uses_the_sub_reason_code() + { + var source = Remove(reason: "not-on-roll") with { WindowType = CheckingWindowType.Post16 }; + source = source with { Answers = new Dictionary { ["reason"] = "not-on-roll", ["not-on-roll-reason"] = "apprentice" } }; + var item = Run(source, cycleMonth: "10"); + Assert.Empty(item.Failures); + Assert.Equal("331", item.RemoveRow!.CorrectionReason); + } + + [Fact] + public void A_16_19_not_on_roll_removal_without_a_sub_reason_fails_the_codes_step_naming_both() + { + var item = Run(Remove(reason: "not-on-roll") with { WindowType = CheckingWindowType.Post16 }, cycleMonth: "10"); + var failure = Assert.Single(item.Failures); + Assert.Equal(EgressRecordBuilder.StepCodes, failure.Step); + Assert.Equal("Correction_Reason", failure.Field); + Assert.Contains("not-on-roll", failure.Reason); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRunServiceTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRunServiceTests.cs new file mode 100644 index 000000000..51617fe4c --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressRunServiceTests.cs @@ -0,0 +1,162 @@ +using DfE.CheckPerformanceData.Application.CheckYourPupilData; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.Journey; +using DfE.CheckPerformanceData.Application.RequestSubmission; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// The pull returns EVERYTHING for the window and output types, decision unfiltered (AB#294553 +// "Retrieval"), and refuses before touching Zendesk when a run already holds the pair. +public sealed class EgressRunServiceTests +{ + private static readonly Guid WindowId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + private static readonly EgressActor Actor = new(Guid.Parse("22222222-2222-2222-2222-222222222222"), "Ops One", "ops@example.com"); + + private readonly IEgressRunRepository _repo = Substitute.For(); + private readonly IEgressTicketSource _tickets = Substitute.For(); + private readonly IRequestStateBlobClient _blobs = Substitute.For(); + private readonly IWindowService _windows = Substitute.For(); + + private EgressRunService Sut() => new(_repo, _tickets, _blobs, _windows, Substitute.For>()); + + private static CheckingWindowDto Window() => new() + { + Id = WindowId, Title = "KS4 June 2026", KeyStage = KeyStages.KS4, CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }; + + private static EgressCandidateRequest Candidate(string reference, string? crm) => + new(Guid.NewGuid(), reference, crm, 142313, "860/4070", new DateTime(2026, 6, 5, 9, 0, 0, DateTimeKind.Utc), RequestStatus.SubmittedCommitted); + + private static RequestState Journey(string reason) => new() + { + SelectedWhatToChange = WhatToChange.Remove, + SelectedPupil = new PupilDto + { + Id = Guid.NewGuid(), Firstname = "Alice", Surname = "Smith", Sex = "F", DateOfBirth = "07/09/2010", Age = 15, + Cypmd_Id = "500001", Identifier = "A860407000011", MatchRef = 555, Laestab = "8604070" + }, + QuestionAnswers = { ["reason"] = new QuestionAnswer { TextValue = reason } } + }; + + [Fact] + public async Task Refuses_before_calling_zendesk_when_a_pair_is_held() + { + var blocker = new EgressBlocker(Guid.NewGuid(), EgressRunStatus.Pulled, "Ops Two", DateTime.UtcNow, null, null); + _repo.FindBlockerAsync(WindowId, EgressOutputType.RemoveLearners, Arg.Any()).Returns(blocker); + _repo.FindBlockerAsync(WindowId, EgressOutputType.NewLearners, Arg.Any()).Returns((EgressBlocker?)null); + + var result = await Sut().StartAsync(WindowId, [EgressOutputType.NewLearners, EgressOutputType.RemoveLearners], Actor, CancellationToken.None); + + var refused = Assert.IsType(result); + var (type, who) = Assert.Single(refused.Blockers); + Assert.Equal(EgressOutputType.RemoveLearners, type); + Assert.Equal("Ops Two", who.StartedByName); + await _tickets.DidNotReceiveWithAnyArgs().GetDecisionStatusesAsync(default!, default); + await _repo.DidNotReceiveWithAnyArgs().CreateRunAsync(default!, default); + } + + [Fact] + public async Task Pulls_every_candidate_with_its_decision_and_journey_values() + { + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + _repo.GetCandidateRequestsAsync(WindowId, WhatToChange.Remove, Arg.Any()) + .Returns([Candidate("REF-1", "1001"), Candidate("REF-2", "1002"), Candidate("REF-3", null), Candidate("REF-4", "1004")]); + _tickets.GetDecisionStatusesAsync(Arg.Is>(ids => ids.SequenceEqual(new long[] { 1001, 1002, 1004 })), Arg.Any()) + .Returns(new Dictionary { [1001] = "auto_approved", [1002] = "rejected" }); + _blobs.GetAsync(WindowId, Arg.Any()).Returns(ci => Journey("pupil-died")); + _blobs.GetAsync(WindowId, "REF-4").Returns((RequestState?)null); + EgressRunCreate? created = null; + _repo.CreateRunAsync(Arg.Do(c => created = c), Arg.Any()).Returns(Guid.NewGuid()); + + var result = await Sut().StartAsync(WindowId, [EgressOutputType.RemoveLearners], Actor, CancellationToken.None); + + Assert.IsType(result); + var output = Assert.Single(created!.Outputs); + Assert.Equal(EgressOutputType.RemoveLearners, output.OutputType); + Assert.Equal(4, output.Records.Count); + var byRef = output.Records.ToDictionary(r => r.ReferenceNumber); + Assert.Equal("auto_approved", byRef["REF-1"].Decision); + Assert.Equal("rejected", byRef["REF-2"].Decision); + Assert.Equal(EgressDecisions.NoTicket, byRef["REF-3"].Decision); + Assert.Equal(EgressDecisions.NotFound, byRef["REF-4"].Decision); + Assert.Equal(1001, byRef["REF-1"].TicketId); + Assert.Equal("Smith", byRef["REF-1"].PupilSurname); + Assert.Equal("8604070", byRef["REF-1"].PupilLaestab); + Assert.Equal(555, byRef["REF-1"].PupilMatchRef); + Assert.Equal("pupil-died", byRef["REF-1"].Answer("reason")); + Assert.Equal(CheckingWindowType.KS4June, byRef["REF-1"].WindowType); + Assert.True(byRef["REF-1"].JourneyFound); + Assert.False(byRef["REF-4"].JourneyFound); + Assert.Equal("Ops One", created.StartedByName); + } + + // S10: only a null journey (not found) was tested — a genuinely failed read (blob storage + // unreachable) must be caught per-record too, not fail the whole pull. + [Fact] + public async Task A_journey_blob_read_that_throws_is_treated_the_same_as_not_found() + { + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + _repo.GetCandidateRequestsAsync(WindowId, WhatToChange.Remove, Arg.Any()) + .Returns([Candidate("REF-1", "1001")]); + _tickets.GetDecisionStatusesAsync(Arg.Any>(), Arg.Any()) + .Returns(new Dictionary { [1001] = "auto_approved" }); + _blobs.GetAsync(WindowId, "REF-1").Returns(Task.FromException(new InvalidOperationException("blob storage unreachable"))); + EgressRunCreate? created = null; + _repo.CreateRunAsync(Arg.Do(c => created = c), Arg.Any()).Returns(Guid.NewGuid()); + + var result = await Sut().StartAsync(WindowId, [EgressOutputType.RemoveLearners], Actor, CancellationToken.None); + + Assert.IsType(result); + var record = Assert.Single(created!.Outputs).Records.Single(); + Assert.False(record.JourneyFound); + Assert.Equal("auto_approved", record.Decision); + } + + [Fact] + public async Task A_conflict_on_insert_is_reported_as_a_refusal_with_the_current_blocker() + { + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + _repo.GetCandidateRequestsAsync(WindowId, WhatToChange.Add, Arg.Any()).Returns([]); + _tickets.GetDecisionStatusesAsync(Arg.Any>(), Arg.Any()).Returns(new Dictionary()); + _repo.CreateRunAsync(Arg.Any(), Arg.Any()) + .Returns>(_ => throw new EgressRunConflictException("held")); + var blocker = new EgressBlocker(Guid.NewGuid(), EgressRunStatus.Transferred, "Ops Two", DateTime.UtcNow, DateTime.UtcNow, "Ops Two"); + _repo.FindBlockerAsync(WindowId, EgressOutputType.NewLearners, Arg.Any()).Returns((EgressBlocker?)null, blocker); + + var result = await Sut().StartAsync(WindowId, [EgressOutputType.NewLearners], Actor, CancellationToken.None); + + var refused = Assert.IsType(result); + Assert.Equal(EgressRunStatus.Transferred, refused.Blockers[0].Blocker.Status); + } + + [Fact] + public async Task A_ticket_source_failure_is_reported_not_thrown() + { + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + _repo.GetCandidateRequestsAsync(WindowId, WhatToChange.Remove, Arg.Any()).Returns([Candidate("REF-1", "1001")]); + _tickets.GetDecisionStatusesAsync(Arg.Any>(), Arg.Any()) + .Returns>>(_ => throw new EgressTicketSourceException("field not configured")); + + var result = await Sut().StartAsync(WindowId, [EgressOutputType.RemoveLearners], Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Equal("field not configured", failed.Reason); + await _repo.DidNotReceiveWithAnyArgs().CreateRunAsync(default!, default); + } + + [Fact] + public async Task An_unknown_window_is_reported() + { + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns((CheckingWindowDto?)null); + Assert.IsType(await Sut().StartAsync(WindowId, [EgressOutputType.NewLearners], Actor, CancellationToken.None)); + } + + [Fact] + public async Task No_output_types_is_refused_as_an_argument_error() + => await Assert.ThrowsAsync(() => Sut().StartAsync(WindowId, [], Actor, CancellationToken.None)); +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransferServiceTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransferServiceTests.cs new file mode 100644 index 000000000..e0df070a6 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransferServiceTests.cs @@ -0,0 +1,523 @@ +using System.Text; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// Transfer is atomic from the user's point of view: every file lands or none does, the database +// is the source of the files (never the pulled payload), and the audit row is written by the +// repository in the same transaction as the run state. +public sealed class EgressTransferServiceTests +{ + private static readonly Guid RunId = Guid.Parse("33333333-3333-3333-3333-333333333333"); + private static readonly EgressActor Actor = new(Guid.Parse("22222222-2222-2222-2222-222222222222"), "Ops One", "ops@example.com"); + private readonly IEgressRunRepository _repo = Substitute.For(); + private readonly IEgressBlobClient _blobs = Substitute.For(); + private readonly IWindowService _windows = Substitute.For(); + + private EgressTransferService Sut() => new(_repo, _blobs, _windows, Substitute.For>()); + + private static RemoveLearnerRow RemoveRow(string ticket) => + new(ticket, "31", "4", "KS4", "4070", "Smith", "Alice", "F", "2010-09-07", "2026", "6", "860", "555", Guid.NewGuid(), long.Parse(ticket), $"REF-{ticket}"); + private static NewLearnerRow NewRow(string ticket) => + new(ticket, "10", "KS4", "860", "4070", "Jones", "Bob", "M", "2010-01-02", "2018-09-04", "", "2026", "6", "142313", "", "A860407000011", "", "10", Guid.NewGuid(), long.Parse(ticket), $"REF-{ticket}"); + + private void RunIs(EgressRunStatus status, params EgressOutputType[] types) + { + _repo.GetRunAsync(RunId, Arg.Any()).Returns(new EgressRunDto(RunId, Guid.NewGuid(), status, Guid.NewGuid(), "Ops One", + DateTime.UtcNow, DateTime.UtcNow, new DateOnly(2026, 6, 8), null, null, [], null, + types.Select(t => new EgressRunOutputDto(Guid.NewGuid(), t, true, [], 2, 2, + $"CYPMD_LDS_KS4_{EgressOutputTypes.FileToken(t)}_2026_06_08.csv", null)).ToList())); + _repo.GetRemoveLearnersAsync(RunId, Arg.Any()).Returns([RemoveRow("1001"), RemoveRow("1002")]); + _repo.GetNewLearnersAsync(RunId, Arg.Any()).Returns([NewRow("2001")]); + _windows.GetByIdAsync(Arg.Any(), Arg.Any()).Returns(new CheckingWindowDto + { + Id = Guid.NewGuid(), Title = "KS4 June 2026", KeyStage = KeyStages.KS4, CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + _blobs.IsConfigured.Returns(true); + _blobs.TargetDescription.Returns("cypmd/extracts_input"); + // M4: the guarded writes now return rows affected — default to "won the race" (1) so + // existing success-path tests are unaffected; the specific lost-the-race tests override this. + _repo.MarkTransferredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(1); + _repo.AbandonAsync(Arg.Any(), Arg.Any()).Returns(1); + } + + [Fact] + public async Task Uploads_every_file_from_the_database_rows_then_marks_transferred_with_the_audit() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + var uploaded = new Dictionary(); + await _blobs.UploadAsync(Arg.Do(n => uploaded[n] = ""), Arg.Do(b => uploaded[uploaded.Keys.Last()] = Encoding.UTF8.GetString(b)), Arg.Any(), RunId, Arg.Any()); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var ok = Assert.IsType(result); + Assert.Equal(2, ok.Files.Count); + Assert.StartsWith("Correction_ID,Correction_Type,Correction_Reason,", uploaded["CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"]); + Assert.Contains("\r\n1001,31,4,KS4,4070,Smith,Alice,F,2010-09-07,2026,6,860,555", uploaded["CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"]); + Assert.StartsWith("Correction_ID,Correction_Type,Key_Stage,", uploaded["CYPMD_LDS_KS4_NewLearners_2026_06_08.csv"]); + // KS4 window → the Remove file ends with the spec's KS4-only Year_Group column. + Assert.StartsWith("Correction_ID,Correction_Type,Correction_Reason,Key_Stage,Establishment_Number,Surname,Forename,Sex,Date_of_Birth,Cycle_Year,Cycle_Month,Local_Authority,Learner_ID,Year_Group\r\n", + uploaded["CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"]); + Assert.Contains("\r\n1001,31,4,KS4,4070,Smith,Alice,F,2010-09-07,2026,6,860,555,", uploaded["CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"]); + Assert.StartsWith("Correction_ID,Correction_Type,Key_Stage,Establishment_Number,Surname,Forename,Sex,Date_of_Birth,Admission_Date,Post_Code,Cycle_Year,Cycle_Month,Local_Authority,URN,ULN,UPN,Learner_ID,Year_Group\r\n", + uploaded["CYPMD_LDS_KS4_NewLearners_2026_06_08.csv"]); + await _repo.Received(1).MarkTransferredAsync(RunId, EgressRunStatus.Transferring, + Arg.Is(a => a.UserName == "Ops One" && a.TargetContainer == "cypmd/extracts_input" + && a.Files[EgressOutputType.RemoveLearners].Records == 2 && a.Files[EgressOutputType.NewLearners].Records == 1 + && a.Files[EgressOutputType.RemoveLearners].Sha256.Length == 64), + Arg.Any(), Arg.Any()); + await _repo.DidNotReceiveWithAnyArgs().MarkTransferFailedAsync(default, default, default!, default!, default); + } + + [Fact] + public async Task A_failure_on_the_second_file_deletes_the_first_and_marks_the_run_failed() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _blobs.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"))); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Contains("already exists", failed.Reason); + await _blobs.Received(1).DeleteIfExistsAsync("CYPMD_LDS_KS4_NewLearners_2026_06_08.csv", Arg.Any()); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Is(r => r.Contains("already exists")), Actor.UserId.ToString(), Arg.Any()); + await _repo.DidNotReceiveWithAnyArgs().MarkTransferredAsync(default, default, default!, default, default); + } + + [Fact] + public async Task Unconfigured_storage_fails_before_anything_is_uploaded() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _blobs.IsConfigured.Returns(false); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Contains("not configured", failed.Reason); + await _blobs.DidNotReceiveWithAnyArgs().UploadAsync(default!, default!, default!, default, default); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Preprocessed, Arg.Any(), Actor.UserId.ToString(), Arg.Any()); + } + + [Fact] + public async Task A_retry_after_failure_reactivates_first_and_is_refused_when_the_pair_is_taken() + { + RunIs(EgressRunStatus.TransferFailed, EgressOutputType.RemoveLearners); + var blocker = new EgressBlocker(Guid.NewGuid(), EgressRunStatus.Pulled, "Ops Two", DateTime.UtcNow, null, null); + _repo.TryReactivateAsync(RunId, Arg.Any()).Returns((EgressOutputType.RemoveLearners, blocker)); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var refused = Assert.IsType(result); + Assert.Equal(blocker, refused.Blocker); + Assert.Equal(EgressOutputType.RemoveLearners, refused.OutputType); + await _blobs.DidNotReceiveWithAnyArgs().UploadAsync(default!, default!, default!, default, default); + } + + [Theory] + [InlineData(EgressRunStatus.Pulled)] + [InlineData(EgressRunStatus.Transferred)] + [InlineData(EgressRunStatus.Abandoned)] + public async Task Only_a_preprocessed_or_failed_run_can_be_transferred(EgressRunStatus status) + { + RunIs(status, EgressOutputType.RemoveLearners); + Assert.Equal(status, Assert.IsType(await Sut().TransferAsync(RunId, Actor, CancellationToken.None)).Status); + } + + // M3: a run whose approved set is empty (every record rejected, undecided, or lost to B1) must + // not send a header-only file and lock the pair forever — this must refuse before any upload. + [Fact] + public async Task A_run_whose_every_output_has_zero_saved_rows_refuses_before_any_upload() + { + _repo.GetRunAsync(RunId, Arg.Any()).Returns(new EgressRunDto(RunId, Guid.NewGuid(), EgressRunStatus.Preprocessed, Guid.NewGuid(), "Ops One", + DateTime.UtcNow, DateTime.UtcNow, new DateOnly(2026, 6, 8), null, null, [], null, + [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 3, 0, "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", null)])); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.DidNotReceiveWithAnyArgs().UploadAsync(default!, default!, default!, default, default); + await _repo.DidNotReceiveWithAnyArgs().TrySetStatusAsync(default, default, default, default); + } + + [Fact] + public async Task A_run_with_at_least_one_non_empty_output_is_still_transferable() + { + _repo.GetRunAsync(RunId, Arg.Any()).Returns(new EgressRunDto(RunId, Guid.NewGuid(), EgressRunStatus.Preprocessed, Guid.NewGuid(), "Ops One", + DateTime.UtcNow, DateTime.UtcNow, new DateOnly(2026, 6, 8), null, null, [], null, + [ + new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 3, 0, "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", null), + new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.NewLearners, true, [], 1, 1, "CYPMD_LDS_KS4_NewLearners_2026_06_08.csv", null) + ])); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _repo.GetRemoveLearnersAsync(RunId, Arg.Any()).Returns([]); + _repo.GetNewLearnersAsync(RunId, Arg.Any()).Returns([NewRow("2001")]); + _windows.GetByIdAsync(Arg.Any(), Arg.Any()).Returns(new CheckingWindowDto + { + Id = Guid.NewGuid(), Title = "KS4 June 2026", KeyStage = KeyStages.KS4, CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }); + _blobs.IsConfigured.Returns(true); + _repo.MarkTransferredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(1); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + } + + // M4: MarkTransferredAsync's own status guard can lose the race even without throwing (e.g. + // an Abandon landed between the Transferring flip and this write) — 0 rows means compensate, + // never report Transferred. + [Fact] + public async Task Zero_rows_from_MarkTransferredAsync_compensates_without_reporting_transferred() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _repo.MarkTransferredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(0); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfExistsAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any()); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Any(), Actor.UserId.ToString(), Arg.Any()); + } + + [Fact] + public async Task BuildFile_renders_the_persisted_rows_for_preview_and_download() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + var text = Encoding.UTF8.GetString(await Sut().BuildFileAsync(RunId, EgressOutputType.RemoveLearners, CancellationToken.None)); + Assert.Equal(3, text.Split("\r\n").Length); + Assert.DoesNotContain("\n\n", text); + } + + [Fact] + public async Task Preview_and_download_use_the_windows_column_set() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _windows.GetByIdAsync(Arg.Any(), Arg.Any()).Returns(new CheckingWindowDto + { + Id = Guid.NewGuid(), Title = "16 to 19 October 2026", KeyStage = KeyStages.Post16, CheckingWindowType = CheckingWindowType.Post16, + StartDate = new DateTime(2026, 10, 1), EndDate = new DateTime(2026, 10, 31) + }); + + var (headers, rows) = await Sut().GetPreviewAsync(RunId, EgressOutputType.RemoveLearners, CancellationToken.None); + var csv = Encoding.UTF8.GetString(await Sut().BuildFileAsync(RunId, EgressOutputType.RemoveLearners, CancellationToken.None)); + + Assert.Equal(["Removal_Year_0", "Removal_Year_1", "Removal_Year_2"], headers.TakeLast(3).ToArray()); + Assert.Equal(2, rows.Count); + Assert.StartsWith("Correction_ID,", csv); + Assert.EndsWith(",Removal_Year_0,Removal_Year_1,Removal_Year_2\r\n1001,", csv[..(csv.IndexOf("\r\n", StringComparison.Ordinal) + 7)]); + } + + // M1: once the CAS flip to Transferring has happened, cancellation of the caller's token must + // not abandon a run with files already uploaded — the upload/commit phase and its compensation + // run with CancellationToken.None. + [Fact] + public async Task After_the_status_flips_to_transferring_the_callers_cancellation_no_longer_skips_compensation() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + using var cts = new CancellationTokenSource(); + _blobs.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(_ => { cts.Cancel(); throw new OperationCanceledException("client disconnected"); }); + CancellationToken? deleteToken = null; + _blobs.DeleteIfExistsAsync(Arg.Any(), Arg.Do(t => deleteToken = t)).Returns(Task.CompletedTask); + CancellationToken? markFailedToken = null; + _repo.MarkTransferFailedAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Do(t => markFailedToken = t)) + .Returns(1); + + var result = await Sut().TransferAsync(RunId, Actor, cts.Token); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfExistsAsync("CYPMD_LDS_KS4_NewLearners_2026_06_08.csv", Arg.Any()); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Any(), Actor.UserId.ToString(), Arg.Any()); + Assert.Equal(CancellationToken.None, deleteToken); + Assert.Equal(CancellationToken.None, markFailedToken); + } + + // M1: a post-upload DB failure (every file landed, but the commit that marks Transferred + // threw) must not leave the run silently stuck in Transferring with no audit trail. + [Fact] + public async Task MarkTransferredAsync_throwing_after_every_upload_succeeds_compensates_and_marks_transfer_failed() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _repo.MarkTransferredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromException(new InvalidOperationException("database unreachable"))); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfExistsAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any()); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Any(), Actor.UserId.ToString(), Arg.Any()); + await _repo.DidNotReceiveWithAnyArgs().TryReactivateAsync(default, default); + } + + // M1/S3: the compensation delete itself can fail (the ops user has no LDS access to fix it by + // hand) — the reason must say so rather than silently dropping the detail. + [Fact] + public async Task If_the_compensation_delete_itself_fails_the_reason_explains_manual_cleanup_is_needed() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _blobs.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"))); + _blobs.DeleteIfExistsAsync("CYPMD_LDS_KS4_NewLearners_2026_06_08.csv", Arg.Any()) + .Returns(Task.FromException(new InvalidOperationException("network blip"))); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Contains("remove it by hand", failed.Reason); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Is(r => r.Contains("remove it by hand")), Actor.UserId.ToString(), Arg.Any()); + } + + // S3: a PUT that succeeded server-side but whose response was lost must not leave an orphan + // that only LDS (not the ops user) can remove — the failing file itself is swept too, but only + // if it is stamped as this run's own. + [Fact] + public async Task S3_a_failed_upload_also_removes_its_own_blob_if_the_write_actually_landed() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _blobs.UploadAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new TimeoutException("response lost"))); + _blobs.DeleteIfOwnedByRunAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", RunId, Arg.Any()).Returns(true); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfOwnedByRunAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", RunId, Arg.Any()); + } + + // M1: Abandon during Transferring — the lock has no expiry, so a run stuck there (a pod + // restart mid-upload) must be releasable, and any blob it actually wrote must be swept so it + // does not block a same-named retry, without ever touching a blob owned by another run. + [Fact] + public async Task Abandoning_a_transferring_run_sweeps_only_the_blobs_this_run_owns() + { + RunIs(EgressRunStatus.Transferring, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _blobs.DeleteIfOwnedByRunAsync("CYPMD_LDS_KS4_NewLearners_2026_06_08.csv", RunId, Arg.Any()).Returns(true); + _blobs.DeleteIfOwnedByRunAsync("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", RunId, Arg.Any()).Returns(false); + + var result = await Sut().AbandonAsync(RunId, CancellationToken.None); + + var abandoned = Assert.IsType(result); + Assert.Equal(["CYPMD_LDS_KS4_NewLearners_2026_06_08.csv"], abandoned.RemovedFiles); + await _repo.Received(1).AbandonAsync(RunId, Arg.Any()); + } + + // R1: the repository write must happen — and be committed — before any blob is swept. A + // concurrent TransferAsync can flip Transferring -> Transferred between the read at the top + // of AbandonAsync and the write here; if the sweep ran first (the pre-fix ordering) it would + // delete the files that transfer had just uploaded, and this call would still go on to lose + // the race (rows == 0) and report AlreadyTransferred — leaving the run Transferred, with a + // Succeeded audit row, and no files in LDS. Reading rows-affected from the write is what + // tells this method which case actually happened, so the write must come first. + [Fact] + public async Task Abandoning_a_transferring_run_that_lost_the_race_reports_AlreadyTransferred_and_never_sweeps_blobs() + { + RunIs(EgressRunStatus.Transferring, EgressOutputType.NewLearners, EgressOutputType.RemoveLearners); + _repo.AbandonAsync(RunId, Arg.Any()).Returns(0); + + var result = await Sut().AbandonAsync(RunId, CancellationToken.None); + + Assert.IsType(result); + await _blobs.DidNotReceiveWithAnyArgs().DeleteIfOwnedByRunAsync(default!, default, default); + } + + // M1: once this method starts changing state, the caller's token (RequestAborted) must not be + // able to abandon it half-way — the repository write itself must use CancellationToken.None, + // not whatever token the caller passed in. + [Fact] + public async Task Abandoning_a_run_passes_None_to_the_repository_write_regardless_of_the_callers_token() + { + RunIs(EgressRunStatus.Transferring, EgressOutputType.RemoveLearners); + using var cts = new CancellationTokenSource(); + + var result = await Sut().AbandonAsync(RunId, cts.Token); + + Assert.IsType(result); + await _repo.Received(1).AbandonAsync(RunId, CancellationToken.None); + } + + [Fact] + public async Task Abandoning_a_run_that_is_not_transferring_never_touches_blob_storage() + { + RunIs(EgressRunStatus.Pulled, EgressOutputType.RemoveLearners); + + var result = await Sut().AbandonAsync(RunId, CancellationToken.None); + + var abandoned = Assert.IsType(result); + Assert.Empty(abandoned.RemovedFiles); + await _blobs.DidNotReceiveWithAnyArgs().DeleteIfOwnedByRunAsync(default!, default, default); + await _repo.Received(1).AbandonAsync(RunId, Arg.Any()); + } + + [Fact] + public async Task Abandoning_an_already_transferred_run_is_refused() + { + RunIs(EgressRunStatus.Transferred, EgressOutputType.RemoveLearners); + + var result = await Sut().AbandonAsync(RunId, CancellationToken.None); + + Assert.IsType(result); + await _repo.DidNotReceiveWithAnyArgs().AbandonAsync(default, default); + } + + [Fact] + public async Task Abandoning_an_unknown_run_reports_not_found() + { + _repo.GetRunAsync(RunId, Arg.Any()).Returns((EgressRunDto?)null); + + var result = await Sut().AbandonAsync(RunId, CancellationToken.None); + + Assert.IsType(result); + } + + private static EgressRunDto OtherRun(Guid id, EgressRunStatus status) => + new(id, Guid.NewGuid(), status, Guid.NewGuid(), "Ops Two", DateTime.UtcNow, DateTime.UtcNow, new DateOnly(2026, 6, 8), null, null, [], null, []); + + private const string RemoveFile = "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"; + + // Follow-up to R1 (Abandon crash-window orphan): if the process died between writing + // `Abandoned` and finishing the blob sweep, a file stamped with that run's id is still in LDS + // with no UI path to remove it. The next transfer for the same file name is that path: a + // colliding file owned by an ABANDONED run is reclaimed (metadata-checked delete) and the + // create-only upload retried once. + [Fact] + public async Task A_colliding_file_left_by_an_abandoned_run_is_reclaimed_and_the_upload_retried() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + var orphanOwner = Guid.NewGuid(); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException(RemoveFile)), Task.CompletedTask); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns(orphanOwner); + _repo.GetRunAsync(orphanOwner, Arg.Any()).Returns(OtherRun(orphanOwner, EgressRunStatus.Abandoned)); + _blobs.DeleteIfOwnedByRunAsync(RemoveFile, orphanOwner, Arg.Any()).Returns(true); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfOwnedByRunAsync(RemoveFile, orphanOwner, Arg.Any()); + await _blobs.Received(2).UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()); + await _repo.DidNotReceiveWithAnyArgs().MarkTransferFailedAsync(default, default, default!, default!, default); + } + + // The same path also clears a leftover from THIS run's own earlier attempt — the case where + // compensation's delete failed and the reason said "remove it by hand". + [Fact] + public async Task A_colliding_file_left_by_this_runs_own_earlier_attempt_is_reclaimed() + { + RunIs(EgressRunStatus.TransferFailed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.TransferFailed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException(RemoveFile)), Task.CompletedTask); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns(RunId); + _blobs.DeleteIfOwnedByRunAsync(RemoveFile, RunId, Arg.Any()).Returns(true); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfOwnedByRunAsync(RemoveFile, RunId, Arg.Any()); + await _blobs.Received(2).UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()); + } + + // A file another run actually TRANSFERRED is the real same-stage/same-day collision and must + // never be touched — this is the case the create-only upload exists for. + [Theory] + [InlineData(EgressRunStatus.Transferred)] + [InlineData(EgressRunStatus.Transferring)] + public async Task A_colliding_file_owned_by_a_live_run_is_never_reclaimed(EgressRunStatus ownerStatus) + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + var owner = Guid.NewGuid(); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException(RemoveFile))); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns(owner); + _repo.GetRunAsync(owner, Arg.Any()).Returns(OtherRun(owner, ownerStatus)); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Contains("already exists", failed.Reason); + await _blobs.DidNotReceive().DeleteIfOwnedByRunAsync(RemoveFile, owner, Arg.Any()); + await _blobs.Received(1).UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()); + } + + // Exactly one reclaim: if the retried upload collides again, something else is writing the + // same name and the transfer fails rather than looping. + [Fact] + public async Task A_reclaimed_file_that_collides_again_on_retry_fails_without_a_second_reclaim() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + var orphanOwner = Guid.NewGuid(); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException(RemoveFile))); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns(orphanOwner); + _repo.GetRunAsync(orphanOwner, Arg.Any()).Returns(OtherRun(orphanOwner, EgressRunStatus.Abandoned)); + _blobs.DeleteIfOwnedByRunAsync(RemoveFile, orphanOwner, Arg.Any()).Returns(true); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + Assert.IsType(result); + await _blobs.Received(1).DeleteIfOwnedByRunAsync(RemoveFile, orphanOwner, Arg.Any()); + await _blobs.Received(2).UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, Arg.Any(), Actor.UserId.ToString(), Arg.Any()); + } + + // Same-stage/same-day collision: the file name carries the preprocessing date, so retrying the + // SAME run tomorrow reuses today's name and collides again. The recorded reason must say what + // actually gets the operator out, not just "already exists". + [Fact] + public async Task A_collision_with_another_runs_live_file_records_who_wrote_it_and_the_way_out() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + var owner = Guid.NewGuid(); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException("extracts_input/" + RemoveFile))); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns(owner); + _repo.GetRunAsync(owner, Arg.Any()).Returns(OtherRun(owner, EgressRunStatus.Transferred)); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Equal("A file named extracts_input/CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv already exists in the LDS container. " + + "Another egress run wrote it (its status is Transferred), so it was left in place. " + + "Two checking windows of the same key stage cannot transfer on the same day. " + + "Abandon this run and start a new one on a later day, or ask LDS to remove the file and then retry.", failed.Reason); + await _repo.Received(1).MarkTransferFailedAsync(RunId, EgressRunStatus.Transferring, failed.Reason, Actor.UserId.ToString(), Arg.Any()); + } + + [Fact] + public async Task A_collision_with_a_file_this_service_never_stamped_says_so() + { + RunIs(EgressRunStatus.Preprocessed, EgressOutputType.RemoveLearners); + _repo.TrySetStatusAsync(RunId, EgressRunStatus.Preprocessed, EgressRunStatus.Transferring, Arg.Any()).Returns(true); + _blobs.UploadAsync(RemoveFile, Arg.Any(), Arg.Any(), RunId, Arg.Any()) + .Returns(Task.FromException(new EgressBlobAlreadyExistsException("extracts_input/" + RemoveFile))); + _blobs.GetOwnerRunIdAsync(RemoveFile, Arg.Any()).Returns((Guid?)null); + + var result = await Sut().TransferAsync(RunId, Actor, CancellationToken.None); + + var failed = Assert.IsType(result); + Assert.Equal("A file named extracts_input/CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv already exists in the LDS container. " + + "It was not written by this service, so it was left in place. Ask LDS to remove or rename it, then retry.", failed.Reason); + // The only ownership-checked delete is S3's "did my own in-flight write land" probe with + // THIS run's id; nothing is ever attempted under any other owner. + await _blobs.DidNotReceive().DeleteIfOwnedByRunAsync(Arg.Any(), Arg.Is(g => g != RunId), Arg.Any()); + await _blobs.DidNotReceiveWithAnyArgs().DeleteIfExistsAsync(default!, default); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransformTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransformTests.cs new file mode 100644 index 000000000..b19b57714 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/EgressTransformTests.cs @@ -0,0 +1,113 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.Journey; +using DfE.CheckPerformanceData.Domain.Enums; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// The four transforms the preprocessing steps are built from. Each is a pure function so the step +// list can report them one at a time and so a wrong LA split or reason code is caught here, not +// in a CSV LDS has already ingested. +public sealed class EgressTransformTests +{ + [Theory] + [InlineData("8734603", "873", "4603")] + [InlineData("873/4603", "873", "4603")] + [InlineData(" 8604070 ", "860", "4070")] + public void Laestab_splits_into_three_digit_LA_and_four_digit_establishment(string raw, string la, string estab) + { + Assert.True(LaestabSplitter.TrySplit(raw, out var actualLa, out var actualEstab)); + Assert.Equal(la, actualLa); + Assert.Equal(estab, actualEstab); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("87346")] + [InlineData("87346031")] + [InlineData("ABCDEFG")] + public void Anything_but_seven_digits_is_refused(string? raw) + { + Assert.False(LaestabSplitter.TrySplit(raw, out var la, out var estab)); + Assert.Equal(string.Empty, la); + Assert.Equal(string.Empty, estab); + } + + [Theory] + [InlineData("pupil-died", "4")] + [InlineData("not-on-roll", "6")] + [InlineData("child-missing-education", "501")] + [InlineData("permanently-left-england", "3")] + [InlineData("year-group-change", "17")] + [InlineData("dual-registered-moved", "332")] + public void KS4_remove_reason_becomes_the_bare_LDS_code(string reason, string expected) + => Assert.Equal(expected, CorrectionCodes.RemoveReasonCode(CheckingWindowType.KS4June, reason, null)); + + // LDS_CYPMD_Data specification v2.4, "Remove Learner" K11 — the 16-19 codes. Not on roll is + // split by the journey's not-on-roll-reason; "other" always has evidence in CYPMD → 329. + [Theory] + [InlineData("student-died", null, "4")] + [InlineData("not-at-end-of-16-19-study", null, "325")] + [InlineData("other", null, "329")] + [InlineData("not-on-roll", "apprentice", "331")] + [InlineData("not-on-roll", "external-candidate", "328")] + [InlineData("not-on-roll", "international-student", "326")] + [InlineData("not-on-roll", "other", "329")] + public void Post16_remove_reason_becomes_the_spec_code(string reason, string? notOnRollReason, string expected) + => Assert.Equal(expected, CorrectionCodes.RemoveReasonCode(CheckingWindowType.Post16, reason, notOnRollReason)); + + [Theory] + [InlineData(CheckingWindowType.KS4June, "other", null)] + [InlineData(CheckingWindowType.KS4June, "completed-ks4-elsewhere", null)] + [InlineData(CheckingWindowType.Post16, "not-on-roll", null)] // sub-reason missing + [InlineData(CheckingWindowType.Post16, "not-on-roll", "something")] + [InlineData(CheckingWindowType.Post16, "pupil-died", null)] // KS4 wording on a 16-19 window + [InlineData(CheckingWindowType.KS4June, "", null)] + [InlineData(CheckingWindowType.KS4June, null, null)] + public void An_unmapped_reason_has_no_code_rather_than_a_guess(CheckingWindowType windowType, string? reason, string? notOnRollReason) + => Assert.Null(CorrectionCodes.RemoveReasonCode(windowType, reason, notOnRollReason)); + + [Theory] + [InlineData("2007-06-01", "2007-06-01")] + [InlineData("01/06/2007", "2007-06-01")] + [InlineData("2007-06-01 00:00:00.0000000", "2007-06-01")] + public void Dates_standardise_to_iso(string raw, string expected) + { + Assert.True(EgressDates.TryToIso(raw, out var iso)); + Assert.Equal(expected, iso); + } + + [Theory] + [InlineData("")] + [InlineData(null)] + [InlineData("31/02/2007")] + [InlineData("June 2007")] + public void Unparseable_dates_are_refused(string? raw) + => Assert.False(EgressDates.TryToIso(raw, out _)); + + [Fact] + public void Answers_flatten_to_strings_with_dates_as_iso() + { + var state = new RequestState + { + QuestionAnswers = + { + ["first-name"] = new QuestionAnswer { TextValue = " Annie " }, + ["date-of-birth"] = new QuestionAnswer { DateValue = new DateAnswer { Day = 7, Month = 9, Year = 2010 } }, + ["admission-date"] = new QuestionAnswer { DateValue = new DateAnswer() }, + ["country"] = new QuestionAnswer { CodeValue = "FR", TextValue = "France" }, + ["years"] = new QuestionAnswer { SelectedValues = ["2024-2025", "2025-2026"] }, + ["blank"] = new QuestionAnswer() + } + }; + + var flat = EgressAnswers.Flatten(state); + + Assert.Equal("Annie", flat["first-name"]); + Assert.Equal("2010-09-07", flat["date-of-birth"]); + Assert.False(flat.ContainsKey("admission-date")); // incomplete date = no answer + Assert.Equal("FR", flat["country"]); // the stable code wins over the label + Assert.Equal("2024-2025|2025-2026", flat["years"]); + Assert.False(flat.ContainsKey("blank")); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/LdsSpecValidatorTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/LdsSpecValidatorTests.cs new file mode 100644 index 000000000..a8a53a629 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/LdsSpecValidatorTests.cs @@ -0,0 +1,180 @@ +using DfE.CheckPerformanceData.Application.Egress; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// "All values in all fields are permitted values as defined by the spec" (AB#292610). One failure +// per offending field, naming the field, so the ops user can correct the source and re-run. +public sealed class LdsSpecValidatorTests +{ + [Fact] + public void A_complete_remove_row_passes() + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.Remove())); + + [Fact] + public void A_complete_new_learner_row_passes() + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.New())); + + // Review finding (18 Sep): the file goes to an external organisation and may be opened in a + // spreadsheet, where a cell beginning with = + - @ is evaluated as a formula (OWASP CSV + // injection). Surname and Forename are the only free-text cells — typed by the school in the + // Add journey. Rejecting here keeps the file spec-pure (no apostrophe prefix LDS never asked + // for) and fails the record with a named reason instead of silently rewriting a name. + [Theory] + [InlineData("=1+1")] + [InlineData("+cmd")] + [InlineData("-2")] + [InlineData("@SUM(A1)")] + public void A_new_learner_name_beginning_with_a_formula_trigger_fails_that_field(string bad) + { + var failures = LdsSpecValidator.Validate(SampleRows.New() with { Forename = bad }); + + var failure = Assert.Single(failures); + Assert.Equal("Forename", failure.Field); + Assert.Equal("must not begin with =, +, - or @", failure.Reason); + } + + [Theory] + [InlineData("=HYPERLINK(\"http://x\")")] + [InlineData("-Smith")] + public void A_remove_row_surname_beginning_with_a_formula_trigger_fails_that_field(string bad) + { + var failures = LdsSpecValidator.Validate(SampleRows.Remove() with { Surname = bad }); + + var failure = Assert.Single(failures); + Assert.Equal("Surname", failure.Field); + Assert.Equal("must not begin with =, +, - or @", failure.Reason); + } + + // A hyphen INSIDE a name is ordinary (double-barrelled surnames); only a leading trigger is refused. + [Fact] + public void A_hyphenated_surname_passes() + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.Remove() with { Surname = "Smith-Jones" })); + + [Theory] + [InlineData("Local_Authority", "87")] + [InlineData("Local_Authority", "")] + [InlineData("Establishment_Number", "460")] + [InlineData("Sex", "X")] + [InlineData("Sex", "")] + [InlineData("Date_of_Birth", "01/06/2007")] + [InlineData("Correction_Reason", "")] + [InlineData("Key_Stage", "KS3")] + [InlineData("Cycle_Month", "13")] + [InlineData("Learner_ID", "")] + [InlineData("Surname", "")] + public void A_remove_row_fails_on_the_named_field(string field, string bad) + { + var row = SampleRows.Remove(); + row = field switch + { + "Local_Authority" => row with { LocalAuthority = bad }, + "Establishment_Number" => row with { EstablishmentNumber = bad }, + "Sex" => row with { Sex = bad }, + "Date_of_Birth" => row with { DateOfBirth = bad }, + "Correction_Reason" => row with { CorrectionReason = bad }, + "Key_Stage" => row with { KeyStage = bad }, + "Cycle_Month" => row with { CycleMonth = bad }, + "Learner_ID" => row with { LearnerId = bad }, + "Surname" => row with { Surname = bad }, + _ => throw new ArgumentOutOfRangeException(nameof(field)) + }; + + var failures = LdsSpecValidator.Validate(row); + + var failure = Assert.Single(failures); + Assert.Equal(field, failure.Field); + Assert.Equal("Validate against LDS spec", failure.Step); + Assert.Equal(88856, failure.TicketId); + } + + [Theory] + [InlineData("Sex", "Z")] + [InlineData("Admission_Date", "")] + [InlineData("Year_Group", "14")] + [InlineData("UPN", "A8815412000110000")] + [InlineData("URN", "")] + public void A_new_learner_row_fails_on_the_named_field(string field, string bad) + { + var row = SampleRows.New(); + row = field switch + { + "Sex" => row with { Sex = bad }, + "Admission_Date" => row with { AdmissionDate = bad }, + "Year_Group" => row with { YearGroup = bad }, + "UPN" => row with { Upn = bad }, + "URN" => row with { SchoolUrn = bad }, + _ => throw new ArgumentOutOfRangeException(nameof(field)) + }; + + var failure = Assert.Single(LdsSpecValidator.Validate(row)); + Assert.Equal(field, failure.Field); + } + + [Fact] + public void New_learner_optional_fields_may_be_blank() + { + var row = SampleRows.New() with { Upn = "", Uln = "", LearnerId = "", Postcode = "" }; + Assert.Empty(LdsSpecValidator.Validate(row)); + } + + // v2.4: both sheets permit F, M and U ("can allow 'U' (unknown)"). + [Fact] + public void Sex_U_is_permitted_on_both_files() + { + Assert.Empty(LdsSpecValidator.Validate(SampleRows.New() with { Sex = "U" })); + Assert.Empty(LdsSpecValidator.Validate(SampleRows.Remove() with { Sex = "U" })); + } + + // v2.4 New Learner: Year_Group is 1-13 and NULL-able; ULN is varchar(11); Post_Code varchar(8). + [Theory] + [InlineData("1")] + [InlineData("13")] + [InlineData("")] + public void New_learner_year_group_may_be_blank_or_any_year_1_to_13(string yearGroup) + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.New() with { YearGroup = yearGroup })); + + [Theory] + [InlineData("Year_Group", "0")] + [InlineData("Year_Group", "14")] + [InlineData("Year_Group", "ten")] + [InlineData("ULN", "123456789012")] + [InlineData("ULN", "12345678A")] + [InlineData("Post_Code", "SW1A 1AA extra")] + public void New_learner_out_of_range_values_fail_on_the_named_field(string field, string bad) + { + var row = field switch + { + "Year_Group" => SampleRows.New() with { YearGroup = bad }, + "ULN" => SampleRows.New() with { Uln = bad }, + "Post_Code" => SampleRows.New() with { Postcode = bad }, + _ => throw new ArgumentOutOfRangeException(nameof(field)) + }; + var failure = Assert.Single(LdsSpecValidator.Validate(row)); + Assert.Equal(field, failure.Field); + } + + [Fact] + public void New_learner_eleven_digit_ULN_and_eight_character_postcode_pass() + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.New() with { Uln = "12345678901", Postcode = "SW1A 1AA" })); + + [Theory] + [InlineData("Year_Group", "14")] + [InlineData("Removal_Year_0", "yes")] + [InlineData("Removal_Year_2", "true")] // spec value is upper-case TRUE/FALSE + public void Remove_row_key_stage_specific_values_fail_on_the_named_field(string field, string bad) + { + var row = field switch + { + "Year_Group" => SampleRows.Remove() with { YearGroup = bad }, + "Removal_Year_0" => SampleRows.Remove() with { RemovalYear0 = bad }, + "Removal_Year_2" => SampleRows.Remove() with { RemovalYear2 = bad }, + _ => throw new ArgumentOutOfRangeException(nameof(field)) + }; + var failure = Assert.Single(LdsSpecValidator.Validate(row)); + Assert.Equal(field, failure.Field); + } + + [Fact] + public void Remove_row_key_stage_specific_values_may_be_blank_or_valid() + => Assert.Empty(LdsSpecValidator.Validate(SampleRows.Remove() with { YearGroup = "12", RemovalYear0 = "TRUE", RemovalYear1 = "FALSE", RemovalYear2 = "" })); +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Egress/ZendeskEgressTicketSourceTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Egress/ZendeskEgressTicketSourceTests.cs new file mode 100644 index 000000000..30625ab1a --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Egress/ZendeskEgressTicketSourceTests.cs @@ -0,0 +1,92 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.ZendeskClient; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.Infrastructure.ZendeskClient; +using DfE.CheckPerformanceData.Infrastructure.ZendeskClient.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Egress; + +// The real source reads ONE custom field ("Decision status") from tickets fetched by id in +// batches of 100 — Zendesk's show_many ceiling. Production configures no field ids at all, so an +// unconfigured id must refuse loudly rather than return "every ticket unknown". +public sealed class ZendeskEgressTicketSourceTests +{ + private const long DecisionFieldId = 19056253670034; + private readonly IZendeskApi _api = Substitute.For(); + + private ZendeskEgressTicketSource Sut(long? fieldId = DecisionFieldId) => new( + _api, + Options.Create(new ZendeskTicketFieldSettings { DecisionStatusId = fieldId }), + // Polly's RetryStrategyOptions requires MaxRetryAttempts >= 1 (0 fails options validation); + // 1 keeps retries minimal so a failing-API test still runs fast. + Options.Create(new PollySettings { MaxRetryAttempts = 1, BaseDelayMilliseconds = 1, JitterMilliseconds = 0 }), + Substitute.For>()); + + private static Ticket TicketWith(long id, params (long FieldId, object? Value)[] fields) => new() + { + Id = id, + CustomFields = fields.Select(f => new CustomField { Id = f.FieldId, Value = f.Value }).ToList() + }; + + [Fact] + public async Task Reads_the_decision_field_for_each_ticket_and_omits_tickets_zendesk_did_not_return() + { + // All four requested ids go into one call (batch size 100); Zendesk simply omits ticket 4 + // from its response, which is the "not returned by Zendesk" case this test is pinning. + _api.ShowManyTickets("1,2,3,4").Returns(new ListViewTicketsResponse + { + Tickets = [TicketWith(1, (DecisionFieldId, "auto_approved")), TicketWith(2, (DecisionFieldId, "Rejected")), TicketWith(3, (999, "x"))] + }); + + var result = await Sut().GetDecisionStatusesAsync([1, 2, 3, 4], CancellationToken.None); + + Assert.Equal("auto_approved", result[1]); + Assert.Equal("rejected", result[2]); + Assert.False(result.ContainsKey(3)); // no decision field on the ticket = unknown, not "approved" + Assert.False(result.ContainsKey(4)); // not returned by Zendesk + } + + [Fact] + public async Task Batches_ids_one_hundred_at_a_time() + { + var ids = Enumerable.Range(1, 250).Select(i => (long)i).ToList(); + _api.ShowManyTickets(Arg.Any()).Returns(ci => new ListViewTicketsResponse + { + Tickets = ci.Arg().Split(',').Select(s => TicketWith(long.Parse(s), (DecisionFieldId, "approved"))).ToList() + }); + + var result = await Sut().GetDecisionStatusesAsync(ids, CancellationToken.None); + + Assert.Equal(250, result.Count); + await _api.Received(3).ShowManyTickets(Arg.Any()); + await _api.Received(1).ShowManyTickets(Arg.Is(s => s.Split(',').Length == 50)); + } + + [Fact] + public async Task No_ids_means_no_call() + { + var result = await Sut().GetDecisionStatusesAsync([], CancellationToken.None); + Assert.Empty(result); + await _api.DidNotReceive().ShowManyTickets(Arg.Any()); + } + + [Theory] + [InlineData(null)] + [InlineData(0L)] + public async Task An_unconfigured_decision_field_refuses_to_pull(long? fieldId) + { + var ex = await Assert.ThrowsAsync(() => Sut(fieldId).GetDecisionStatusesAsync([1], CancellationToken.None)); + Assert.Contains("Decision status", ex.Message); + await _api.DidNotReceive().ShowManyTickets(Arg.Any()); + } + + [Fact] + public async Task An_api_failure_surfaces_as_a_ticket_source_exception() + { + _api.ShowManyTickets(Arg.Any()).Returns>(_ => throw new HttpRequestException("boom")); + await Assert.ThrowsAsync(() => Sut().GetDecisionStatusesAsync([1], CancellationToken.None)); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Journey/RequestServiceTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Journey/RequestServiceTests.cs index 116e71a10..bee757d99 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/Journey/RequestServiceTests.cs +++ b/tests/DfE.CheckPerformanceData.UnitTests/Journey/RequestServiceTests.cs @@ -665,6 +665,22 @@ await _requestNotificationService.DidNotReceive() .NotifySubmissionConfirmedAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } + [Fact] + public async Task Submit_stamps_the_schools_laestab_on_the_request_row() + { + // AB#294553: the egress needs the school's establishment number for a new learner, and the + // synthetic Add pupil has none — so the row carries the claim value from submit onward. + var (journey, config) = MakeSubmission(); + SetupConfig(config); + ChangeRequestData? captured = null; + _requestRepository.UpsertAsync(Arg.Do(d => captured = d)).Returns(Guid.NewGuid()); + _currentUser.OrganisationLaestab.Returns("860/4070"); + + await _sut.SubmitRequestAsync(WindowId, journey); + + Assert.Equal("860/4070", captured!.OrganisationLaestab); + } + [Fact] public async Task ConfirmRequestAsync_SavesJourneyBlobBeforeSendingEmail() { diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Startup/BlobStorageExtensionsTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Startup/BlobStorageExtensionsTests.cs index 7afac48a2..401ce49b6 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/Startup/BlobStorageExtensionsTests.cs +++ b/tests/DfE.CheckPerformanceData.UnitTests/Startup/BlobStorageExtensionsTests.cs @@ -1,3 +1,4 @@ +using Azure.Storage.Blobs; using DfE.CheckPerformanceData.Application.CheckYourPupilData; using DfE.CheckPerformanceData.Application.Journey; using DfE.CheckPerformanceData.Application.RequestSubmission; @@ -46,6 +47,32 @@ private static ServiceProvider BuildWebBlobServices() return services.BuildServiceProvider(validateScopes: true); } + private static IReadOnlyDictionary BuildClients(Dictionary values) + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddMemoryCache(); + services.AddSingleton(new StubHostEnvironment()); + services.AddCpdBlobStorage(configuration); + using var provider = services.BuildServiceProvider(validateScopes: true); + return provider.GetRequiredService>(); + } + + [Fact] + public void Registers_the_egress_client_only_when_its_connection_string_is_present() + { + var with = BuildClients(new Dictionary + { + ["ConnectionStrings:AzureStorage"] = "UseDevelopmentStorage=true", + ["ConnectionStrings:EgressStorage"] = "UseDevelopmentStorage=true" + }); + Assert.True(with.ContainsKey("egress")); + + var without = BuildClients(new Dictionary { ["ConnectionStrings:AzureStorage"] = "UseDevelopmentStorage=true" }); + Assert.False(without.ContainsKey("egress")); + } + private sealed class StubHostEnvironment : IHostEnvironment { public string EnvironmentName { get; set; } = Environments.Production; diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Startup/EgressExtensionsTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Startup/EgressExtensionsTests.cs new file mode 100644 index 000000000..81f878e97 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Startup/EgressExtensionsTests.cs @@ -0,0 +1,104 @@ +using Azure.Storage.Blobs; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Infrastructure.Egress; +using DfE.CheckPerformanceData.Persistence.Contexts; +using DfE.CheckPerformanceData.Web.Startup; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Startup; + +// The web host never registered a Zendesk client before this feature. Review finding B1: the safe +// default is the real Zendesk client (matching the worker's configured default), never the dev +// outbox fake — the fake is opt-in only, and never available in Production regardless of config. +public sealed class EgressExtensionsTests +{ + private static IHostEnvironment Env(string environmentName) + { + var env = Substitute.For(); + env.EnvironmentName = environmentName; + return env; + } + + private static ServiceProvider Build(Dictionary values, string environmentName = "Development") + { + var config = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + var services = new ServiceCollection(); + services.AddSingleton(config); + services.AddLogging(); + services.AddSingleton(Substitute.For()); + services.AddSingleton>(new Dictionary()); + services.AddSingleton(TimeProvider.System); + services.AddCpdEgress(config, Env(environmentName)); + return services.BuildServiceProvider(); + } + + // B1(a): with no Zendesk:UseFake configured at all, the code takes the real-client branch — + // proven here because the real branch's own settings validation is what throws. Before the + // fix this scenario silently resolved the dev outbox fake instead. + [Fact] + public void Default_is_the_real_zendesk_source_so_missing_settings_fail_fast() + { + var ex = Assert.Throws( + () => Build(new Dictionary())); + Assert.Contains("section is missing", ex.Message); + } + + [Fact] + public void UseFake_true_in_a_non_production_environment_selects_the_dev_outbox_source() + { + using var sp = Build( + new Dictionary { ["Zendesk:UseFake"] = "true" }, + environmentName: "Development"); + using var scope = sp.CreateScope(); + Assert.IsType(scope.ServiceProvider.GetRequiredService()); + Assert.NotNull(scope.ServiceProvider.GetRequiredService()); + } + + [Fact] + public void UseFake_true_in_production_refuses_at_startup() + { + var ex = Assert.Throws(() => Build( + new Dictionary { ["Zendesk:UseFake"] = "true" }, + environmentName: "Production")); + Assert.Contains("Production", ex.Message); + } + + [Fact] + public void Real_zendesk_is_selected_when_the_fake_is_switched_off() + { + using var sp = Build(new Dictionary + { + ["Zendesk:UseFake"] = "false", + ["ZendeskSettings:Subdomain"] = "esfa-preprod", + ["ZendeskSettings:Domain"] = "zendesk", + ["ZendeskSettings:ClientId"] = "id", + ["ZendeskSettings:ClientSecret"] = "secret", + ["SchoolCheckingExercise:TargetViewTitle"] = "View" + }); + using var scope = sp.CreateScope(); + Assert.IsType(scope.ServiceProvider.GetRequiredService()); + } + + // B1(b): Default_is_the_real_zendesk_source_so_missing_settings_fail_fast above only proves + // the default indirectly (missing settings on the real branch throw). This proves it directly: + // with no Zendesk:UseFake key present at all — not "false", simply absent — and full, + // complete ZendeskSettings configured, DI resolves the real client type outright. + [Fact] + public void Default_resolves_the_real_zendesk_client_when_UseFake_is_not_configured_at_all() + { + using var sp = Build(new Dictionary + { + ["ZendeskSettings:Subdomain"] = "esfa-preprod", + ["ZendeskSettings:Domain"] = "zendesk", + ["ZendeskSettings:ClientId"] = "id", + ["ZendeskSettings:ClientSecret"] = "secret", + ["SchoolCheckingExercise:TargetViewTitle"] = "View" + }); + using var scope = sp.CreateScope(); + Assert.IsType(scope.ServiceProvider.GetRequiredService()); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryGroupingTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryGroupingTests.cs index 56fe223a5..392a63899 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryGroupingTests.cs +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryGroupingTests.cs @@ -25,10 +25,11 @@ public void Groups_Have_Null_ParentKey() var groups = entries.Where(e => e.ParentKey is null).ToList(); - // Six: the Amendment requests group was retired (its per-window requests page is reached + // Seven: the Amendment requests group was retired (its per-window requests page is reached // from the windows table now) and the Storage administration group went with the blob - // browser moving under Danger zone, which is itself a root group registered everywhere. - Assert.Equal(6, groups.Count); + // browser moving under Danger zone, which is itself a root group registered everywhere; + // plus the Data egress group (AB#294553). + Assert.Equal(7, groups.Count); var groupKeys = groups.Select(g => g.Key).ToHashSet(); Assert.Contains("dashboard", groupKeys); Assert.Contains("cms-admin", groupKeys); @@ -37,6 +38,7 @@ public void Groups_Have_Null_ParentKey() Assert.Contains("danger-zone", groupKeys); Assert.Contains("window-admin", groupKeys); Assert.Contains("messages-group", groupKeys); + Assert.Contains("egress-group", groupKeys); } // --- Tiles_Have_NonNull_ParentKey_Matching_An_Existing_Entry --- @@ -67,11 +69,12 @@ public void All_Keys_Are_Unique_Across_Registrations() var keys = entries.Select(e => e.Key).ToList(); - // 30: the Amendment requests group and its Uncommitted requests tile were retired + // 32: the Amendment requests group and its Uncommitted requests tile were retired // together and the Storage administration group went when the blob browser moved under // Danger zone, which this overload registers along with the browser but without the - // gated Reset seed data tile. - Assert.Equal(30, keys.Count); + // gated Reset seed data tile — plus the Data egress group and its Start a new egress + // tile (AB#294553). + Assert.Equal(32, keys.Count); Assert.Equal(keys.Count, keys.Distinct().Count()); } diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryTests.cs index ba69fd562..7a19527d7 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryTests.cs +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/Admin/AdminNavRegistryTests.cs @@ -17,12 +17,13 @@ public void AddAdminNavEntries_Registers_Hierarchical_Entries() using var provider = services.BuildServiceProvider(); var entries = provider.GetServices().ToList(); - // 30: the Amendment requests group and its single Uncommitted requests tile were retired + // 32: the Amendment requests group and its single Uncommitted requests tile were retired // (the requests page is per window now, reached from the windows table) and the Storage // administration group went when the blob browser moved under Danger zone; the Danger // zone group and that browser are registered unconditionally, Reset seed data is not - // (includeResetSeedData defaults to false). - Assert.Equal(30, entries.Count); + // (includeResetSeedData defaults to false) — plus the Data egress group and its + // Start a new egress tile (AB#294553). + Assert.Equal(32, entries.Count); var titles = entries.Select(e => e.Title).ToList(); Assert.Contains("Dashboard", titles); diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/AppSettingsEgressStorageTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/AppSettingsEgressStorageTests.cs new file mode 100644 index 000000000..d52460f00 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/AppSettingsEgressStorageTests.cs @@ -0,0 +1,38 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Web; + +// The LDS egress account is "absent unless the environment supplies it": with no +// ConnectionStrings:EgressStorage the egress refuses to transfer with a clear message and the dev +// cleanup skips its blob sweep. A local-Azurite default in appsettings.json broke that contract on +// every deployed environment that had not been given an account — the app believed it had one at +// 127.0.0.1:10000 inside the pod, every transfer and every cleanup failed after the SDK's retries, +// and the review app for PR #441 was left with egress rows its start-up seeder could not delete. +// Local runs get the string from docker-compose.yaml and the launch profiles, never from here. +public sealed class AppSettingsEgressStorageTests +{ + [Fact] + public void The_egress_storage_connection_string_has_no_default_in_appsettings() + { + var json = ReadAppSettings("appsettings.json"); + Assert.True(json.RootElement.TryGetProperty("ConnectionStrings", out var connectionStrings)); + + Assert.False(connectionStrings.TryGetProperty("EgressStorage", out _), + "appsettings.json must not ship a ConnectionStrings:EgressStorage default — an environment without an " + + "egress account must read as 'not configured', not as a dead local Azurite endpoint."); + } + + private static JsonDocument ReadAppSettings(string fileName, [CallerFilePath] string testFile = "") + { + var dir = new DirectoryInfo(Path.GetDirectoryName(testFile)!); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "Makefile"))) + { + dir = dir.Parent; + } + if (dir is null) throw new InvalidOperationException("Could not locate repository root."); + + var path = Path.Combine(dir.FullName, "src", "DfE.CheckPerformanceData.Web", fileName); + return JsonDocument.Parse(File.ReadAllText(path)); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/DevEgressControllerTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/DevEgressControllerTests.cs new file mode 100644 index 000000000..63909899c --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/DevEgressControllerTests.cs @@ -0,0 +1,33 @@ +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.RequestSubmission; +using DfE.CheckPerformanceData.Persistence.Contexts; +using DfE.CheckPerformanceData.Web.Controllers; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Web.Controllers; + +// The seeder writes rows that look like real requests, so it must be unreachable wherever the +// dev tools are off and in Production regardless of the flag — the DevPipelineController rule. +public sealed class DevEgressControllerTests +{ + private static DevEgressController Build(bool toolsEnabled, string environment) + { + var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["Dev:ToolsEnabled"] = toolsEnabled ? "true" : "false" }).Build(); + var env = Substitute.For(); + env.EnvironmentName.Returns(environment); + return new DevEgressController(config, Substitute.For(), Substitute.For(), Substitute.For(), env); + } + + [Theory] + [InlineData(false, "Development")] + [InlineData(true, "Production")] + public async Task Seed_and_cleanup_are_404_when_not_allowed(bool toolsEnabled, string environment) + { + var sut = Build(toolsEnabled, environment); + Assert.IsType(await sut.Seed(Guid.NewGuid(), "RemoveLearners", "approved", 1, "860/4070", 142313, "pupil-died", CancellationToken.None)); + Assert.IsType(await sut.Cleanup(Guid.NewGuid(), CancellationToken.None)); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/EgressControllerTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/EgressControllerTests.cs new file mode 100644 index 000000000..c379ae978 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/Controllers/EgressControllerTests.cs @@ -0,0 +1,503 @@ +using System.Reflection; +using DfE.CheckPerformanceData.Application.CurrentUser; +using DfE.CheckPerformanceData.Application.Egress; +using DfE.CheckPerformanceData.Application.WindowManagement; +using DfE.CheckPerformanceData.Domain.Enums; +using DfE.CheckPerformanceData.Web.Admin; +using DfE.CheckPerformanceData.Web.Admin.Nav; +using DfE.CheckPerformanceData.Web.Controllers.Egress; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ViewFeatures; +using NSubstitute; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Web.Controllers; + +public sealed class EgressControllerTests +{ + private static readonly Guid RunId = Guid.Parse("33333333-3333-3333-3333-333333333333"); + private static readonly Guid WindowId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + private readonly IEgressRunService _runs = Substitute.For(); + private readonly IEgressPreprocessor _preprocessor = Substitute.For(); + private readonly IEgressTransferService _transfer = Substitute.For(); + private readonly IEgressBlobClient _blobs = Substitute.For(); + private readonly IWindowService _windows = Substitute.For(); + private readonly ICurrentUserService _user = Substitute.For(); + + private EgressController Build() + { + _user.UserId.Returns("22222222-2222-2222-2222-222222222222"); + _user.DisplayName.Returns("Ops One"); + _user.Email.Returns("ops@example.com"); + _blobs.TargetDescription.Returns("cypmd/extracts_input"); + _windows.GetAllDataAsync(Arg.Any()).Returns(new PageResult { Windows = [Window()] }); + _runs.ListAsync(Arg.Any()).Returns([]); + var controller = new EgressController(_runs, _preprocessor, _transfer, _blobs, _windows, _user) + { + ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() } + }; + controller.TempData = new TempDataDictionary(controller.HttpContext, Substitute.For()); + return controller; + } + + private static CheckingWindowDto Window() => new() + { + Id = WindowId, Title = "KS4 June 2026", KeyStage = KeyStages.KS4, CheckingWindowType = CheckingWindowType.KS4June, + StartDate = new DateTime(2026, 6, 1), EndDate = new DateTime(2026, 6, 30) + }; + + private static EgressRunDto Run(EgressRunStatus status) => new(RunId, WindowId, status, Guid.NewGuid(), "Ops One", DateTime.UtcNow, + null, null, null, null, [], null, [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 0, null, null, null)]); + + [Fact] + public void Controller_is_gated_by_the_egress_section() + { + var gate = typeof(EgressController).GetCustomAttribute(); + Assert.NotNull(gate); + Assert.Equal(AdminNavKeys.Egress, gate!.SectionKey); + } + + [Fact] + public async Task Start_with_nothing_selected_redisplays_with_errors() + { + var result = await Build().Start(new PullForm { WindowId = null, OutputTypes = [] }, CancellationToken.None); + + var view = Assert.IsType(result); + Assert.Equal("Index", view.ViewName); + var model = Assert.IsType(view.Model); + Assert.False(model.IsValid); + await _runs.DidNotReceiveWithAnyArgs().StartAsync(default, default!, default!, default); + } + + // S9: an unbindable OutputTypes value (e.g. OutputTypes=garbage) previously bound as + // default(EgressOutputType) — NewLearners — with nobody reading the resulting ModelState + // error, so the request silently proceeded. A window is selected here so only the + // OutputTypes binding failure is under test. + [Fact] + public async Task Start_with_an_unbindable_output_type_redisplays_with_an_error_rather_than_defaulting() + { + var controller = Build(); + controller.ModelState.AddModelError(nameof(PullForm.OutputTypes), "The value 'garbage' is not valid."); + + var result = await controller.Start( + new PullForm { WindowId = WindowId, OutputTypes = [EgressOutputType.NewLearners] }, CancellationToken.None); + + var view = Assert.IsType(result); + Assert.Equal("Index", view.ViewName); + var model = Assert.IsType(view.Model); + Assert.NotNull(model.OutputTypesError); + await _runs.DidNotReceiveWithAnyArgs().StartAsync(default, default!, default!, default); + } + + [Fact] + public async Task Start_redirects_to_results_and_passes_the_actor() + { + EgressActor? actor = null; + _runs.StartAsync(WindowId, Arg.Any>(), Arg.Do(a => actor = a), Arg.Any()) + .Returns(new EgressStartResult.Started(RunId)); + + var result = await Build().Start(new PullForm { WindowId = WindowId, OutputTypes = [EgressOutputType.RemoveLearners] }, CancellationToken.None); + + var redirect = Assert.IsType(result); + Assert.Equal(nameof(EgressController.Results), redirect.ActionName); + Assert.Equal(RunId, redirect.RouteValues!["id"]); + Assert.Equal("Ops One", actor!.DisplayName); + } + + [Fact] + public async Task Start_shows_who_holds_the_pair_when_refused() + { + var blocker = new EgressBlocker(Guid.NewGuid(), EgressRunStatus.Transferred, "Ops Two", new DateTime(2026, 6, 8, 9, 0, 0, DateTimeKind.Utc), + new DateTime(2026, 6, 8, 14, 38, 0, DateTimeKind.Utc), "Ops Two"); + _runs.StartAsync(WindowId, Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(new EgressStartResult.Refused([(EgressOutputType.RemoveLearners, blocker)])); + + var view = Assert.IsType(await Build().Start(new PullForm { WindowId = WindowId, OutputTypes = [EgressOutputType.RemoveLearners] }, CancellationToken.None)); + + var model = Assert.IsType(view.Model); + var refusal = Assert.Single(model.Refusals); + Assert.Contains("Remove learners", refusal); + Assert.Contains("Ops Two", refusal); + Assert.Contains("already been transferred", refusal); + } + + [Theory] + [InlineData(EgressRunStatus.Pulled, nameof(EgressController.Results))] + [InlineData(EgressRunStatus.Preprocessing, nameof(EgressController.Preprocessing))] + [InlineData(EgressRunStatus.PreprocessingFailed, nameof(EgressController.Failed))] + [InlineData(EgressRunStatus.Preprocessed, nameof(EgressController.Summary))] + [InlineData(EgressRunStatus.TransferFailed, nameof(EgressController.Summary))] + [InlineData(EgressRunStatus.Transferring, nameof(EgressController.Summary))] + [InlineData(EgressRunStatus.Transferred, nameof(EgressController.Complete))] + public async Task Resume_opens_the_page_the_status_implies(EgressRunStatus status, string action) + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(status)); + var redirect = Assert.IsType(await Build().Resume(RunId, CancellationToken.None)); + Assert.Equal(action, redirect.ActionName); + } + + [Fact] + public async Task Resume_of_an_abandoned_or_unknown_run_goes_home_with_a_message() + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Abandoned)); + var redirect = Assert.IsType(await Build().Resume(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Index), redirect.ActionName); + + _runs.GetAsync(RunId, Arg.Any()).Returns((EgressRunDto?)null); + Assert.IsType(await Build().Resume(RunId, CancellationToken.None)); + } + + // Second-pass nit: EgressTransferService's "abandoned while the transfer was in progress" + // Failed result sets TransferErrorKey then Transfer redirects to Summary — but Summary + // redirects an Abandoned run straight on to Resume without ever reading it, so the entry was + // left to surface incorrectly as a transfer error on the next unrelated run's Summary page. + // Clearing it in Resume's abandoned-run branch stops that leak at its one landing point. + [Fact] + public async Task Resume_of_an_abandoned_run_clears_any_leftover_transfer_error() + { + var controller = Build(); + controller.TempData[EgressController.TransferErrorKey] = "This run was abandoned while the transfer was in progress."; + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Abandoned)); + + await controller.Resume(RunId, CancellationToken.None); + + Assert.Null(controller.TempData[EgressController.TransferErrorKey]); + } + + [Theory] + [InlineData(EgressRunStatus.Pulled)] + [InlineData(EgressRunStatus.Abandoned)] + public async Task Summary_refuses_a_run_that_is_not_ready(EgressRunStatus status) + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(status)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + var redirect = Assert.IsType(await Build().Summary(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Resume), redirect.ActionName); + } + + // Review finding (18 Sep): ConnectionStrings:EgressStorage arrives only from deployment + // configuration, and the only check was at the very last step — an ops user could pull, + // preprocess and reach Summary before learning the environment had no target account. The + // Pull page and the Summary now carry the flag so the views can warn up front. Pull and + // preprocess are NOT blocked: preview and download are still useful without an account. + [Fact] + public async Task Index_tells_the_view_when_egress_storage_is_not_configured() + { + _blobs.IsConfigured.Returns(false); + + var view = Assert.IsType(await Build().Index(CancellationToken.None)); + + var model = Assert.IsType(view.Model); + Assert.True(model.StorageNotConfigured); + } + + [Fact] + public async Task Summary_tells_the_view_when_egress_storage_is_not_configured() + { + _blobs.IsConfigured.Returns(false); + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Preprocessed)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + + var view = Assert.IsType(await Build().Summary(RunId, CancellationToken.None)); + + var model = Assert.IsType(view.Model); + Assert.True(model.StorageNotConfigured); + } + + [Fact] + public async Task Index_does_not_warn_when_egress_storage_is_configured() + { + _blobs.IsConfigured.Returns(true); + + var view = Assert.IsType(await Build().Index(CancellationToken.None)); + + Assert.False(Assert.IsType(view.Model).StorageNotConfigured); + } + + // S10: Start's other two non-happy outcomes had no coverage. + [Fact] + public async Task Start_shows_the_window_error_when_the_window_no_longer_exists() + { + _runs.StartAsync(WindowId, Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(new EgressStartResult.WindowNotFound()); + + var view = Assert.IsType(await Build().Start(new PullForm { WindowId = WindowId, OutputTypes = [EgressOutputType.RemoveLearners] }, CancellationToken.None)); + + var model = Assert.IsType(view.Model); + Assert.Equal("Select a checking window", model.WindowError); + } + + [Fact] + public async Task Start_shows_the_pull_error_when_zendesk_could_not_be_read() + { + _runs.StartAsync(WindowId, Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(new EgressStartResult.PullFailed("Zendesk timed out")); + + var view = Assert.IsType(await Build().Start(new PullForm { WindowId = WindowId, OutputTypes = [EgressOutputType.RemoveLearners] }, CancellationToken.None)); + + var model = Assert.IsType(view.Model); + Assert.Equal("Zendesk timed out", model.PullError); + } + + // S10: Results/Failed/Complete each guard their status but had no test. + [Theory] + [InlineData(EgressRunStatus.Pulled)] + [InlineData(EgressRunStatus.PreprocessingFailed)] + [InlineData(EgressRunStatus.Preprocessed)] + public async Task Results_renders_for_the_statuses_it_allows(EgressRunStatus status) + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(status)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + var view = Assert.IsType(await Build().Results(RunId, CancellationToken.None)); + Assert.Equal("Results", view.ViewName); + } + + [Fact] + public async Task Results_redirects_to_resume_for_a_status_it_does_not_allow() + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Transferring)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + var redirect = Assert.IsType(await Build().Results(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Resume), redirect.ActionName); + } + + // Nit: a Transferred/Abandoned run has nothing left to preprocess. A PreprocessingFailed run + // has nothing left to preprocess either — the POST already refuses it, and until this fix the + // GET still rendered the "Run preprocessing" button/page for it regardless. + [Theory] + [InlineData(EgressRunStatus.Transferred)] + [InlineData(EgressRunStatus.Abandoned)] + [InlineData(EgressRunStatus.PreprocessingFailed)] + public async Task Preprocessing_redirects_to_resume_for_a_finished_run(EgressRunStatus status) + { + var run = Run(status) with { Outputs = [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 3, null, null, null)] }; + _runs.GetAsync(RunId, Arg.Any()).Returns(run); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + + var redirect = Assert.IsType(await Build().Preprocessing(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Resume), redirect.ActionName); + } + + [Fact] + public async Task Failed_renders_only_for_preprocessing_failed() + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.PreprocessingFailed)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + var view = Assert.IsType(await Build().Failed(RunId, CancellationToken.None)); + Assert.Equal("Failed", view.ViewName); + + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Pulled)); + var redirect = Assert.IsType(await Build().Failed(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Resume), redirect.ActionName); + } + + [Fact] + public async Task Complete_renders_only_for_transferred() + { + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Transferred)); + _windows.GetByIdAsync(WindowId, Arg.Any()).Returns(Window()); + var view = Assert.IsType(await Build().Complete(RunId, CancellationToken.None)); + Assert.Equal("Complete", view.ViewName); + + _runs.GetAsync(RunId, Arg.Any()).Returns(Run(EgressRunStatus.Preprocessed)); + var redirect = Assert.IsType(await Build().Complete(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Resume), redirect.ActionName); + } + + // S10: no PreprocessingRun (the no-JS fallback) outcome was covered. + private static async IAsyncEnumerable One(EgressProgress p) { await Task.Yield(); yield return p; } + + [Fact] + public async Task PreprocessingRun_redirects_to_summary_when_the_pipeline_completes() + { + _preprocessor.RunAsync(RunId, Arg.Any()) + .Returns(One(new EgressProgress(8, 8, "Save to database", "done", 1, 1, 0, true, false, "Saved 1 record(s).", EgressRunStatus.Preprocessed))); + + var redirect = Assert.IsType(await Build().PreprocessingRun(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Summary), redirect.ActionName); + } + + [Fact] + public async Task PreprocessingRun_redirects_to_failed_when_the_pipeline_fails() + { + _preprocessor.RunAsync(RunId, Arg.Any()) + .Returns(One(new EgressProgress(8, 8, "Save to database", "failed", 1, 0, 1, true, true, "Preprocessing stopped.", EgressRunStatus.PreprocessingFailed))); + + var redirect = Assert.IsType(await Build().PreprocessingRun(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Failed), redirect.ActionName); + } + + [Fact] + public async Task PreprocessingRun_goes_home_with_the_message_when_it_does_not_complete() + { + _preprocessor.RunAsync(RunId, Arg.Any()) + .Returns(One(new EgressProgress(0, 8, "Preprocessing", "failed", 0, 0, 0, true, true, "This run is Transferring and cannot be preprocessed.", null))); + + var controller = Build(); + var redirect = Assert.IsType(await controller.PreprocessingRun(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Index), redirect.ActionName); + Assert.Equal("This run is Transferring and cannot be preprocessed.", controller.TempData[EgressController.BannerKey]); + } + + // S10/Nit: Preview must come from the saved rows via EgressColumnSets, not by splitting the + // CSV text — a quoted value containing a comma (a real surname, e.g. "Smith, Jr") would + // otherwise shift every column after it. + [Fact] + public async Task Preview_builds_the_table_from_the_saved_rows_not_the_csv_text() + { + var run = Run(EgressRunStatus.Preprocessed) with + { + Outputs = [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 2, 2, "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", null)] + }; + _runs.GetAsync(RunId, Arg.Any()).Returns(run); + _transfer.GetPreviewAsync(RunId, EgressOutputType.RemoveLearners, Arg.Any()) + .Returns((EgressColumnSets.RemoveLearners.Select(c => c.Header).ToList(), + (IReadOnlyList>)[["1001", "31", "4", "KS4", "4070", "Smith, Jr", "Alice", "F", "2010-09-07", "2026", "6", "860", "555"]])); + + var view = Assert.IsType(await Build().Preview(RunId, EgressOutputType.RemoveLearners, CancellationToken.None)); + + Assert.Equal("Preview", view.ViewName); + var model = Assert.IsType(view.Model); + Assert.Equal(EgressColumnSets.RemoveLearners.Select(c => c.Header), model.Headers); + var row = Assert.Single(model.Rows); + Assert.Equal("Smith, Jr", row[5]); + Assert.Equal(13, row.Count); + await _transfer.DidNotReceiveWithAnyArgs().BuildFileAsync(default, default, default); + } + + [Fact] + public async Task Preview_404s_when_the_run_has_not_reached_this_output_yet() + { + var run = Run(EgressRunStatus.Pulled); + _runs.GetAsync(RunId, Arg.Any()).Returns(run); + + Assert.IsType(await Build().Preview(RunId, EgressOutputType.RemoveLearners, CancellationToken.None)); + } + + // S10: only the success/Failed branches of Transfer were covered; Refused was not. + // Nit: EgressTransferResult.Refused now carries the specific output type instead of the + // controller hard-coding RemoveLearners and string-replacing the description — NewLearners + // here proves the real type is used, not the old placeholder. + [Fact] + public async Task Transfer_refused_shows_who_holds_the_pair_and_returns_to_summary() + { + var blocker = new EgressBlocker(Guid.NewGuid(), EgressRunStatus.Pulled, "Ops Two", DateTime.UtcNow, null, null); + _transfer.TransferAsync(RunId, Arg.Any(), Arg.Any()) + .Returns(new EgressTransferResult.Refused(EgressOutputType.NewLearners, blocker)); + + var controller = Build(); + var redirect = Assert.IsType(await controller.Transfer(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Summary), redirect.ActionName); + var message = Assert.IsType(controller.TempData[EgressController.TransferErrorKey]); + Assert.Contains("Ops Two", message); + Assert.Contains("New learners", message); + } + + [Fact] + public async Task Transfer_of_an_unknown_run_returns_not_found() + { + _transfer.TransferAsync(RunId, Arg.Any(), Arg.Any()) + .Returns(new EgressTransferResult.NotFound()); + + Assert.IsType(await Build().Transfer(RunId, CancellationToken.None)); + } + + [Fact] + public async Task Transfer_success_redirects_to_complete_and_failure_back_to_summary_with_the_reason() + { + _transfer.TransferAsync(RunId, Arg.Any(), Arg.Any()) + .Returns(new EgressTransferResult.Transferred([(EgressOutputType.RemoveLearners, "f.csv", 2)], DateTime.UtcNow)); + var ok = Assert.IsType(await Build().Transfer(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Complete), ok.ActionName); + + _transfer.TransferAsync(RunId, Arg.Any(), Arg.Any()) + .Returns(new EgressTransferResult.Failed("Blob upload refused")); + var controller = Build(); + var back = Assert.IsType(await controller.Transfer(RunId, CancellationToken.None)); + Assert.Equal(nameof(EgressController.Summary), back.ActionName); + Assert.Equal("Blob upload refused", controller.TempData[EgressController.TransferErrorKey]); + } + + // M3: never Complete for an empty approved set. + [Fact] + public async Task Transfer_of_an_empty_approved_set_redirects_to_summary_not_complete() + { + _transfer.TransferAsync(RunId, Arg.Any(), Arg.Any()) + .Returns(new EgressTransferResult.NothingToTransfer()); + + var redirect = Assert.IsType(await Build().Transfer(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Summary), redirect.ActionName); + } + + [Fact] + public async Task Download_returns_csv_with_the_runs_file_name() + { + var run = Run(EgressRunStatus.Preprocessed) with + { + Outputs = [new EgressRunOutputDto(Guid.NewGuid(), EgressOutputType.RemoveLearners, true, [], 2, 2, "CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", null)] + }; + _runs.GetAsync(RunId, Arg.Any()).Returns(run); + _transfer.BuildFileAsync(RunId, EgressOutputType.RemoveLearners, Arg.Any()).Returns([65, 44, 66]); + + var file = Assert.IsType(await Build().Download(RunId, EgressOutputType.RemoveLearners, CancellationToken.None)); + + Assert.Equal("text/csv; charset=utf-8", file.ContentType); + Assert.Equal("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", file.FileDownloadName); + } + + // S10: rewritten to assert the banner text the outcome actually produces, not merely that the + // mock was invoked — the four EgressAbandonResult branches each need a distinct message. + [Fact] + public async Task Abandon_of_an_untransferred_run_reports_nothing_was_transferred() + { + _transfer.AbandonAsync(RunId, Arg.Any()).Returns(new EgressAbandonResult.Abandoned([])); + var controller = Build(); + + var redirect = Assert.IsType(await controller.Abandon(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Index), redirect.ActionName); + Assert.Equal("The egress run was abandoned. Nothing was transferred.", controller.TempData[EgressController.BannerKey]); + } + + [Fact] + public async Task Abandon_of_a_run_that_swept_blobs_names_what_was_removed() + { + _transfer.AbandonAsync(RunId, Arg.Any()) + .Returns(new EgressAbandonResult.Abandoned(["CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv"])); + var controller = Build(); + + var redirect = Assert.IsType(await controller.Abandon(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Index), redirect.ActionName); + var banner = Assert.IsType(controller.TempData[EgressController.BannerKey]); + Assert.Contains("CYPMD_LDS_KS4_RemoveLearners_2026_06_08.csv", banner); + Assert.DoesNotContain("Nothing was transferred", banner); + } + + [Fact] + public async Task Abandon_of_an_already_transferred_run_refuses_without_re_abandoning() + { + _transfer.AbandonAsync(RunId, Arg.Any()).Returns(new EgressAbandonResult.AlreadyTransferred()); + var controller = Build(); + + var redirect = Assert.IsType(await controller.Abandon(RunId, CancellationToken.None)); + + Assert.Equal(nameof(EgressController.Index), redirect.ActionName); + var banner = Assert.IsType(controller.TempData[EgressController.BannerKey]); + Assert.Contains("already been transferred", banner); + } + + [Fact] + public async Task Abandon_of_an_unknown_run_returns_not_found() + { + _transfer.AbandonAsync(RunId, Arg.Any()).Returns(new EgressAbandonResult.NotFound()); + + Assert.IsType(await Build().Abandon(RunId, CancellationToken.None)); + } +} diff --git a/tests/DfE.CheckPerformanceData.UnitTests/Web/Views/EgressViewSourceTests.cs b/tests/DfE.CheckPerformanceData.UnitTests/Web/Views/EgressViewSourceTests.cs new file mode 100644 index 000000000..b2f9f8eb8 --- /dev/null +++ b/tests/DfE.CheckPerformanceData.UnitTests/Web/Views/EgressViewSourceTests.cs @@ -0,0 +1,271 @@ +using System.Runtime.CompilerServices; + +namespace DfE.CheckPerformanceData.Application.UnitTests.Web.Views; + +// Pins the contracts the E2E tier cannot guard on the always-on gate (AB#294553): every screen +// works without JavaScript, the progress page is an enhancement over a real form, tabs are one +// per output type, and the target container is stated but never editable. +public sealed class EgressViewSourceTests +{ + private static string View(string name) => File.ReadAllText(Path.Combine( + RepoRoot, "src", "DfE.CheckPerformanceData.Web", "Views", "Egress", name)); + + private static string Script() => File.ReadAllText(Path.Combine( + RepoRoot, "src", "DfE.CheckPerformanceData.Web", "wwwroot", "js", "egress-preprocess.js")); + + [Fact] + public void The_egress_views_render_under_the_admin_layout() + => Assert.Contains("Layout = \"_AdminLayout\";", View("_ViewStart.cshtml")); + + [Fact] + public void Pull_page_is_a_plain_form_with_a_window_select_and_output_type_checkboxes() + { + var view = View("Index.cshtml"); + Assert.Contains("
", view); + Assert.Contains("Pull data from Zendesk", view); + Assert.Contains("data-testid=\"egress-saved-runs\"", view); + Assert.Contains("data-testid=\"egress-completed-runs\"", view); + } + + [Fact] + public void Pull_page_lists_refusals_in_an_error_summary() + { + var view = View("Index.cshtml"); + Assert.Contains("@foreach (var refusal in Model.Refusals)", view); + Assert.Contains("govuk-error-summary", view); + } + + // Nit: every banner this page shows is neutral (abandoned, already transferred, refused) — + // role="alert" is reserved for a success banner GOV.UK doesn't have here. + [Fact] + public void Pull_page_banner_is_a_neutral_region_not_an_alert() + { + var view = View("Index.cshtml"); + Assert.Contains("role=\"region\"", view); + Assert.DoesNotContain("role=\"alert\"", view); + } + + // Nit: WindowId's aria-describedby must not carry a trailing space when there is no error. + // Review finding (18 Sep): warn about a missing egress storage account on the first screen + // and on the confirm screen, not only in the failure banner after Confirm. + [Fact] + public void Pull_page_warns_when_egress_storage_is_not_configured() + { + var view = View("Index.cshtml"); + Assert.Contains("@if (Model.StorageNotConfigured)", view); + Assert.Contains("data-testid=\"egress-storage-not-configured\"", view); + Assert.Contains("ConnectionStrings:EgressStorage", view); + } + + [Fact] + public void Summary_warns_when_egress_storage_is_not_configured() + { + var view = View("Summary.cshtml"); + Assert.Contains("@if (Model.StorageNotConfigured)", view); + Assert.Contains("data-testid=\"egress-storage-not-configured\"", view); + } + + [Fact] + public void Window_select_aria_describedby_has_no_trailing_space_without_an_error() + { + var view = View("Index.cshtml"); + Assert.Contains("aria-describedby=\"WindowId-hint@(Model.WindowError is not null ? \" WindowId-error\" : \"\")\"", view); + } + + // Nit: failed stages must get the red tag; every other stage keeps blue. + [Fact] + public void Saved_runs_give_failed_stages_the_red_tag() + { + var view = View("Index.cshtml"); + Assert.Contains("", view); + Assert.Contains("govuk-tag--red", view); + Assert.Contains("EgressRunStatus.PreprocessingFailed or EgressRunStatus.TransferFailed => \"govuk-tag--red\"", view); + } + + // S5: the output-types error must sit inside the form group between hint and checkboxes, be + // referenced by the fieldset's aria-describedby, and the group must carry the error class — + // fallout from the Task 12 duplicate-summary workaround having moved the error outside entirely. + [Fact] + public void Pull_page_associates_the_output_types_error_with_its_fieldset() + { + var view = View("Index.cshtml"); + Assert.Contains("govuk-form-group @(Model.OutputTypesError is not null ? \"govuk-form-group--error\" : \"\")", view); + // The library appends its own auto-generated OutputTypes-hint id to whatever value is + // supplied here (verified live), so only the error id needs adding. + Assert.Contains("", view); + var hintIndex = view.IndexOf("", StringComparison.Ordinal); + var errorIndex = view.IndexOf("id=\"OutputTypes-error\"", StringComparison.Ordinal); + var itemsIndex = view.IndexOf("= 0 && errorIndex > hintIndex && itemsIndex > errorIndex, + "the error message must render between the hint and the checkbox items"); + Assert.Contains("", view); + } + + [Fact] + public void Results_page_has_one_tab_per_output_type_and_offers_save_and_abandon() + { + var view = View("Results.cshtml"); + Assert.Contains("", view); + Assert.Contains("@foreach (var output in Model.Run.Outputs)", view); + Assert.Contains("id=\"@output.OutputType.ToString().ToLowerInvariant()\"", view); + Assert.Contains("Proceed to preprocessing", view); + Assert.Contains("Save and exit", view); + Assert.Contains("Abandon run", view); + Assert.Contains("RunPageViewModel.RawColumns", view); + } + + // Nit: rewritten from developer-note phrasing ("discarded by the preprocessing filter, not + // here") into plain user-facing language. + [Fact] + public void Results_page_explains_filtering_in_plain_language() + { + var view = View("Results.cshtml"); + Assert.DoesNotContain("discarded by the preprocessing filter", view); + Assert.Contains("Only approved and auto-approved requests will be included", view); + } + + // Nit: every page using _RunHeader has an

, so its caption must + // be govuk-caption-xl, matched to that heading size. + [Fact] + public void RunHeader_caption_size_matches_the_pages_heading() + { + var view = View("_RunHeader.cshtml"); + Assert.Contains("govuk-caption-xl", view); + Assert.DoesNotContain("govuk-caption-l\"", view); + } + + // M2: a PreprocessingFailed run released its pair — Failed.cshtml's own copy says "start a new + // run" — so Results must not offer a path back into preprocessing for it. + [Fact] + public void Results_page_hides_proceed_to_preprocessing_for_a_preprocessing_failed_run() + { + var view = View("Results.cshtml"); + Assert.Contains("Model.Run.Status != EgressRunStatus.PreprocessingFailed", view); + } + + [Fact] + public void Preprocessing_page_is_a_real_form_the_script_enhances() + { + var view = View("Preprocessing.cshtml"); + Assert.Contains("data-module=\"egress-preprocess\"", view); + Assert.Contains("data-stream-url=\"@Model.StreamUrl\"", view); + Assert.Contains("", progressbarOpen, StringComparison.Ordinal); + var liveRegion = view.IndexOf("aria-live=\"polite\"", StringComparison.Ordinal); + Assert.True(liveRegion < progressbarOpen || liveRegion > progressbarClose, + "the aria-live region must not be nested inside the role=\"progressbar\" element"); + } + + // M4: the lock has no expiry, so a run stuck in Preprocessing after a restart must be + // releasable from this page, not only from Results/Summary. + [Fact] + public void Preprocessing_page_offers_abandon() + { + var view = View("Preprocessing.cshtml"); + Assert.Contains(" (o.OutputRecordCount ?? 0) == 0)", view); + Assert.Contains("data-testid=\"egress-nothing-to-transfer\"", view); + Assert.Contains("@if (offerConfirm)", view); + } + + // Nit: a Transferring run already has a transfer in flight; Summary must not offer Confirm + // for it either. + [Fact] + public void Summary_does_not_offer_confirm_while_a_transfer_is_already_in_flight() + { + var view = View("Summary.cshtml"); + Assert.Contains("Model.Run.Status != EgressRunStatus.Transferring", view); + } + + [Fact] + public void Failed_page_lists_every_failure_with_step_field_and_reason() + { + var view = View("Failed.cshtml"); + Assert.Contains("@foreach (var failure in Model.Run.Failures)", view); + Assert.Contains("@failure.Step", view); + Assert.Contains("@failure.Field", view); + Assert.Contains("@failure.Reason", view); + Assert.Contains("No records were transferred", view); + } + + [Fact] + public void Complete_page_shows_the_panel_and_the_transfer_summary() + { + var view = View("Complete.cshtml"); + Assert.Contains("govuk-panel--confirmation", view); + Assert.Contains("Transfer complete", view); + Assert.Contains("@Model.Run.TransferredByName", view); + Assert.Contains("@Model.Run.TransferredAtUtc", view); + Assert.Contains("Start a new egress", view); + } + + [Fact] + public void Every_egress_view_renders_exactly_one_h1() + { + foreach (var name in new[] { "Index.cshtml", "Results.cshtml", "Preprocessing.cshtml", "Failed.cshtml", "Summary.cshtml", "Preview.cshtml", "Complete.cshtml" }) + { + var view = View(name); + // Complete.cshtml's single heading is

, so counting "

Path.GetFullPath(Path.Combine( + Path.GetDirectoryName(ThisFilePath())!, "..", "..", "..", "..")); + + private static string ThisFilePath([CallerFilePath] string path = "") => path; +}