diff --git a/config/default/core.entity_form_display.paragraph.raw_data_component_upload.default.yml b/config/default/core.entity_form_display.paragraph.raw_data_component_upload.default.yml
index 19ccbefd5..f4383a6a4 100644
--- a/config/default/core.entity_form_display.paragraph.raw_data_component_upload.default.yml
+++ b/config/default/core.entity_form_display.paragraph.raw_data_component_upload.default.yml
@@ -5,6 +5,7 @@ dependencies:
config:
- field.field.paragraph.raw_data_component_upload.field_agency_component
- field.field.paragraph.raw_data_component_upload.field_request_data_csv
+ - field.field.paragraph.raw_data_component_upload.section_ix_xi_data
- paragraphs.paragraphs_type.raw_data_component_upload
module:
- file
@@ -30,6 +31,13 @@ content:
settings:
progress_indicator: throbber
third_party_settings: { }
+ section_ix_xi_data:
+ type: file_generic
+ weight: 2
+ region: content
+ settings:
+ progress_indicator: throbber
+ third_party_settings: { }
hidden:
created: true
status: true
diff --git a/config/default/core.entity_view_display.paragraph.raw_data_component_upload.default.yml b/config/default/core.entity_view_display.paragraph.raw_data_component_upload.default.yml
index 8f9da98b3..8ab1464d7 100644
--- a/config/default/core.entity_view_display.paragraph.raw_data_component_upload.default.yml
+++ b/config/default/core.entity_view_display.paragraph.raw_data_component_upload.default.yml
@@ -5,6 +5,7 @@ dependencies:
config:
- field.field.paragraph.raw_data_component_upload.field_agency_component
- field.field.paragraph.raw_data_component_upload.field_request_data_csv
+ - field.field.paragraph.raw_data_component_upload.section_ix_xi_data
- paragraphs.paragraphs_type.raw_data_component_upload
module:
- file
@@ -29,5 +30,13 @@ content:
third_party_settings: { }
weight: 1
region: content
+ section_ix_xi_data:
+ type: file_default
+ label: above
+ settings:
+ use_description_as_link_text: false
+ third_party_settings: { }
+ weight: 2
+ region: content
hidden:
search_api_excerpt: true
diff --git a/config/default/field.field.paragraph.raw_data_component_upload.field_request_data_csv.yml b/config/default/field.field.paragraph.raw_data_component_upload.field_request_data_csv.yml
index daabf956b..d2497f59e 100644
--- a/config/default/field.field.paragraph.raw_data_component_upload.field_request_data_csv.yml
+++ b/config/default/field.field.paragraph.raw_data_component_upload.field_request_data_csv.yml
@@ -12,7 +12,7 @@ field_name: field_request_data_csv
entity_type: paragraph
bundle: raw_data_component_upload
label: 'CSV file'
-description: ''
+description: 'Component Raw Data Report'
required: true
translatable: false
default_value: { }
diff --git a/config/default/field.field.paragraph.raw_data_component_upload.section_ix_xi_data.yml b/config/default/field.field.paragraph.raw_data_component_upload.section_ix_xi_data.yml
new file mode 100644
index 000000000..a6e205f8c
--- /dev/null
+++ b/config/default/field.field.paragraph.raw_data_component_upload.section_ix_xi_data.yml
@@ -0,0 +1,27 @@
+uuid: 36d4670e-76ee-47d2-bf44-c6cf284376f0
+langcode: en
+status: true
+dependencies:
+ config:
+ - field.storage.paragraph.section_ix_xi_data
+ - paragraphs.paragraphs_type.raw_data_component_upload
+ module:
+ - file
+id: paragraph.raw_data_component_upload.section_ix_xi_data
+field_name: section_ix_xi_data
+entity_type: paragraph
+bundle: raw_data_component_upload
+label: 'Section IX-XI CSV file'
+description: 'Section IX-XI Data'
+required: true
+translatable: false
+default_value: { }
+default_value_callback: ''
+settings:
+ handler: 'default:file'
+ handler_settings: { }
+ file_directory: 'request_data_tool/components/[date:custom:Y]-[date:custom:m]'
+ file_extensions: csv
+ max_filesize: ''
+ description_field: false
+field_type: file
diff --git a/config/default/field.storage.paragraph.section_ix_xi_data.yml b/config/default/field.storage.paragraph.section_ix_xi_data.yml
new file mode 100644
index 000000000..3c414b3b9
--- /dev/null
+++ b/config/default/field.storage.paragraph.section_ix_xi_data.yml
@@ -0,0 +1,23 @@
+uuid: 496fcb4b-f83e-46b1-baae-3a22a3dedee7
+langcode: en
+status: true
+dependencies:
+ module:
+ - file
+ - paragraphs
+id: paragraph.section_ix_xi_data
+field_name: section_ix_xi_data
+entity_type: paragraph
+type: file
+settings:
+ target_type: file
+ display_field: false
+ display_default: false
+ uri_scheme: private
+module: file
+locked: false
+cardinality: 1
+translatable: true
+indexes: { }
+persist_with_no_fields: false
+custom_storage: false
diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index ee25a933e..3904090cf 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -93,8 +93,14 @@ data records are checked. Blank records are skipped; quoted commas, escaped
quotes, and multiline values are supported. Record numbers include the header
and blank records and are not physical line numbers for multiline CSVs.
-The first validation error produces a human-readable message identifying the
-CSV record. Column-count errors include the expected and actual counts. Missing, unreadable, empty, and non-CSV uploads also
+Validation scans the entire CSV and collects all applicable errors, including
+multiple errors on the same record. Each message identifies its CSV record.
+Duplicate messages within a record are listed once. Column-count errors include
+the expected and actual counts; other checks on that malformed record are
+skipped, but later records are still checked. Invalid dates are reported without
+running dependent date calculations. CSV rows are streamed; accumulated error
+messages and seen request numbers consume memory in proportion to their count.
+Missing, unreadable, empty, and non-CSV uploads also
produce messages. Validation failures finish the queue item without generating
XML or changing any existing XML attachment. Correct the CSV and click Generate
XML Report again to retry. Each upload result identifies its component and filename,
@@ -228,6 +234,7 @@ field. `field_component_uploads` is an unlimited Paragraphs reference to
- Required `field_agency_component`, reusing the existing paragraph field storage.
- Required, single-value `field_request_data_csv`, accepting CSV files only and
storing them privately.
+- Required, single-value `section_ix_xi_data`, also accepting private CSV files.
A subset of the agency's components is allowed. At least one upload is needed
for processing; reports can otherwise be saved without uploads. Duplicate
@@ -244,7 +251,8 @@ changed.
Import the exported configuration and rebuild caches before running the queue.
The old node-level CSV field is replaced, with no migration of existing uploads.
This follows the pre-launch assumption that existing upload content is disposable.
-The queue payload remains the parent node ID. Edits during processing are not
+The queue payload includes the parent node ID, requester UID, and a unique
+notification ID. Edits during processing are not
locked or snapshotted. CSV conversion produces one partial annual report XML document
only after every component upload passes validation.
@@ -291,9 +299,10 @@ file is replaced. The existing `foia_export_xml` module is unchanged.
## Processed request statistics
`RequestStatisticsAggregator` makes a separate streaming pass after validation,
-retaining only four counters per component. Every data row counts, including
-consultations and rows without statute codes. Headers and blank records are
-skipped using the same rules as validation.
+retaining only four counters per component. Only rows with a nonblank Column I
+count, including consultations and rows without statute codes. Appeal-only rows
+and other rows without Column I are excluded from all counters. Headers and
+blank records are skipped using the same rules as validation.
- Pending at start: I is before October 1 of the previous year.
- Received: I is within the fiscal year, including both boundaries.
@@ -478,8 +487,8 @@ Date Received are excluded; a closed date without a received date is rejected.
The start is the later of Appeal Date Received and October 1 of the previous
year. The end is Appeal Date Closed, or September 30 of the report year when
Y is blank.
-Elapsed calendar days exclude the starting day and include the ending day;
-same-day completion is zero. Weekends and holidays count.
+Elapsed working days exclude the starting day and include the ending day;
+same-day completion is zero. Weekends and federal holidays do not count.
Day-frequency maps keep memory independent of the number of rows. They yield
the exact median (averaging the middle pair for even counts), average rounded
@@ -497,7 +506,7 @@ blank Column Y. It retains only the ten earliest received dates per component
and the ten earliest across the agency. Repeated dates remain separate items;
fewer than ten pending appeals produce only the items available.
-Pending days are elapsed calendar days from the actual received date through
+Pending days are elapsed working days from the actual received date through
September 30 of the report year, without clamping to the fiscal-year start.
Same-day receipt is zero days. XML receipt dates use `YYYY-MM-DD`.
`OldestPendingAppealSection` follows the appeal response times, with `OPA1`,
@@ -596,7 +605,7 @@ by descending pending days, then ascending receipt date for ties. Duplicate
rows remain separate entries. Empty components retain their empty container
and association. `OldestPendingRequestSection` follows pending perfected
requests, with ISO receipt dates and `OPR1`, `OPR2`, etc.; `OPR0` is the agency.
-The existing oldest-appeal section continues using calendar days.
+The oldest-appeal section also uses the shared working-day calendar.
## Expedited processing
@@ -624,3 +633,217 @@ For appeal rows (Column X populated), CSV validation requires E–P and T–W
to be blank. Q, R, and S may contain expedited-processing data and retain
their date and dependency checks. Appeal rows are exempt from the rule that
S = G requires M = E, since M must remain blank on these rows.
+
+## Personnel and cost
+
+`PersonnelAndCostSection` follows fee waivers. Every uploaded component and
+the agency have the six personnel and cost values sourced from Section IX-XI
+Columns A-D, with calculated staff and cost totals. `PC1`, `PC2`, etc. and
+agency `PC0` link to Organization entries through
+`PersonnelAndCostOrganizationAssociation`.
+
+## Fees collected
+
+`FeesCollectedAggregator` uses Section IX-XI Column E for each component's
+fees, with agency fees summed across components. `FeesCollectedCostPercent`
+is fees divided by the matching `TotalCostAmount` from personnel and cost,
+rounded half-up to four decimal places without multiplying by 100. The agency
+ratio uses total agency fees divided by total agency costs. A zero total cost
+produces `0.0000`, even when fees are nonzero. Fee amounts are also formatted
+to four decimal places, including `0.0000` for zero. Exact decimal arithmetic
+avoids floating-point rounding. Column W in the raw request CSV no longer
+supplies this section. Existing FC/ORG links and zero-count entries remain.
+
+## Subsection use
+
+`SubsectionUsedSection` uses Column F (Subsection (c) Exclusions) from each
+component's Section IX-XI CSV for `TimesUsedQuantity`. Agency quantities sum
+component quantities with exact integer arithmetic. Zero values remain present.
+`SU1`, `SU2`, etc. and agency `SU0` retain the existing Organization links through
+`SubsectionUsedOrganizationAssociation`. These values are collected in the same
+pass as personnel, cost, and fee totals.
+
+## Subsection posting
+
+`SubsectionPostSection` uses Section IX-XI Column G for `PostedbyFOIAQuantity`
+and Column H for `PostedbyProgramQuantity`. Agency quantities sum component
+quantities using exact integer arithmetic, including zeros. `SP1`, `SP2`, etc.
+and agency `SP0` retain Organization links through
+`SubsectionPostOrganizationAssociation`. These values are collected in the
+same pass as personnel, cost, fee, and exclusion totals.
+
+## Backlog
+
+`BacklogAggregator` counts rows whose request or appeal interval exceeds the
+working days allowed in Column D (exactly that many days does not qualify).
+Rows with blank Days Allowed are excluded from both counts. Requests use J, falling back
+to I, through K; appeals use X through Y. Open rows use September 30 of the
+report year. Actual start dates are retained, including prior fiscal years.
+No track or disposition filter applies. Rows with no start or end dates for
+an item are skipped; a closing date without a start produces an error.
+
+`BacklogSection` follows subsection posting, emits both counts including zeros,
+and associates `BK1`, `BK2`, etc. and agency `BK0` with Organization entries.
+Agency counts sum component counts. Appeal response-time summaries now also
+use working days, retaining their existing fiscal-start clamp and year-end
+fallback; backlog intervals use the full X-to-Y interval specified above.
+
+## Processed consultations
+
+`ConsultationStatisticsAggregator` streams rows with Column C equal to `Y`.
+Column I before October 1 counts as pending at start; October 1 through
+September 30 (inclusive) counts as received. Column K within those inclusive
+boundaries counts as processed; blank K or K after September 30 counts as
+pending at end. Blank I contributes to neither receipt-date counter.
+
+`ProcessedConsultationSection` emits all four counters, including zeros, for
+each component and the agency total. `PCN1`, `PCN2`, etc. and `PCN0` link to the
+Organization entries through `ProcessingStatisticsOrganizationAssociation`.
+Existing CSV validation rules remain in effect, including the restriction on
+completion dates outside the fiscal year.
+
+## Oldest pending consultations
+
+`OldestPendingRequestAggregator` also supports a consultation-only listing:
+Column C must equal `Y`, Column I must contain a receipt date, and Column K
+must be blank. Working days run from the actual receipt date through September
+30 of the report year, excluding the receipt day, weekends, and the shared
+federal holidays. Dates before the fiscal year are not clamped.
+
+`OldestPendingConsultationSection` follows processed consultations and uses
+`OPC1`, `OPC2`, etc. and agency `OPC0` with the usual Organization associations.
+Each listing retains at most ten items, ordered by pending working days
+(descending), then receipt date (ascending). Duplicate dates remain separate
+items. Empty components retain an empty `OldestPendingItems` entry; fewer than
+ten items are emitted when fewer qualify. Only ten items per component and
+ten agency-wide are retained in memory.
+
+## Processed request comparison
+
+`ProcessedRequestComparisonSection` reuses the existing request statistics:
+Column I within the fiscal year supplies `ItemsReceivedCurrentYearQuantity`,
+and Column K within the fiscal year supplies `ItemsProcessedCurrentYearQuantity`.
+October 1 and September 30 are included. Both last-year quantities are `0`.
+No additional CSV pass is needed.
+
+Each component and the agency have a `ProcessingComparison`, including zero
+counts, with `PRC1`, `PRC2`, etc. and `PRC0` linked to Organization entries via
+`ProcessingComparisonOrganizationAssociation`. Agency quantities are the sums
+of component quantities.
+
+## Backlogged request comparison
+
+`BackloggedRequestComparisonSection` reuses the request counts from
+`BacklogSection` for `BacklogCurrentYearQuantity`. `BacklogLastYearQuantity`
+is always `0`. No additional CSV pass is needed.
+
+Each component and the agency have a `BacklogComparison`, including zero
+counts. `BLR1`, `BLR2`, etc. and agency `BLR0` link to their Organization entries
+through `BacklogComparisonOrganizationAssociation`.
+
+## Processed appeal comparison
+
+`ProcessedAppealComparisonSection` reuses the received and processed counts
+from `ProcessedAppealSection`. Both last-year quantities are `0`. Each
+component and the agency have a `ProcessingComparison`, including zero counts,
+with `APC1`, `APC2`, etc. and `APC0` linked to Organization entries through
+`ProcessingComparisonOrganizationAssociation`. No additional CSV pass is
+needed; the request and appeal comparisons share the same XML formatter.
+
+## Backlogged appeal comparison
+
+`BackloggedAppealComparisonSection` reuses `BackloggedAppealQuantity` from
+`BacklogSection` for its current-year count and emits `0` for last year.
+Every component and the agency have a `BacklogComparison`, including zeros,
+with `ABC1`, `ABC2`, etc. and `ABC0` linked to Organization entries through
+`BacklogComparisonOrganizationAssociation`. Request and appeal backlog
+comparisons share the same XML formatter and need no additional CSV pass.
+
+### Appeal receipt date validation
+
+Every data row must have exactly one receipt date: Column I for an initial
+request or Column X for an appeal. If I is blank, X is required; both blank
+produces "Appeal Date Received cannot be blank". The existing Column X
+exclusion rule rejects rows containing both I and X.
+
+When populated, X must be a real MM/DD/YYYY date (single-digit months/days are
+accepted), no later than September 30 of the report year. Earlier fiscal years
+are allowed. Errors identify Column X and the CSV record; the queue adds the
+filename and component when saving messages.
+
+Column Y (Appeal Date Closed) may be blank when Z (Appeal Disposition) is blank.
+When populated, it must be a valid MM/DD/YYYY date within the inclusive fiscal
+year and cannot precede a valid X date. Y requires Z, and Z requires Y; these
+presence checks are independent of date validity. All failures identify Column
+Y and the record and are collected with other validation messages.
+
+Column Z dispositions `Affirmed`, `Affirmed on Appeal`, and
+`Partially Affirmed & Partially Reversed/Remanded` require a nonblank AA or AC
+(or both). `Closed for Other Reasons` requires nonblank AA even when AC has
+data. Surrounding whitespace is ignored. These presence checks do not change
+the disposition labels recognized by XML aggregation.
+
+Column AB requires nonblank explanatory text when AA contains `Other` as a
+complete comma-separated reason. Whitespace around entries is ignored; a
+longer reason such as `Improper Request for Other Reasons` does not trigger
+this rule. Failures identify Column AB and the CSV record.
+
+Nonblank Column AC requires Column Z to be exactly `Affirmed on Appeal` or
+`Partially Affirmed & Partially Reversed/Remanded`, ignoring surrounding
+whitespace. The shorter `Affirmed` label does not satisfy this rule. Blank AC
+adds no disposition requirement. Errors identify Column AC and the record.
+
+## Completion notifications
+
+New queue items record the user who clicks Generate XML Report and a unique
+notification ID. Successful XML generation, validation failure, and XML
+processing exceptions store a personal notification. On the requester's next
+normal HTML GET page visit, it becomes a Drupal status/error message linking
+to the report. No email is sent and no polling is performed. Rebuild Drupal's
+cache after deploying these service definitions (`ddev drush cr` locally).
+
+Delivery checks current node view access and suppresses deleted/inaccessible
+reports. AJAX and non-HTML requests do not consume notices. Notices and delivery
+receipts expire after 30 days; receipts suppress repeated failure notices for
+retries of the same queue item. A successful retry supersedes an unread failure.
+Older queue items without requester metadata still process normally but cannot
+notify a user. Notification storage failures are logged without altering XML
+processing or queue retry behavior.
+
+## Section IX-XI upload
+
+Each component paragraph now requires two single-file private CSV uploads:
+`field_request_data_csv` (help: Component Raw Data Report) and
+`section_ix_xi_data` (help: Section IX-XI Data). The second upload follows the
+same private download access rules, CSV extension restriction, and upload size
+settings as the first. It appears after the original upload on edit and view
+displays. Its presence and contents are validated; Columns A-D populate personnel
+and cost XML totals. Existing paragraphs need the second file when edited.
+Import the field and display configuration before running the updated code.
+
+## Section IX-XI CSV validation
+
+The first nonblank record must exactly match the eight header names and order
+from `OIP FY23 Section IX-XI Datav final.csv`, stored in
+`SectionDataCsvValidator::HEADERS`. Exactly one nonblank data row must follow.
+All eight cells are required: B accepts decimal notation (including whole
+numbers), and the other seven require integers. Optional signs and surrounding
+numeric whitespace are accepted; scientific notation, separators, currency,
+and blank cells are rejected. Headers are not trimmed. UTF-8 BOMs and blank
+lines are ignored, and record numbers include blank lines.
+
+The queue checks both CSVs for every component, collects all errors with file,
+component, record, and column context, and retains the existing XML if either
+file fails. Successful files receive `CSV validated.` messages. Section IX-XI
+Columns A-D populate the personnel and cost section.
+
+## Personnel and cost from Section IX-XI data
+
+Queue processing now supplies `PersonnelAndCostSection` with Column A full-time
+employees, Column B equivalent full-time employees, Column C processing costs,
+and Column D litigation costs. Total staff is A+B and total cost is C+D.
+Agency quantities sum the corresponding component values. Existing PC/ORG
+references and zero values are preserved. Exact decimal arithmetic uses the
+existing Brick Math dependency, avoiding floating-point rounding or integer
+overflow; output does not impose an additional decimal precision limit.
+Section IX-XI columns E-H supply fees and subsection statistics.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.module b/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.module
index 668d5db60..6f6736d96 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.module
+++ b/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.module
@@ -18,6 +18,7 @@ use Drupal\node\NodeInterface;
use Drupal\file\FileInterface;
use Drupal\file\FileReferenceResolver;
use Drupal\foia_raw_data_to_report\Form\GenerateXmlReportForm;
+use Drupal\foia_raw_data_to_report\Form\ComponentPlaceholders;
/**
* Implements hook_node_view().
@@ -61,11 +62,12 @@ function foia_raw_data_to_report_file_access(FileInterface $file, $operation, Ac
$access = AccessResult::neutral()->cachePerUser()->addCacheableDependency($file);
$is_report_file = FALSE;
$report_fields = ['field_request_data_csv', 'field_request_data_xml'];
+ $upload_fields = ['field_request_data_csv', 'section_ix_xi_data'];
foreach ($resolver->getReferences($file) as $usage) {
$entity = $resolver->loadEntityFromUsage($usage);
$node = $entity;
$parent_field = $usage->fieldName;
- if ($entity instanceof ParagraphInterface && $entity->bundle() === 'raw_data_component_upload' && $usage->fieldName === 'field_request_data_csv') {
+ if ($entity instanceof ParagraphInterface && $entity->bundle() === 'raw_data_component_upload' && in_array($usage->fieldName, $upload_fields, TRUE)) {
$is_report_file = TRUE;
$access->addCacheableDependency($entity);
$node = $entity->getParentEntity();
@@ -168,10 +170,11 @@ function foia_raw_data_to_report_form_node_form_alter(&$form, FormStateInterface
}
$form['field_component_uploads']['#prefix'] = '';
$form['field_component_uploads']['#suffix'] = '
';
+ ComponentPlaceholders::build($form, $form_state);
$form['field_agency']['widget'][0]['target_id']['#ajax'] = [
'callback' => 'foia_raw_data_to_report_refresh_components',
'wrapper' => 'raw-data-component-uploads',
- 'event' => 'change',
+ 'event' => 'autocompleteclose change',
];
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.services.yml b/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.services.yml
index a7a2b9e48..5725026b1 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.services.yml
+++ b/docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.services.yml
@@ -1,3 +1,16 @@
services:
foia_raw_data_to_report.csv_validator:
class: Drupal\foia_raw_data_to_report\CsvValidator
+ logger.channel.foia_raw_data_to_report:
+ parent: logger.channel_base
+ arguments: ['foia_raw_data_to_report']
+ foia_raw_data_to_report.notifications:
+ class: Drupal\foia_raw_data_to_report\ReportNotifications
+ arguments: ['@keyvalue.expirable', '@entity_type.manager', '@current_user', '@messenger', '@lock', '@logger.channel.foia_raw_data_to_report']
+ foia_raw_data_to_report.notification_subscriber:
+ class: Drupal\foia_raw_data_to_report\EventSubscriber\ReportNotificationSubscriber
+ arguments: ['@foia_raw_data_to_report.notifications']
+ tags:
+ - { name: event_subscriber }
+ foia_raw_data_to_report.section_csv_validator:
+ class: Drupal\foia_raw_data_to_report\SectionDataCsvValidator
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php
index b55259a92..2a1a26f94 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php
@@ -21,7 +21,8 @@ final class AppealDispositionAggregator {
* Counts each nonblank disposition once per row, then sums component counts.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
*
* @return array
* Component counts keyed by entity ID and overall counts, keyed by outcome.
@@ -57,7 +58,7 @@ public function aggregate(array $sources): array {
}
// Reject unmapped values rather than silently omitting requests.
if (!isset(self::DISPOSITIONS[$code])) {
- throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $id, $record));
+ throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $source['component_label'] ?? $id, $record));
}
$components[$id][self::DISPOSITIONS[$code]]++;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php
index 5dbbd64a7..a2b1cb45e 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php
@@ -28,7 +28,8 @@ final class AppealNonExemptionDenialAggregator {
* Counts each distinct reason once per row, then sums component counts.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
*
* @return array
* Component and overall counts keyed by XML reason code.
@@ -67,7 +68,7 @@ public function aggregate(array $sources): array {
foreach ($reasons as $reason) {
// Unknown labels must not silently disappear from the report.
if (!isset(self::REASONS[$reason])) {
- throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $id, $record, $reason));
+ throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $source['component_label'] ?? $id, $record, $reason));
}
$components[$id][self::REASONS[$reason]]++;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php
index 4c042625d..732c640cb 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php
@@ -3,7 +3,7 @@
namespace Drupal\foia_raw_data_to_report;
/**
- * Streams Columns X and Y into calendar-day response time distributions.
+ * Streams Columns X and Y into working-day response time distributions.
*/
final class AppealResponseTimeAggregator {
@@ -11,7 +11,8 @@ final class AppealResponseTimeAggregator {
* Calculates component and agency statistics without retaining CSV rows.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year.
*
@@ -22,6 +23,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
$timezone = new \DateTimeZone('UTC');
$start = new \DateTimeImmutable(($fiscal_year - 1) . '-10-01', $timezone);
$end = new \DateTimeImmutable($fiscal_year . '-09-30', $timezone);
+ $working_days = new WorkingDays();
$histograms = [];
foreach ($sources as $source) {
$id = $source['component_id'];
@@ -45,7 +47,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
$header = FALSE;
continue;
}
- $context = sprintf('Component %s, CSV record %d', $id, $record);
+ $context = sprintf('Component %s, CSV record %d', $source['component_label'] ?? $id, $record);
// Only rows with an appeal contribute to response time statistics.
$received_text = trim($columns[23]);
$completed = trim($columns[24]);
@@ -61,9 +63,9 @@ public function aggregate(array $sources, int $fiscal_year): array {
if ($finish < $begin || $finish > $end) {
throw new \RuntimeException($context . ': Response time dates must be in chronological order and within the report fiscal year.');
}
- // Elapsed calendar days exclude the starting day; same-day is zero.
+ // Working days exclude the starting day; same-day is zero.
// UTC dates avoid daylight-saving changes affecting the calculation.
- $days = (int) $begin->diff($finish)->days;
+ $days = $working_days->count($begin->format('m/d/Y'), $finish->format('m/d/Y'));
$histograms[$id][$days] = ($histograms[$id][$days] ?? 0) + 1;
}
if (!feof($stream)) {
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php
index 0ffe9488b..8dc87f492 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php
@@ -21,7 +21,8 @@ final class AppealStatisticsAggregator {
* Returns component counters and agency totals without retaining CSV rows.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year.
*
@@ -56,7 +57,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
// Rows without an appeal do not contribute to any of the counters.
$received_text = trim($columns[23]);
$completed = trim($columns[24]);
- $context = sprintf('Component %s, CSV record %d', $source['component_id'], $record);
+ $context = sprintf('Component %s, CSV record %d', $source['component_label'] ?? $source['component_id'], $record);
if ($received_text === '') {
if ($completed !== '') {
throw new \RuntimeException($context . ': Appeal Date Closed requires an Appeal Date Received.');
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
index 3d8c2ed21..a4f3fdbec 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
@@ -31,7 +31,8 @@ final class AppliedExemptionsAggregator {
* Counts each distinct exemption once per row, then sums component counts.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $column
* Zero-based CSV column: 15 for requests, 28 for appeals.
*
@@ -77,7 +78,7 @@ public function aggregate(array $sources, int $column = 15): array {
foreach ($codes as $code) {
// Unknown codes must not silently disappear from the report.
if (!isset(self::EXEMPTIONS[$code])) {
- throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column %s.', $id, $record, $code, $column_name));
+ throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column %s.', $source['component_label'] ?? $id, $record, $code, $column_name));
}
$components[$id][$code]++;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/BacklogAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/BacklogAggregator.php
new file mode 100644
index 000000000..4da1faa82
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/BacklogAggregator.php
@@ -0,0 +1,114 @@
+ 0, 'appeals' => 0];
+ $year_end = sprintf('09/30/%04d', $fiscal_year);
+ $working_days = new WorkingDays();
+ $components = [];
+ foreach ($sources as $source) {
+ $id = $source['component_id'];
+ $components[$id] ??= $empty;
+ $stream = @fopen($source['uri'], 'rb');
+ if ($stream === FALSE) {
+ throw new \RuntimeException('Unable to reopen a validated CSV for backlog counts.');
+ }
+ try {
+ $header = TRUE;
+ $record = 0;
+ while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+ $record++;
+ if ($columns === [NULL]) {
+ continue;
+ }
+ if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+ throw new \RuntimeException('CSV column count changed after validation.');
+ }
+ if ($header) {
+ $header = FALSE;
+ continue;
+ }
+ $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $id, basename($source['uri']), $record);
+ // Column D supplies the threshold for both requests and appeals.
+ // Consultations leave Days Allowed blank and are excluded.
+ $days_allowed = trim($columns[3]);
+ if ($days_allowed === '') {
+ continue;
+ }
+ if (!in_array($days_allowed, ['20', '30'], TRUE)) {
+ throw new \RuntimeException($context . ': Column D: Expected Days Allowed to be 20 or 30.');
+ }
+ // Request age starts at J, falling back to I, without track filters.
+ $perfected = trim($columns[9]);
+ $request_start = $perfected !== '' ? $perfected : trim($columns[8]);
+ $intervals = [
+ 'requests' => [$request_start, trim($columns[10]), 'J or I', 'K'],
+ 'appeals' => [trim($columns[23]), trim($columns[24]), 'X', 'Y'],
+ ];
+ foreach ($intervals as $kind => [$start, $end, $start_column, $end_column]) {
+ // Rows without a received/perfected date do not identify an item.
+ if ($start === '') {
+ if ($end !== '') {
+ throw new \RuntimeException($context . ': Column ' . $end_column . ' requires a start date in Column ' . $start_column . '.');
+ }
+ continue;
+ }
+ $start_date = $this->calendarDate($start, $context . ', Column ' . $start_column);
+ $end = $end === '' ? $year_end : $end;
+ $end_date = $this->calendarDate($end, $context . ', Column ' . $end_column);
+ if ($end_date < $start_date) {
+ throw new \RuntimeException($context . ': End date cannot precede start date for ' . $kind . '.');
+ }
+ // Use actual receipt dates, including time before the fiscal year.
+ if ($working_days->count($start, $end) > (int) $days_allowed) {
+ $components[$id][$kind]++;
+ }
+ }
+ }
+ if (!feof($stream)) {
+ throw new \RuntimeException('Unable to finish reading a CSV for backlog counts.');
+ }
+ }
+ finally {
+ fclose($stream);
+ }
+ }
+ $overall = $empty;
+ foreach ($components as $counts) {
+ foreach ($counts as $code => $quantity) {
+ $overall[$code] += $quantity;
+ }
+ }
+ return ['components' => $components, 'overall' => $overall];
+ }
+
+ /**
+ * Parses calendar dates defensively if a file changed after validation.
+ */
+ private function calendarDate(string $value, string $context): \DateTimeImmutable {
+ if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+ || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+ throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+ }
+ return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC'));
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ConsultationStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/ConsultationStatisticsAggregator.php
new file mode 100644
index 000000000..78b17a34e
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/ConsultationStatisticsAggregator.php
@@ -0,0 +1,123 @@
+ 0,
+ 'received' => 0,
+ 'processed' => 0,
+ 'pending_end' => 0,
+ ];
+
+ /**
+ * Returns component counters and agency totals without retaining CSV rows.
+ *
+ * @param array $sources
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
+ * @param int $fiscal_year
+ * The validated report year.
+ *
+ * @return array
+ * Component counts keyed by entity ID, plus overall totals.
+ */
+ public function aggregate(array $sources, int $fiscal_year): array {
+ $start = ($fiscal_year - 1) * 10000 + 1001;
+ $end = $fiscal_year * 10000 + 930;
+ $components = [];
+ foreach ($sources as $source) {
+ $counts = self::EMPTY_COUNTS;
+ $stream = @fopen($source['uri'], 'rb');
+ if ($stream === FALSE) {
+ throw new \RuntimeException('Unable to reopen a validated CSV for consultation statistics.');
+ }
+ try {
+ $header = TRUE;
+ $record = 0;
+ while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+ $record++;
+ if ($columns === [NULL]) {
+ continue;
+ }
+ if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+ throw new \RuntimeException('CSV column count changed after validation.');
+ }
+ if ($header) {
+ $header = FALSE;
+ continue;
+ }
+ // Only Column C = Y identifies a consultation.
+ if (trim($columns[2]) !== 'Y') {
+ continue;
+ }
+ $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $source['component_id'], basename($source['uri']), $record);
+ // Count receipt dates before or within the inclusive fiscal year.
+ $received = trim($columns[8]);
+ if ($received !== '') {
+ $received_date = $this->calendarDate($received, $context . ', Column I (Date Initially Received)');
+ if ($received_date < $start) {
+ $counts['pending_start']++;
+ }
+ elseif ($received_date <= $end) {
+ $counts['received']++;
+ }
+ }
+
+ // Consultations closed after year-end or still open are pending.
+ $completed = trim($columns[10]);
+ if ($completed === '') {
+ $counts['pending_end']++;
+ }
+ else {
+ $completed_date = $this->calendarDate($completed, $context . ', Column K (Date Completed)');
+ if ($completed_date >= $start && $completed_date <= $end) {
+ $counts['processed']++;
+ }
+ elseif ($completed_date > $end) {
+ $counts['pending_end']++;
+ }
+ }
+ }
+ if (!feof($stream)) {
+ throw new \RuntimeException('Unable to finish reading a CSV for consultation statistics.');
+ }
+ }
+ finally {
+ fclose($stream);
+ }
+ $id = $source['component_id'];
+ $components[$id] ??= self::EMPTY_COUNTS;
+ foreach ($counts as $key => $value) {
+ $components[$id][$key] += $value;
+ }
+ }
+
+ $overall = self::EMPTY_COUNTS;
+ foreach ($components as $counts) {
+ foreach ($counts as $key => $value) {
+ $overall[$key] += $value;
+ }
+ }
+ return ['components' => $components, 'overall' => $overall];
+ }
+
+ /**
+ * Converts a validated date to YYYYMMDD, guarding against changed CSV data.
+ */
+ private function calendarDate(string $value, string $context): int {
+ if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+ || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+ throw new \RuntimeException($context . ': CSV contains an invalid date after validation: ' . ($value === '' ? '[blank]' : $value) . '. Expected MM/DD/YYYY.');
+ }
+ return (int) $parts[3] * 10000 + (int) $parts[1] * 100 + (int) $parts[2];
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
index 7773388b5..612ca874a 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
@@ -12,6 +12,14 @@ final class CsvValidator {
*/
public const EXPECTED_COLUMNS = 29;
+ /**
+ * Allowed exemption codes for Column P (Disposition Exemption(s) Applied).
+ */
+ public const ALLOWED_DISPOSITION_EXEMPTIONS = [
+ '1', '2', '3', '4', '5', '6',
+ '7a', '7b', '7c', '7d', '7e', '7f', '8', '9',
+ ];
+
/**
* Returns human-readable errors, or an empty array when validation passes.
*
@@ -24,7 +32,7 @@ final class CsvValidator {
* The report node's Year; the fiscal year ends on September 30.
*
* @return string[]
- * Validation errors. Stops at the first error to keep output bounded.
+ * All validation errors, in record and validation-check order.
*/
public function validate(string $uri, int $fiscal_year): array {
// A missing or invalid report year must not bypass fiscal-year checks.
@@ -37,6 +45,7 @@ public function validate(string $uri, int $fiscal_year): array {
}
try {
+ $errors = [];
$record = 0;
$has_content = FALSE;
$request_numbers = [];
@@ -46,12 +55,15 @@ public function validate(string $uri, int $fiscal_year): array {
if ($columns === [NULL]) {
continue;
}
+ // Even a malformed first record is the header, not a data row.
+ $is_header = !$has_content;
+ $has_content = TRUE;
if (count($columns) !== self::EXPECTED_COLUMNS) {
- return [sprintf('CSV record %d has %d columns; expected %d columns to match the OIP request raw data template. Please correct the file and try again.', $record, count($columns), self::EXPECTED_COLUMNS)];
+ $errors[] = sprintf('CSV record %d has %d columns; expected %d columns to match the OIP request raw data template. Please correct the file and try again.', $record, count($columns), self::EXPECTED_COLUMNS);
+ // Column positions are unreliable; continue with the next record.
+ continue;
}
- // The first nonblank record is the header; only check its column count.
- if (!$has_content) {
- $has_content = TRUE;
+ if ($is_header) {
continue;
}
@@ -60,14 +72,15 @@ public function validate(string $uri, int $fiscal_year): array {
if ($appeal_received !== '') {
foreach (array_merge(array_slice($columns, 4, 12), array_slice($columns, 19, 4)) as $value) {
if (trim($value) !== '') {
- return [sprintf('CSV record %d: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.', $record)];
+ $errors[] = sprintf('CSV record %d: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.', $record);
+ break;
}
}
}
// Column A: Component is required, including for consultation rows.
if (trim($columns[0]) === '') {
- return [sprintf('CSV record %d: Column A: Component cannot be blank', $record)];
+ $errors[] = sprintf('CSV record %d: Column A: Component cannot be blank', $record);
}
// Column B: Request Number is required and must be unique in this CSV.
@@ -75,18 +88,20 @@ public function validate(string $uri, int $fiscal_year): array {
// keys (including leading zeroes). Ignore surrounding whitespace.
$request_number = trim($columns[1]);
if ($request_number === '') {
- return [sprintf('CSV record %d: Column B: Request Number cannot be blank', $record)];
+ $errors[] = sprintf('CSV record %d: Column B: Request Number cannot be blank', $record);
}
$request_key = 'request:' . $request_number;
- if (isset($request_numbers[$request_key])) {
- return [sprintf('CSV record %d: Column B: Request Number is duplicate', $record)];
+ if ($request_number !== '' && isset($request_numbers[$request_key])) {
+ $errors[] = sprintf('CSV record %d: Column B: Request Number is duplicate', $record);
+ }
+ if ($request_number !== '') {
+ $request_numbers[$request_key] = TRUE;
}
- $request_numbers[$request_key] = TRUE;
// Column C: Is This a Consultation must be an uppercase Y or N.
$consultation = trim($columns[2]);
if (!in_array($consultation, ['Y', 'N'], TRUE)) {
- return [sprintf("CSV record %d: Column C must have 'Y' or 'N' entered", $record)];
+ $errors[] = sprintf("CSV record %d: Column C must have 'Y' or 'N' entered", $record);
}
// Column C: Consultations may contain values only in A, B, C, I and K.
@@ -96,14 +111,14 @@ public function validate(string $uri, int $fiscal_year): array {
if (!in_array($index, [0, 1, 2, 8, 10], TRUE) && trim($value) !== '') {
// This 29-column template spans A through AC.
$column = $index < 26 ? chr(65 + $index) : 'A' . chr(65 + $index - 26);
- return [sprintf("CSV record %d: Column %s must be blank when Column C is 'Y'; only columns A, B, C, I, and K may contain values", $record, $column)];
+ $errors[] = sprintf("CSV record %d: Column %s must be blank when Column C is 'Y'; only columns A, B, C, I, and K may contain values", $record, $column);
}
}
}
// Column D: Non-consultation rows require Days Allowed to be 20 or 30.
if ($consultation !== 'Y' && !in_array(trim($columns[3]), ['20', '30'], TRUE)) {
- return [sprintf('CSV record %d: Column D: Must complete Days Allowed with 20 or 30', $record)];
+ $errors[] = sprintf('CSV record %d: Column D: Must complete Days Allowed with 20 or 30', $record);
}
// Column E: Exemption 3 Statutes is optional. The Column C check above
@@ -115,7 +130,7 @@ public function validate(string $uri, int $fiscal_year): array {
// Detect digit groups separated by non-comma characters; whitespace
// around comma-separated IDs is allowed, as with other column checks.
if (preg_match('/[0-9][^0-9,]+[0-9]/', $statutes)) {
- return [sprintf('CSV record %d: Column E: Separate multiple entries in Column E with a comma', $record)];
+ $errors[] = sprintf('CSV record %d: Column E: Separate multiple entries in Column E with a comma', $record);
}
// Column E: Every comma-separated entry must be an integer ID 1-77.
@@ -124,7 +139,8 @@ public function validate(string $uri, int $fiscal_year): array {
foreach (explode(',', $statutes) as $statute) {
$statute = trim($statute);
if (!ctype_digit($statute) || (int) $statute < 1 || (int) $statute > 77) {
- return [sprintf('CSV record %d: Entries in Column E must be a valid ID between 1 - 77', $record)];
+ $errors[] = sprintf('CSV record %d: Entries in Column E must be a valid ID between 1 - 77', $record);
+ continue;
}
$statute_ids[] = (int) $statute;
}
@@ -134,10 +150,10 @@ public function validate(string $uri, int $fiscal_year): array {
$has_other_statute = in_array(77, $statute_ids, TRUE);
foreach ([5, 6, 7] as $index) {
if ($has_other_statute && trim($columns[$index]) === '') {
- return [sprintf('CSV record %d: Column E: If Ex. 3 Code 77 is entered, columns F, G, and H must contain information', $record)];
+ $errors[] = sprintf('CSV record %d: Column E: If Ex. 3 Code 77 is entered, columns F, G, and H must contain information', $record);
}
if (!$has_other_statute && trim($columns[$index]) !== '') {
- return [sprintf('CSV record %d: Column E: If Ex. 3 Codes 1 - 76 is entered in Column E, columns F, G, and H must remain blank', $record)];
+ $errors[] = sprintf('CSV record %d: Column E: If Ex. 3 Codes 1 - 76 is entered in Column E, columns F, G, and H must remain blank', $record);
}
}
@@ -145,96 +161,92 @@ public function validate(string $uri, int $fiscal_year): array {
// Match a complete comma-separated exemption, not a substring of 13.
$exemptions = array_map('trim', explode(',', $columns[15]));
if (!in_array('3', $exemptions, TRUE)) {
- return [sprintf("CSV record %d: Column E: If Column E contains information, a '3' must be listed in Column P", $record)];
+ $errors[] = sprintf("CSV record %d: Column E: If Column E contains information, a '3' must be listed in Column P", $record);
}
}
// Column F: Other Exemption 3 Statutes requires code 77 in Column E.
if (trim($columns[5]) !== '' && !$has_other_statute) {
- return [sprintf('CSV record %d: Column F: Ex. 3 Code 77 must appear in Column E if there is data in Column F', $record)];
+ $errors[] = sprintf('CSV record %d: Column F: Ex. 3 Code 77 must appear in Column E if there is data in Column F', $record);
}
// Column G: Information Withheld requires code 77 in E and data in F.
if (trim($columns[6]) !== '' && (!$has_other_statute || trim($columns[5]) === '')) {
- return [sprintf('CSV record %d: Column G: Ex. 3 Code 77 must appear in Column E if there is data in Column G', $record)];
+ $errors[] = sprintf('CSV record %d: Column G: Ex. 3 Code 77 must appear in Column E if there is data in Column G', $record);
}
// Column H: Case Citation requires code 77 in E and data in F and G.
if (trim($columns[7]) !== '' && (!$has_other_statute || trim($columns[5]) === '' || trim($columns[6]) === '')) {
- return [sprintf('CSV record %d: Column H: Ex. 3 Code 77 must appear in Column E if there is data in Column H', $record)];
+ $errors[] = sprintf('CSV record %d: Column H: Ex. 3 Code 77 must appear in Column E if there is data in Column H', $record);
}
// Column I: Optional, but the Column X check requires it to be blank
// for appeal rows. Validate date contents only when present.
$initially_received = trim($columns[8]);
- $received_date = NULL;
+ $received_date = $this->parseDate($initially_received);
if ($initially_received !== '') {
// Column I: Require a real calendar date in month/day/four-digit year
// order. Allow single-digit months/days as used in the reference CSV.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $initially_received, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column I: Date Initially Received must be a valid date in MM/DD/YYYY format', $record)];
+ if ($received_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column I: Date Initially Received must be a valid date in MM/DD/YYYY format', $record);
}
// Column I: September 30 of the report year is the latest date.
// Compare dates without time zones; earlier years are allowed.
- $received_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($received_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column I: Date Initially Received is later than the fiscal year', $record)];
+ if ($received_date !== NULL && ($received_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column I: Date Initially Received is later than the fiscal year', $record);
}
}
// Column J: Date Perfected is optional; validate only nonblank values.
// The Column C check already requires J to be blank for consultations.
$perfected = trim($columns[9]);
+ $perfected_date = $this->parseDate($perfected);
if ($perfected !== '') {
// Column J: Require a real date using the same format as Column I.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $perfected, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column J: Date Perfected must be a valid date in MM/DD/YYYY format', $record)];
+ if ($perfected_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column J: Date Perfected must be a valid date in MM/DD/YYYY format', $record);
}
// Column J: The date cannot exceed September 30 of the report year.
- $perfected_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($perfected_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column J: Date Perfected is later than the fiscal year', $record)];
+ if ($perfected_date !== NULL && ($perfected_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column J: Date Perfected is later than the fiscal year', $record);
}
// Column J: A perfected date requires an uppercase S, C or E in M.
if (!in_array(trim($columns[12]), ['S', 'C', 'E'], TRUE)) {
- return [sprintf('CSV record %d: Column J: If data is entered in Column J, Column M must contain capital S, C, or E', $record)];
+ $errors[] = sprintf('CSV record %d: Column J: If data is entered in Column J, Column M must contain capital S, C, or E', $record);
}
// Column J: Perfection cannot precede the initially received date.
- if ($received_date !== NULL && $perfected_date < $received_date) {
- return [sprintf('CSV record %d: Column J: Date Perfected cannot be prior to Date Initially Received', $record)];
+ if ($received_date !== NULL && $perfected_date !== NULL && $perfected_date < $received_date) {
+ $errors[] = sprintf('CSV record %d: Column J: Date Perfected cannot be prior to Date Initially Received', $record);
}
}
// Column K: Date Completed is required when N contains a disposition.
$completed = trim($columns[10]);
+ $completed_date = $this->parseDate($completed);
if ($completed === '' && trim($columns[13]) !== '') {
- return [sprintf('CSV record %d: Column K: Must complete Column K if Disposition is listed in Column N', $record)];
+ $errors[] = sprintf('CSV record %d: Column K: Must complete Column K if Disposition is listed in Column N', $record);
}
// Column K: Otherwise optional, including for consultation rows.
if ($completed !== '') {
// Column K: Require a real date using the same format as I and J.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $completed, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column K: Date Completed must be a valid date in MM/DD/YYYY format', $record)];
+ if ($completed_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column K: Date Completed must be a valid date in MM/DD/YYYY format', $record);
}
// Column K: Completion must fall within the report's fiscal year,
// from October 1 of the previous year through September 30 inclusive.
- $completed_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($completed_date < ($fiscal_year - 1) * 10000 + 1001 || $completed_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column K: Date Completed must fall within the fiscal year (10/01/%04d through 09/30/%04d)', $record, $fiscal_year - 1, $fiscal_year)];
+ if ($completed_date !== NULL && ($completed_date < ($fiscal_year - 1) * 10000 + 1001 || $completed_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column K: Date Completed must fall within the fiscal year (10/01/%04d through 09/30/%04d)', $record, $fiscal_year - 1, $fiscal_year);
}
// Column K: Completion cannot precede J when a perfected date exists.
- if ($perfected !== '' && $completed_date < $perfected_date) {
- return [sprintf('CSV record %d: Column K: Date Completed cannot be prior to Date Perfected', $record)];
+ if ($perfected_date !== NULL && $completed_date !== NULL && $completed_date < $perfected_date) {
+ $errors[] = sprintf('CSV record %d: Column K: Date Completed cannot be prior to Date Perfected', $record);
}
}
@@ -242,19 +254,19 @@ public function validate(string $uri, int $fiscal_year): array {
$days_tolled = trim($columns[11]);
if ($days_tolled !== '') {
if ($perfected === '') {
- return [sprintf('CSV record %d: Column L: Days Tolled only permitted if Perfected Date is entered', $record)];
+ $errors[] = sprintf('CSV record %d: Column L: Days Tolled only permitted if Perfected Date is entered', $record);
}
// Column L: Accept non-negative whole days, including zero.
if (!ctype_digit($days_tolled)) {
- return [sprintf('CSV record %d: Column L: Days Tolled must be a non-negative integer', $record)];
+ $errors[] = sprintf('CSV record %d: Column L: Days Tolled must be a non-negative integer', $record);
}
// Column L: When K is present, limit tolling to elapsed working days.
// Exclude J, include K, and subtract weekends and supplied holidays.
// With no completed date, the upper limit cannot yet be checked.
- if ($completed !== '' && (int) $days_tolled > $this->countWorkingDays($perfected, $completed)) {
- return [sprintf('CSV record %d: Column L: Days Tolled exceeds working days between Perfected and Completed Dates (Columns J and K)', $record)];
+ if (ctype_digit($days_tolled) && $perfected_date !== NULL && $completed_date !== NULL && $completed_date >= $perfected_date && (int) $days_tolled > $this->countWorkingDays($perfected, $completed)) {
+ $errors[] = sprintf('CSV record %d: Column L: Days Tolled exceeds working days between Perfected and Completed Dates (Columns J and K)', $record);
}
}
@@ -262,163 +274,259 @@ public function validate(string $uri, int $fiscal_year): array {
// requires uppercase S, C or E whenever a perfected date is present.
// Non-appeal rows with G in S require E in M. Appeal rows keep M blank.
if ($appeal_received === '' && trim($columns[18]) === 'G' && trim($columns[12]) !== 'E') {
- return [sprintf('CSV record %d: Column M: If Column S contains a G, Column M must contain an E', $record)];
+ $errors[] = sprintf('CSV record %d: Column M: If Column S contains a G, Column M must contain an E', $record);
}
// Column N: Completed non-consultation requests need a disposition.
$disposition = trim($columns[13]);
if ($completed !== '' && $consultation === 'N' && $disposition === '') {
- return [sprintf('CSV record %d: Column N: If Column K has Date Completed AND Column C = N (No), then Column N must have Disposition entered', $record)];
+ $errors[] = sprintf('CSV record %d: Column N: If Column K has Date Completed AND Column C = N (No), then Column N must have Disposition entered', $record);
}
// Column N: Disposition code 12 requires explanatory data in O.
if ($disposition === '12' && trim($columns[14]) === '') {
- return [sprintf('CSV record %d: Column N: Column O must contain data for Disposition Code 12', $record)];
+ $errors[] = sprintf('CSV record %d: Column N: Column O must contain data for Disposition Code 12', $record);
}
// Column N: Codes 1, 2, 3, 4, 5 and 7 require a perfected date in J.
if (in_array($disposition, ['1', '2', '3', '4', '5', '7'], TRUE) && $perfected === '') {
- return [sprintf('CSV record %d: Column N: If Disposition Code 1, 2, 3, 4, 5, or 7 is entered in Column N, Column J must have a Perfected Date', $record)];
+ $errors[] = sprintf('CSV record %d: Column N: If Disposition Code 1, 2, 3, 4, 5, or 7 is entered in Column N, Column J must have a Perfected Date', $record);
}
// Column N: Codes 8 and 9 require the perfected date in J to be blank.
if (in_array($disposition, ['8', '9'], TRUE) && $perfected !== '') {
- return [sprintf('CSV record %d: Column N: If Disposition Code 8 or 9 is listed in Column N, Column J must be blank for that request', $record)];
+ $errors[] = sprintf('CSV record %d: Column N: If Disposition Code 8 or 9 is listed in Column N, Column J must be blank for that request', $record);
}
// Column O: An Other Reason requires disposition code 12 in N.
// This is the reverse of the Column N check requiring O for code 12.
if (trim($columns[14]) !== '' && $disposition !== '12') {
- return [sprintf('CSV record %d: Column O: If Column O contains information, Column N must contain Disposition Code 12', $record)];
+ $errors[] = sprintf('CSV record %d: Column O: If Column O contains information, Column N must contain Disposition Code 12', $record);
}
// Column P: Optional alphanumeric exemptions must be comma-separated.
// Allow whitespace around entries, but not in place of a comma.
$applied_exemptions = trim($columns[15]);
if ($applied_exemptions !== '' && !preg_match('/^[a-zA-Z0-9]+(?:\s*,\s*[a-zA-Z0-9]+)*$/', $applied_exemptions)) {
- return [sprintf('CSV record %d: Column P: Multiple exemptions must be separated with a comma (e.g., 3,5,7a,7c,7d)', $record)];
+ $errors[] = sprintf('CSV record %d: Column P: Multiple exemptions must be separated with a comma (e.g., 3,5,7a,7c,7d)', $record);
+ }
+ // Column P: Each comma-separated entry must match an allowed code.
+ $exemptions = array_map('trim', explode(',', $applied_exemptions));
+ if ($applied_exemptions !== '' && array_diff($exemptions, self::ALLOWED_DISPOSITION_EXEMPTIONS)) {
+ $errors[] = sprintf('CSV record %d: Column P: Disposition Exemption(s) Applied must contain only these values: %s', $record, implode(', ', self::ALLOWED_DISPOSITION_EXEMPTIONS));
}
// Column P: Disposition code 3 requires at least one exemption.
if ($disposition === '3' && $applied_exemptions === '') {
- return [sprintf('CSV record %d: Column P: Column P must contain exemption(s) for Disposition Code 3', $record)];
+ $errors[] = sprintf('CSV record %d: Column P: Column P must contain exemption(s) for Disposition Code 3', $record);
}
// Column P: Exemptions are allowed only for disposition codes 2 or 3.
if ($applied_exemptions !== '' && !in_array($disposition, ['2', '3'], TRUE)) {
- return [sprintf('CSV record %d: Column P: Column N must list Disposition Code 2 or 3 if Column P contains exemptions', $record)];
+ $errors[] = sprintf('CSV record %d: Column P: Column N must list Disposition Code 2 or 3 if Column P contains exemptions', $record);
}
// Column P: A complete exemption 3 entry requires statute data in E.
// This complements the Column E check requiring exemption 3 in P.
- $exemptions = array_map('trim', explode(',', $applied_exemptions));
if (in_array('3', $exemptions, TRUE) && $statutes === '') {
- return [sprintf("CSV record %d: Column P: If Column P contains a '3', Column E must contain information", $record)];
+ $errors[] = sprintf("CSV record %d: Column P: If Column P contains a '3', Column E must contain information", $record);
}
// Column Q: Request for EP - Date Received is optional.
$ep_received = trim($columns[16]);
+ $ep_received_date = $this->parseDate($ep_received);
if ($ep_received !== '') {
// Column Q: Require a real date using the same format as I, J and K.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $ep_received, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column Q: Request for EP - Date Received must be a valid date in MM/DD/YYYY format', $record)];
+ if ($ep_received_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column Q: Request for EP - Date Received must be a valid date in MM/DD/YYYY format', $record);
}
// Column Q: September 30 of the report year is the latest date.
// Earlier years are allowed, as for Date Initially Received.
- $ep_received_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($ep_received_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column Q: Request for EP - Date Received is later than the fiscal year', $record)];
+ if ($ep_received_date !== NULL && ($ep_received_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column Q: Request for EP - Date Received is later than the fiscal year', $record);
}
}
// Column Q: A received date is required when R or S has data.
$ep_determined = trim($columns[17]);
+ $ep_determined_date = $this->parseDate($ep_determined);
if ($ep_received === '' && ($ep_determined !== '' || trim($columns[18]) !== '')) {
- return [sprintf('CSV record %d: Column Q: Column Q must contain value if there is value in either Columns R or S', $record)];
+ $errors[] = sprintf('CSV record %d: Column Q: Column Q must contain value if there is value in either Columns R or S', $record);
}
if ($ep_determined !== '') {
// Column R: Require a real date using the same format as Column Q.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $ep_determined, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column R: Request for EP - Date of Determination must be a valid date in MM/DD/YYYY format', $record)];
+ if ($ep_determined_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column R: Request for EP - Date of Determination must be a valid date in MM/DD/YYYY format', $record);
}
// Column R: Require the report fiscal year, including its boundaries.
- $ep_determined_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($ep_determined_date < ($fiscal_year - 1) * 10000 + 1001 || $ep_determined_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column R: Request for EP - Date of Determination is outside the fiscal year', $record)];
+ if ($ep_determined_date !== NULL && ($ep_determined_date < ($fiscal_year - 1) * 10000 + 1001 || $ep_determined_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column R: Request for EP - Date of Determination is outside the fiscal year', $record);
}
// Column R: Determination cannot precede a received date in Q.
- if ($ep_received !== '' && $ep_determined_date < $ep_received_date) {
- return [sprintf('CSV record %d: Column R: Request Expedited Processing - Date of Determination cannot be prior to Request Expedited Processing - Date Received', $record)];
+ if ($ep_received_date !== NULL && $ep_determined_date !== NULL && $ep_determined_date < $ep_received_date) {
+ $errors[] = sprintf('CSV record %d: Column R: Request Expedited Processing - Date of Determination cannot be prior to Request Expedited Processing - Date Received', $record);
}
}
// Column S: A determination date in R requires uppercase G or D.
if ($ep_determined !== '' && !in_array(trim($columns[18]), ['G', 'D'], TRUE)) {
- return [sprintf('CSV record %d: Column S: Must have G or D in Column S', $record)];
+ $errors[] = sprintf('CSV record %d: Column S: Must have G or D in Column S', $record);
}
- // Column T: Request for FW - Date Adjudication Began is optional.
+ // Column T: A value in U, V, or W requires an adjudication start date.
$fw_began = trim($columns[19]);
+ if ($fw_began === '' && (trim($columns[20]) !== '' || trim($columns[21]) !== '' || trim($columns[22]) !== '')) {
+ $errors[] = sprintf('CSV record %d: Column T: Column T must contain a value if there is data in Columns U, V, or W', $record);
+ }
+ $fw_began_date = $this->parseDate($fw_began);
if ($fw_began !== '') {
// Column T: Require a real date using the same format as Column Q.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $fw_began, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column T: Request for FW - Date Adjudication Began must be a valid date in MM/DD/YYYY format', $record)];
+ if ($fw_began_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column T: Request for FW - Date Adjudication Began must be a valid date in MM/DD/YYYY format', $record);
}
// Column T: September 30 of the report year is the latest date.
// Earlier years are allowed, as for the other received dates.
- $fw_began_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($fw_began_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column T: Request for FW - Date Adjudication Began is later than the fiscal year', $record)];
+ if ($fw_began_date !== NULL && ($fw_began_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column T: Request for FW - Date Adjudication Began is later than the fiscal year', $record);
}
}
// Column U: Request for FW - Date Adjudication Completed is optional.
$fw_completed = trim($columns[20]);
+ $fw_completed_date = $this->parseDate($fw_completed);
if ($fw_completed !== '') {
// Column U: Require a real date using the same format as Column T.
- if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $fw_completed, $date_parts)
- || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
- return [sprintf('CSV record %d: Column U: Request for FW - Date Adjudication Completed must be a valid date in MM/DD/YYYY format', $record)];
+ if ($fw_completed_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column U: Request for FW - Date Adjudication Completed must be a valid date in MM/DD/YYYY format', $record);
}
// Column U: Require the report fiscal year, including its boundaries.
- $fw_completed_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
- if ($fw_completed_date < ($fiscal_year - 1) * 10000 + 1001 || $fw_completed_date > $fiscal_year * 10000 + 930) {
- return [sprintf('CSV record %d: Column U: Request for FW - Date Adjudication Completed is outside of the fiscal year', $record)];
+ if ($fw_completed_date !== NULL && ($fw_completed_date < ($fiscal_year - 1) * 10000 + 1001 || $fw_completed_date > $fiscal_year * 10000 + 930)) {
+ $errors[] = sprintf('CSV record %d: Column U: Request for FW - Date Adjudication Completed is outside of the fiscal year', $record);
}
// Column U: Completion cannot precede the start date in T, if given.
- if ($fw_began !== '' && $fw_completed_date < $fw_began_date) {
- return [sprintf('CSV record %d: Column U: Request for Fee Waiver - Date Adjudication Completed cannot be prior to Request for Fee Waiver - Date Adjudication Began', $record)];
+ if ($fw_began_date !== NULL && $fw_completed_date !== NULL && $fw_completed_date < $fw_began_date) {
+ $errors[] = sprintf('CSV record %d: Column U: Request for Fee Waiver - Date Adjudication Completed cannot be prior to Request for Fee Waiver - Date Adjudication Began', $record);
}
}
// Column V: A completion date in U requires uppercase G or D.
if ($fw_completed !== '' && !in_array(trim($columns[21]), ['G', 'D'], TRUE)) {
- return [sprintf('CSV record %d: Column V: Must complete G or D in Column V', $record)];
+ $errors[] = sprintf('CSV record %d: Column V: Must complete G or D in Column V', $record);
+ }
+
+ // Column X: Require an appeal receipt date when I is blank.
+ // The earlier Column X exclusion rule already forbids I and X together.
+ if ($appeal_received === '' && $initially_received === '') {
+ $errors[] = sprintf('CSV record %d: Column X: Appeal Date Received cannot be blank', $record);
+ }
+ $appeal_received_date = NULL;
+ if ($appeal_received !== '') {
+ // Column X: Require a real date, no later than fiscal year-end.
+ $appeal_received_date = $this->parseDate($appeal_received);
+ if ($appeal_received_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column X: Appeal Date Received must be a valid date in MM/DD/YYYY format', $record);
+ }
+ elseif ($appeal_received_date > $fiscal_year * 10000 + 930) {
+ $errors[] = sprintf('CSV record %d: Column X: Appeal Date Received is later than the fiscal year', $record);
+ }
+ }
+
+ // Column Y: A populated Appeal Date Closed must be a real date.
+ $appeal_closed = trim($columns[24]);
+ $appeal_closed_date = $this->parseDate($appeal_closed);
+ if ($appeal_closed !== '') {
+ if ($appeal_closed_date === NULL) {
+ $errors[] = sprintf('CSV record %d: Column Y: Appeal Date Closed must be a valid date in MM/DD/YYYY format', $record);
+ }
+ // Column Y: Closure must be within the inclusive fiscal year.
+ elseif ($appeal_closed_date < ($fiscal_year - 1) * 10000 + 1001 || $appeal_closed_date > $fiscal_year * 10000 + 930) {
+ $errors[] = sprintf('CSV record %d: Column Y: Appeal Date Closed is outside the fiscal year', $record);
+ }
+ }
+
+ // Column Y: A closed date requires a disposition in Column Z.
+ $appeal_disposition = trim($columns[25]);
+ if ($appeal_closed !== '' && $appeal_disposition === '') {
+ $errors[] = sprintf('CSV record %d: Column Y: Appeal Disposition is required if Appeal Closed Date is entered ', $record);
+ }
+
+ // Column Y: A disposition in Column Z requires a closed date.
+ if ($appeal_disposition !== '' && $appeal_closed === '') {
+ $errors[] = sprintf('CSV record %d: Column Y: Appeal Closed Date is required if Appeal Disposition is entered', $record);
+ }
+
+ // Column Y: Compare valid dates only; closure cannot precede receipt.
+ if ($appeal_closed_date !== NULL && $appeal_received_date !== NULL && $appeal_closed_date < $appeal_received_date) {
+ $errors[] = sprintf('CSV record %d: Column Y: Appeal Date Closed cannot be prior to Appeal Date Received', $record);
+ }
+
+ // Column Z: Affirmed appeals need denial reasons or exemptions.
+ // Include the reference CSV's full label, Affirmed on Appeal.
+ $affirmed_dispositions = [
+ 'Affirmed',
+ 'Affirmed on Appeal',
+ 'Partially Affirmed & Partially Reversed/Remanded',
+ ];
+ if (in_array($appeal_disposition, $affirmed_dispositions, TRUE)
+ && trim($columns[26]) === '' && trim($columns[28]) === '') {
+ $errors[] = sprintf('CSV record %d: Column Z: If Column Z has "Affirmed" or "Partially Affirmed & Partially Reversed/Remanded", there must be an entry in Column AA and/or Column AC', $record);
+ }
+
+ // Column Z: Other closures require at least one reason in AA.
+ if ($appeal_disposition === 'Closed for Other Reasons' && trim($columns[26]) === '') {
+ $errors[] = sprintf('CSV record %d: Column Z: If Column Z has appeal disposition "Closed for Other Reasons", then Column AA must have at least one reason entered', $record);
+ }
+
+ // Column AB: An Other reason in AA requires explanatory text.
+ // AA may contain multiple comma-separated reasons; match a whole entry.
+ $appeal_denial_reasons = array_map('trim', explode(',', $columns[26]));
+ if (in_array('Other', $appeal_denial_reasons, TRUE) && trim($columns[27]) === '') {
+ $errors[] = sprintf('CSV record %d: Column AB: If Column AA has "Other" entered, there must be a value in Column AB', $record);
+ }
+
+ // Column AC: Appeal exemptions require one of these dispositions in Z.
+ $exemption_dispositions = [
+ 'Affirmed on Appeal',
+ 'Partially Affirmed & Partially Reversed/Remanded',
+ ];
+ if (trim($columns[28]) !== '' && !in_array($appeal_disposition, $exemption_dispositions, TRUE)) {
+ $errors[] = sprintf('CSV record %d: Column AC: If AC has exemptions, Column Z must be Affirmed on Appeal or Partially Affirmed & Partially Reversed/Remanded', $record);
}
}
if (!feof($stream)) {
- return ['The CSV file could not be read completely. Please upload it again.'];
+ $errors[] = 'The CSV file could not be read completely. Please upload it again.';
}
if (!$has_content) {
- return ['The CSV file is empty. Please upload a CSV with 29 columns matching the OIP request raw data template.'];
+ $errors[] = 'The CSV file is empty. Please upload a CSV with 29 columns matching the OIP request raw data template.';
}
- return [];
+ // A repeated list-entry failure should appear only once per record.
+ return array_values(array_unique($errors));
}
finally {
fclose($stream);
}
}
+ /**
+ * Parses a real calendar date, returning NULL for blank or invalid input.
+ *
+ * Dependent comparisons run only when both dates parsed successfully.
+ */
+ private function parseDate(string $value): ?int {
+ if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+ || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+ return NULL;
+ }
+ return (int) $parts[3] * 10000 + (int) $parts[1] * 100 + (int) $parts[2];
+ }
+
/**
* Counts weekdays after the start date through the end date, minus holidays.
*
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php
index 0046c18bc..59c8ca4ee 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php
@@ -44,7 +44,8 @@ final class DispositionAggregator {
* Counts each nonblank disposition once per row, then sums component counts.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
*
* @return array
* Component counts keyed by entity ID and overall counts, keyed by code.
@@ -80,7 +81,7 @@ public function aggregate(array $sources): array {
}
// Reject unmapped values rather than silently omitting requests.
if (!preg_match('/^(?:[1-9]|1[0-2])$/', $code)) {
- throw new \RuntimeException(sprintf('Component %s, CSV record %d: Column N must contain a disposition code from 1 through 12.', $id, $record));
+ throw new \RuntimeException(sprintf('Component %s, CSV record %d: Column N must contain a disposition code from 1 through 12.', $source['component_label'] ?? $id, $record));
}
$components[$id][(int) $code]++;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/EventSubscriber/ReportNotificationSubscriber.php b/docroot/modules/custom/foia_raw_data_to_report/src/EventSubscriber/ReportNotificationSubscriber.php
new file mode 100644
index 000000000..89d7f3e62
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/EventSubscriber/ReportNotificationSubscriber.php
@@ -0,0 +1,39 @@
+getRequest();
+ if ($event->isMainRequest() && $request->isMethod('GET') && !$request->isXmlHttpRequest() && $request->getRequestFormat() === 'html' && !$request->query->has('_wrapper_format')) {
+ $this->notifications->deliver();
+ }
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public static function getSubscribedEvents(): array {
+ // After routing (32), before dynamic page cache (27).
+ return [KernelEvents::REQUEST => ['onRequest', 30]];
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php
index 87ce2ea5d..ac5f47bf3 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php
@@ -16,7 +16,8 @@ final class ExpeditedProcessingAggregator {
* Counts granted, denied, and adjudications within ten working days.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
*
* @return array
* Component counts keyed by entity ID and overall counts, keyed by outcome.
@@ -55,7 +56,7 @@ public function aggregate(array $sources): array {
$components[$id][self::OUTCOMES[$outcome]]++;
$received = trim($columns[16]);
$determined = trim($columns[17]);
- $context = sprintf('Component %s, CSV %s, record %d', $id, basename($source['uri']), $record);
+ $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $id, basename($source['uri']), $record);
$start = $this->calendarDate($received, $context . ', Column Q');
// R may be blank: count the outcome, but no adjudication interval.
if ($determined === '') {
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/FeesCollectedAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/FeesCollectedAggregator.php
new file mode 100644
index 000000000..ba23a2e2a
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/FeesCollectedAggregator.php
@@ -0,0 +1,40 @@
+ $values) {
+ $components[$id] = $this->calculate($values);
+ }
+ return [
+ 'components' => $components,
+ 'overall' => $this->calculate($statistics['overall']),
+ ];
+ }
+
+ /**
+ * Divides fees by total cost, without multiplying the ratio by 100.
+ */
+ private function calculate(array $values): array {
+ $amount = BigDecimal::of($values['FeesCollectedAmount']);
+ $cost = BigDecimal::of($values['TotalCostAmount']);
+ return [
+ 'amount' => (string) $amount->toScale(4),
+ // A zero denominator produces zero, even if fees are nonzero.
+ 'ratio' => $cost->isZero() ? '0.0000' : (string) $amount->dividedBy($cost, 4, RoundingMode::HalfUp),
+ ];
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Form/ComponentPlaceholders.php b/docroot/modules/custom/foia_raw_data_to_report/src/Form/ComponentPlaceholders.php
new file mode 100644
index 000000000..5ce3221f8
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Form/ComponentPlaceholders.php
@@ -0,0 +1,134 @@
+ $label) {
+ $label = Html::decodeEntities($label);
+ $options[$id] = ['title' => ['data' => ['#title' => $label, '#plain_text' => $label]]];
+ }
+ $form['field_component_uploads']['placeholders'] = [
+ '#type' => 'details',
+ '#title' => t('Add component placeholders'),
+ '#open' => TRUE,
+ '#weight' => -10,
+ 'components' => [
+ '#type' => 'tableselect',
+ '#header' => ['title' => t('Agency Component')],
+ '#options' => $options,
+ '#empty' => t('Select an agency to see its available components.'),
+ '#parents' => ['raw_data_placeholder_components'],
+ '#description' => t('Select components, or use the header checkbox to select or clear all. Existing component uploads are kept; duplicates are skipped.'),
+ ],
+ 'add' => [
+ '#type' => 'submit',
+ '#value' => t('Add placeholders for component data below'),
+ '#name' => 'add_component_placeholders',
+ '#submit' => [[static::class, 'submit']],
+ '#limit_validation_errors' => [],
+ '#ajax' => [
+ 'callback' => 'foia_raw_data_to_report_refresh_components',
+ 'wrapper' => 'raw-data-component-uploads',
+ ],
+ ],
+ 'result' => [
+ '#type' => 'container',
+ '#attributes' => ['role' => 'status'],
+ 'text' => ['#plain_text' => $form_state->get('component_placeholder_result') ?? ''],
+ ],
+ ];
+ }
+
+ /**
+ * Returns accessible components for the currently submitted agency.
+ */
+ protected static function components(FormStateInterface $form_state): array {
+ $node = $form_state->getFormObject()->getEntity();
+ $agency = $form_state->getUserInput()['field_agency'][0]['target_id'] ?? $node->get('field_agency')->target_id;
+ if (is_string($agency) && !ctype_digit($agency)) {
+ $agency = EntityAutocomplete::extractEntityIdFromAutocompleteInput($agency);
+ }
+ $handler = \Drupal::service('plugin.manager.entity_reference_selection')->getInstance([
+ 'target_type' => 'node',
+ 'handler' => 'raw_data_component',
+ 'target_bundles' => ['agency_component' => 'agency_component'],
+ 'agency_id' => $agency ?: 0,
+ 'sort' => ['field' => 'title', 'direction' => 'ASC'],
+ ]);
+ return $handler->getReferenceableEntities()['agency_component'] ?? [];
+ }
+
+ /**
+ * Appends selected components without saving the node or paragraphs.
+ */
+ public static function submit(array &$form, FormStateInterface $form_state): void {
+ $field = 'field_component_uploads';
+ $parents = $form[$field]['widget']['#field_parents'];
+ $state = WidgetBase::getWidgetState($parents, $field, $form_state);
+ $input = $form_state->getUserInput();
+ // Recheck membership and access on submission, including tampered input.
+ $selected = array_intersect_key(static::components($form_state), array_filter($input['raw_data_placeholder_components'] ?? []));
+ $empty = [];
+ foreach ($state['paragraphs'] ?? [] as $delta => $item) {
+ if (in_array($item['mode'], ['remove', 'removed'], TRUE)) {
+ continue;
+ }
+ $paragraph = $item['entity'];
+ $component = $input[$field][$delta]['subform']['field_agency_component'][0]['target_id'] ?? $paragraph->get('field_agency_component')->target_id;
+ $has_component_input = !empty($component);
+ if (is_string($component) && !ctype_digit($component)) {
+ $component = EntityAutocomplete::extractEntityIdFromAutocompleteInput($component);
+ }
+ if ($component) {
+ unset($selected[$component]);
+ }
+ elseif (!$has_component_input && $paragraph->get('field_request_data_csv')->isEmpty() && $paragraph->get('section_ix_xi_data')->isEmpty() && empty($input[$field][$delta]['subform']['field_request_data_csv'][0]['fids']) && empty($input[$field][$delta]['subform']['section_ix_xi_data'][0]['fids'])) {
+ $empty[] = $delta;
+ }
+ }
+ $count = count($selected);
+ foreach ($selected as $id => $label) {
+ if ($empty) {
+ $delta = array_shift($empty);
+ $state['paragraphs'][$delta]['entity']->set('field_agency_component', $id);
+ // Submitted empty input would otherwise override the new default.
+ unset($input[$field][$delta]['subform']['field_agency_component']);
+ }
+ else {
+ $delta = $state['items_count']++;
+ $paragraph = Paragraph::create([
+ 'type' => 'raw_data_component_upload',
+ 'field_agency_component' => $id,
+ ]);
+ $paragraph->setParentEntity($form_state->getFormObject()->getEntity(), $field);
+ $state['paragraphs'][$delta] = [
+ 'entity' => $paragraph,
+ 'display' => EntityFormDisplay::collectRenderDisplay($paragraph, 'default'),
+ 'mode' => 'edit',
+ ];
+ }
+ }
+ unset($input['raw_data_placeholder_components']);
+ $form_state->setUserInput($input);
+ WidgetBase::setWidgetState($parents, $field, $form_state, $state);
+ $form_state->set('component_placeholder_result', t('Added @count component placeholder(s). Existing uploads were preserved.', ['@count' => $count]));
+ $form_state->setRebuild();
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Form/GenerateXmlReportForm.php b/docroot/modules/custom/foia_raw_data_to_report/src/Form/GenerateXmlReportForm.php
index ef3575714..bdc410b0f 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Form/GenerateXmlReportForm.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Form/GenerateXmlReportForm.php
@@ -96,11 +96,13 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
}
/**
- * Enqueues the node ID without reading or processing the CSV file yet.
+ * Enqueues the node and requester without processing the CSV file yet.
*/
protected function generateXmlReport(NodeInterface $node): bool {
return $this->queueFactory->get('raw_data_to_report_processing')->createItem([
'nid' => (int) $node->id(),
+ 'requester_uid' => (int) $this->currentUser()->id(),
+ 'notification_id' => bin2hex(random_bytes(16)),
]) !== FALSE;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php
index 9fd678d0d..1577902b0 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php
@@ -11,7 +11,8 @@ final class OldestPendingAppealAggregator {
* Collects at most ten receipt dates per component and for the agency.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year.
*
@@ -20,6 +21,7 @@ final class OldestPendingAppealAggregator {
*/
public function aggregate(array $sources, int $fiscal_year): array {
$end = new \DateTimeImmutable($fiscal_year . '-09-30', new \DateTimeZone('UTC'));
+ $working_days = new WorkingDays();
$components = [];
$overall = [];
foreach ($sources as $source) {
@@ -49,15 +51,15 @@ public function aggregate(array $sources, int $fiscal_year): array {
if ($received_text === '' || trim($columns[24]) !== '') {
continue;
}
- $context = sprintf('Component %s, CSV record %d, Column X', $id, $record);
+ $context = sprintf('Component %s, CSV record %d, Column X', $source['component_label'] ?? $id, $record);
$received = $this->calendarDate($received_text, $context);
if ($received > $end) {
throw new \RuntimeException($context . ': Appeal Date Received is after the report fiscal year.');
}
- // Use the full elapsed calendar time, including prior fiscal years.
+ // Use elapsed working days, including time in prior fiscal years.
$item = [
'receipt_date' => $received->format('Y-m-d'),
- 'pending_days' => (int) $received->diff($end)->days,
+ 'pending_days' => $working_days->count($received_text, $end->format('m/d/Y')),
];
$this->retainOldest($components[$id], $item);
$this->retainOldest($overall, $item);
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php
index a5f5df04b..ba26861f1 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php
@@ -11,14 +11,17 @@ final class OldestPendingRequestAggregator {
* Collects at most ten receipt dates per component and for the agency.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year.
+ * @param bool $consultations_only
+ * Whether to include only rows with Column C equal to Y.
*
* @return array
* Component and overall lists with receipt_date and pending_days entries.
*/
- public function aggregate(array $sources, int $fiscal_year): array {
+ public function aggregate(array $sources, int $fiscal_year, bool $consultations_only = FALSE): array {
$end = new \DateTimeImmutable($fiscal_year . '-09-30', new \DateTimeZone('UTC'));
$working_days = new WorkingDays();
$components = [];
@@ -45,12 +48,16 @@ public function aggregate(array $sources, int $fiscal_year): array {
$header = FALSE;
continue;
}
+ // Consultation listings include only rows marked Y in Column C.
+ if ($consultations_only && trim($columns[2]) !== 'Y') {
+ continue;
+ }
// Pending requests have a received date and no completed date.
$received_text = trim($columns[8]);
if ($received_text === '' || trim($columns[10]) !== '') {
continue;
}
- $context = sprintf('Component %s, CSV %s, record %d, Column I', $id, basename($source['uri']), $record);
+ $context = sprintf('Component %s, CSV %s, record %d, Column I', $source['component_label'] ?? $id, basename($source['uri']), $record);
$received = $this->calendarDate($received_text, $context);
if ($received > $end) {
throw new \RuntimeException($context . ': Date Initially Received is after the report fiscal year.');
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php
index 781a6fbde..9b9090302 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php
@@ -20,7 +20,8 @@ final class PendingPerfectedRequestsAggregator {
* Calculates component and agency counts, medians, and averages.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year, ending September 30.
*
@@ -59,7 +60,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
if ($perfected === '' || trim($columns[10]) !== '') {
continue;
}
- $context = sprintf('Component %s, CSV %s, record %d', $id, basename($source['uri']), $record);
+ $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $id, basename($source['uri']), $record);
$track = trim($columns[12]);
if (!isset(self::TRACKS[$track])) {
throw new \RuntimeException($context . ': Column M must contain S, C, or E for a perfected request.');
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/PersonnelAndCostAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/PersonnelAndCostAggregator.php
new file mode 100644
index 000000000..8d89e3e92
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/PersonnelAndCostAggregator.php
@@ -0,0 +1,94 @@
+validate($source['uri']);
+ $context = sprintf('Component %s, CSV %s', $source['component_label'] ?? $source['component_id'], basename($source['uri']));
+ if ($errors) {
+ throw new \RuntimeException($context . ': ' . implode(' ', $errors));
+ }
+ $stream = @fopen($source['uri'], 'rb');
+ if ($stream === FALSE) {
+ throw new \RuntimeException($context . ': Unable to reopen Section IX-XI CSV.');
+ }
+ try {
+ if (fread($stream, 3) !== "\xEF\xBB\xBF") {
+ rewind($stream);
+ }
+ $rows = [];
+ while (($row = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+ if ($row !== [NULL]) {
+ $rows[] = $row;
+ }
+ // Retain only this template's two records, even if the file changed.
+ if (count($rows) > 2) {
+ break;
+ }
+ }
+ if (!feof($stream) || count($rows) !== 2 || $rows[0] !== SectionDataCsvValidator::HEADERS || count($rows[1]) !== 8) {
+ throw new \RuntimeException($context . ': Section IX-XI CSV changed after validation.');
+ }
+ $values = [];
+ foreach ($rows[1] as $index => $value) {
+ $value = trim($value);
+ $pattern = $index === 1 ? '/^[+-]?[0-9]+(?:\.[0-9]+)?$/' : '/^[+-]?[0-9]+$/';
+ if (!preg_match($pattern, $value)) {
+ throw new \RuntimeException($context . ': Column ' . chr(65 + $index) . ': Invalid number after validation.');
+ }
+ $values[] = BigDecimal::of($value);
+ }
+ [$employees, $equivalent, $processing, $litigation, $fees, $exclusions, $foia_posts, $program_posts] = $values;
+ $totals = [
+ $employees,
+ $equivalent,
+ $employees->plus($equivalent),
+ $processing,
+ $litigation,
+ $processing->plus($litigation),
+ $fees,
+ $exclusions,
+ $foia_posts,
+ $program_posts,
+ ];
+ $id = $source['component_id'];
+ $components[$id] ??= array_fill_keys($fields, '0');
+ foreach ($fields as $index => $field) {
+ $components[$id][$field] = (string) BigDecimal::of($components[$id][$field])->plus($totals[$index])->strippedOfTrailingZeros();
+ $overall[$field] = (string) BigDecimal::of($overall[$field])->plus($totals[$index])->strippedOfTrailingZeros();
+ }
+ }
+ finally {
+ fclose($stream);
+ }
+ }
+ return ['components' => $components, 'overall' => $overall];
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 0e7f13486..b8f67bc7c 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -12,10 +12,14 @@
use Drupal\node\NodeInterface;
use Drupal\taxonomy\TermInterface;
use Drupal\foia_raw_data_to_report\CsvValidator;
+use Drupal\foia_raw_data_to_report\PersonnelAndCostAggregator;
+use Drupal\foia_raw_data_to_report\SectionDataCsvValidator;
+use Drupal\foia_raw_data_to_report\ReportNotifications;
use Drupal\foia_raw_data_to_report\UploadAssignments;
use Drupal\foia_raw_data_to_report\XmlReportBuilder;
use Drupal\foia_raw_data_to_report\StatuteAggregator;
use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
+use Drupal\foia_raw_data_to_report\ConsultationStatisticsAggregator;
use Drupal\foia_raw_data_to_report\DispositionAggregator;
use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
@@ -25,6 +29,8 @@
use Drupal\foia_raw_data_to_report\OldestPendingRequestAggregator;
use Drupal\foia_raw_data_to_report\ExpeditedProcessingAggregator;
use Drupal\foia_raw_data_to_report\FeeWaiverAggregator;
+use Drupal\foia_raw_data_to_report\FeesCollectedAggregator;
+use Drupal\foia_raw_data_to_report\BacklogAggregator;
use Drupal\foia_raw_data_to_report\ProcessedResponseTimeAggregator;
use Drupal\foia_raw_data_to_report\PendingPerfectedRequestsAggregator;
use Drupal\foia_raw_data_to_report\AppealDispositionAggregator;
@@ -44,7 +50,7 @@ final class RawDataToReportProcessing extends QueueWorkerBase implements Contain
/**
* Constructs the report queue worker.
*/
- public function __construct(array $configuration, $plugin_id, $plugin_definition, protected EntityTypeManagerInterface $entityTypeManager, protected FileSystemInterface $fileSystem, protected FileRepositoryInterface $fileRepository, protected CsvValidator $csvValidator) {
+ public function __construct(array $configuration, $plugin_id, $plugin_definition, protected EntityTypeManagerInterface $entityTypeManager, protected FileSystemInterface $fileSystem, protected FileRepositoryInterface $fileRepository, protected CsvValidator $csvValidator, protected ReportNotifications $notifications, protected SectionDataCsvValidator $sectionCsvValidator) {
parent::__construct($configuration, $plugin_id, $plugin_definition);
}
@@ -60,6 +66,8 @@ public static function create(ContainerInterface $container, array $configuratio
$container->get('file_system'),
$container->get('file.repository'),
$container->get('foia_raw_data_to_report.csv_validator'),
+ $container->get('foia_raw_data_to_report.notifications'),
+ $container->get('foia_raw_data_to_report.section_csv_validator'),
);
}
@@ -120,6 +128,22 @@ public function processItem($data) {
}
$has_errors = $has_errors || (bool) $errors;
$messages[] = $prefix . "\n" . ($errors ? implode("\n", array_unique($errors)) : 'CSV validated.');
+
+ // Check the Section IX-XI file independently, even if raw data failed.
+ $section_source = $supported ? $upload->get('section_ix_xi_data')->entity : NULL;
+ $section_errors = [];
+ if (!$section_source) {
+ $section_errors[] = 'No Section IX-XI CSV file is attached. Please upload it again.';
+ }
+ elseif (strtolower(pathinfo($section_source->getFilename(), PATHINFO_EXTENSION)) !== 'csv') {
+ $section_errors[] = 'Please upload a CSV file. Excel workbooks are not supported by this processor.';
+ }
+ else {
+ $section_errors = $this->sectionCsvValidator->validate($section_source->getFileUri());
+ }
+ $has_errors = $has_errors || (bool) $section_errors;
+ $section_prefix = sprintf('Upload %d — Component: %s — Section IX-XI file: %s', $delta + 1, $component?->label() ?? '(not selected)', $section_source?->getFilename() ?? '(not attached)');
+ $messages[] = $section_prefix . "\n" . ($section_errors ? implode("\n", $section_errors) : 'CSV validated.');
}
// Compare against every linked component, regardless of publication/access.
// Missing uploads are a warning only; CSV validation still controls errors.
@@ -144,6 +168,7 @@ public function processItem($data) {
$node->save();
if ($has_errors) {
// Invalid input is a completed task, not a retryable exception.
+ $this->notifications->record($data, 'validation_failed');
return;
}
try {
@@ -161,9 +186,11 @@ public function processItem($data) {
]);
$stored_node->save();
}
+ $this->notifications->record($data, 'failed');
// Preserve Drush logging and the queue's existing retry behavior.
throw $exception;
}
+ $this->notifications->record($data, 'success');
}
/**
@@ -172,11 +199,18 @@ public function processItem($data) {
protected function generateXmlReport(NodeInterface $node): void {
$components = [];
$sources = [];
+ $section_sources = [];
foreach ($node->get('field_component_uploads') as $item) {
$component = $item->entity->get('field_agency_component')->entity;
$components[] = $component;
+ $section_sources[] = [
+ 'component_id' => $component->id(),
+ 'component_label' => $component->label(),
+ 'uri' => $item->entity->get('section_ix_xi_data')->entity->getFileUri(),
+ ];
$sources[] = [
'component_id' => $component->id(),
+ 'component_label' => $component->label(),
'uri' => $item->entity->get('field_request_data_csv')->entity->getFileUri(),
];
}
@@ -203,7 +237,12 @@ protected function generateXmlReport(NodeInterface $node): void {
$oldest_pending_requests = (new OldestPendingRequestAggregator())->aggregate($sources, $fiscal_year);
$expedited_processing = (new ExpeditedProcessingAggregator())->aggregate($sources);
$fee_waivers = (new FeeWaiverAggregator())->aggregate($sources);
- $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times, $oldest_pending_appeals, $processed_response_times, $information_granted_response_times, $simple_response_increments, $complex_response_increments, $expedited_response_increments, $pending_perfected_requests, $oldest_pending_requests, $expedited_processing, $fee_waivers);
+ $backlog = (new BacklogAggregator())->aggregate($sources, $fiscal_year);
+ $consultation_statistics = (new ConsultationStatisticsAggregator())->aggregate($sources, $fiscal_year);
+ $oldest_pending_consultations = (new OldestPendingRequestAggregator())->aggregate($sources, $fiscal_year, TRUE);
+ $personnel_and_cost = (new PersonnelAndCostAggregator())->aggregate($section_sources);
+ $fees_collected = (new FeesCollectedAggregator())->aggregate($personnel_and_cost);
+ $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times, $oldest_pending_appeals, $processed_response_times, $information_granted_response_times, $simple_response_increments, $complex_response_increments, $expedited_response_increments, $pending_perfected_requests, $oldest_pending_requests, $expedited_processing, $fee_waivers, $fees_collected, $backlog, $consultation_statistics, $oldest_pending_consultations, $personnel_and_cost);
$field = $node->get('field_request_data_xml');
$previous_file = $field->entity;
$item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php
index 5543f81e2..e1aa9b73a 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php
@@ -20,7 +20,8 @@ final class ProcessedResponseTimeAggregator {
* Returns component and agency summaries without retaining individual rows.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param bool $information_granted_only
* Whether to include only disposition codes 1 and 2 in Column N.
*
@@ -152,7 +153,7 @@ private function collectHistograms(array $sources, bool $information_granted_onl
// Prefer J; fall back to I without clamping to the fiscal year.
$perfected = trim($columns[9]);
$start = $perfected !== '' ? $perfected : trim($columns[8]);
- $context = sprintf('Component %s, CSV record %d', $id, $record);
+ $context = sprintf('Component %s, CSV record %d', $source['component_label'] ?? $id, $record);
if ($start === '') {
throw new \RuntimeException($context . ': Completed requests require a date in Column J or I to calculate response time.');
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ReportNotifications.php b/docroot/modules/custom/foia_raw_data_to_report/src/ReportNotifications.php
new file mode 100644
index 000000000..d2e2d1c77
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/ReportNotifications.php
@@ -0,0 +1,97 @@
+storeFactory->get('foia_report_notifications.' . (int) $item['requester_uid']);
+ $store->setWithExpireIfNotExists($item['notification_id'] . ':' . $outcome, [
+ 'nid' => $item['nid'],
+ 'job' => $item['notification_id'],
+ 'outcome' => $outcome,
+ 'delivered' => FALSE,
+ ], self::LIFETIME);
+ }
+ catch (\Throwable $exception) {
+ // A notification storage problem must not regenerate or discard XML.
+ $this->logger->error('Unable to store report notification: @message', ['@message' => $exception->getMessage()]);
+ }
+ }
+
+ /**
+ * Transfers pending notices to the current user's Drupal message session.
+ */
+ public function deliver(): void {
+ if (!$this->currentUser->isAuthenticated()) {
+ return;
+ }
+ $uid = (int) $this->currentUser->id();
+ $lock_name = 'foia_report_notifications.' . $uid;
+ // Concurrent tabs must not deliver the same notice twice.
+ if (!$this->lock->acquire($lock_name)) {
+ return;
+ }
+ try {
+ $store = $this->storeFactory->get($lock_name);
+ foreach ($store->getAll() as $key => $notice) {
+ if ($notice['delivered']) {
+ continue;
+ }
+ // A successful retry supersedes an unread processing failure.
+ $superseded = $notice['outcome'] === 'failed' && $store->has($notice['job'] . ':success');
+ $node = $this->entityTypeManager->getStorage('node')->load($notice['nid']);
+ if (!$superseded && $node instanceof NodeInterface && $node->bundle() === 'raw_data_to_report' && $node->access('view', $this->currentUser)) {
+ $arguments = ['@report' => $node->toLink()->toString()];
+ $message = match ($notice['outcome']) {
+ 'success' => new TranslatableMarkup('CSV to XML processing finished for @report.', $arguments),
+ 'validation_failed' => new TranslatableMarkup('CSV validation failed for @report. Review the report’s messages.', $arguments),
+ default => new TranslatableMarkup('CSV to XML processing failed for @report. Review the report’s messages.', $arguments),
+ };
+ $this->messenger->addMessage($message, $notice['outcome'] === 'success' ? MessengerInterface::TYPE_STATUS : MessengerInterface::TYPE_ERROR);
+ }
+ // Retain a receipt to suppress notifications from retries of this job.
+ $notice['delivered'] = TRUE;
+ $store->setWithExpire($key, $notice, self::LIFETIME);
+ }
+ }
+ catch (\Throwable $exception) {
+ $this->logger->error('Unable to deliver report notification: @message', ['@message' => $exception->getMessage()]);
+ }
+ finally {
+ $this->lock->release($lock_name);
+ }
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php
index d4110c167..2a26f9828 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php
@@ -21,7 +21,8 @@ final class RequestStatisticsAggregator {
* Returns component counters and agency totals without retaining CSV rows.
*
* @param array $sources
- * Component/file pairs, each with component_id and uri keys.
+ * Component/file pairs with component_id and uri keys, plus an optional
+ * component_label for error messages.
* @param int $fiscal_year
* The validated report year.
*
@@ -53,9 +54,15 @@ public function aggregate(array $sources, int $fiscal_year): array {
$header = FALSE;
continue;
}
- // Every data row counts, including consultations and rows without E.
- $context = sprintf('Component %s, CSV %s, record %d', $source['component_id'], basename($source['uri']), $record);
- $received = $this->calendarDate(trim($columns[8]), $context . ', Column I (Date Initially Received)');
+ // Initial request statistics require a receipt date in Column I.
+ // Skip appeal-only rows and any other rows without that date before
+ // counting completions or pending requests for either XML section.
+ $received_text = trim($columns[8]);
+ if ($received_text === '') {
+ continue;
+ }
+ $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $source['component_id'], basename($source['uri']), $record);
+ $received = $this->calendarDate($received_text, $context . ', Column I (Date Initially Received)');
if ($received < $start) {
$counts['pending_start']++;
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/SectionDataCsvValidator.php b/docroot/modules/custom/foia_raw_data_to_report/src/SectionDataCsvValidator.php
new file mode 100644
index 000000000..311085f6f
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/SectionDataCsvValidator.php
@@ -0,0 +1,90 @@
+ $header) {
+ $column = chr(65 + $index);
+ // Header spelling, capitalization, whitespace, and order must match.
+ if ($is_header) {
+ if ($columns[$index] !== $header) {
+ $errors[] = sprintf('CSV record %d: Column %s: Header must be exactly "%s".', $record, $column, $header);
+ }
+ continue;
+ }
+ // B accepts decimal notation; all other cells require whole numbers.
+ // Match text rather than casting, so large integer values stay exact.
+ $value = trim($columns[$index]);
+ $pattern = $index === 1 ? '/^[+-]?[0-9]+(?:\.[0-9]+)?$/' : '/^[+-]?[0-9]+$/';
+ if (!preg_match($pattern, $value)) {
+ $errors[] = sprintf('CSV record %d: Column %s: %s must be %s.', $record, $column, $header, $index === 1 ? 'a decimal number' : 'an integer');
+ }
+ }
+ }
+ if (!feof($stream)) {
+ $errors[] = 'The Section IX-XI CSV file could not be read completely. Please upload it again.';
+ }
+ if (!$has_header) {
+ $errors[] = 'The Section IX-XI CSV file is empty. Upload the header and one data row.';
+ }
+ elseif ($data_rows !== 1) {
+ $errors[] = sprintf('The Section IX-XI CSV must contain exactly one data row after the header; found %d.', $data_rows);
+ }
+ return $errors;
+ }
+ finally {
+ fclose($stream);
+ }
+ }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/UploadAssignments.php b/docroot/modules/custom/foia_raw_data_to_report/src/UploadAssignments.php
index 5080b96f8..52ce36d3b 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/UploadAssignments.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/UploadAssignments.php
@@ -35,6 +35,9 @@ public static function validate(NodeInterface $node): array {
}
$seen[$component->id()] = TRUE;
}
+ if ($upload->get('section_ix_xi_data')->isEmpty()) {
+ $errors[$delta][] = 'Upload a Section IX-XI CSV file for this Agency Component.';
+ }
if ($upload->get('field_request_data_csv')->isEmpty()) {
$errors[$delta][] = 'Upload a CSV file for this Agency Component.';
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 73268515d..3b9fe42c2 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -73,11 +73,21 @@ final class XmlReportBuilder {
* Component and agency counters from ExpeditedProcessingAggregator.
* @param array $fee_waivers
* Component and agency counters from FeeWaiverAggregator.
+ * @param array $fees_collected
+ * Component and agency totals in cents from FeesCollectedAggregator.
+ * @param array $backlog
+ * Component and agency counters from BacklogAggregator.
+ * @param array $consultation_statistics
+ * Consultation counters by component and overall.
+ * @param array $oldest_pending_consultations
+ * Component and overall lists of the ten oldest pending consultations.
+ * @param array $personnel_and_cost
+ * Exact component and agency totals from PersonnelAndCostAggregator.
*
* @return string
* The serialized report XML.
*/
- public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = [], array $oldest_pending_appeals = [], array $processed_response_times = [], array $information_granted_response_times = [], array $simple_response_increments = [], array $complex_response_increments = [], array $expedited_response_increments = [], array $pending_perfected_requests = [], array $oldest_pending_requests = [], array $expedited_processing = [], array $fee_waivers = []): string {
+ public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = [], array $oldest_pending_appeals = [], array $processed_response_times = [], array $information_granted_response_times = [], array $simple_response_increments = [], array $complex_response_increments = [], array $expedited_response_increments = [], array $pending_perfected_requests = [], array $oldest_pending_requests = [], array $expedited_processing = [], array $fee_waivers = [], array $fees_collected = [], array $backlog = [], array $consultation_statistics = [], array $oldest_pending_consultations = [], array $personnel_and_cost = []): string {
$document = new \DOMDocument('1.0', 'UTF-8');
$document->formatOutput = TRUE;
$root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -193,6 +203,41 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
$this->addFeeWaivers($document, $root, $fee_waivers, $component_map);
}
+ $this->addPersonnelAndCost($document, $root, $component_map, $personnel_and_cost);
+
+ if ($fees_collected !== []) {
+ $this->addFeesCollected($document, $root, $fees_collected, $component_map);
+ }
+
+ $this->addSubsectionUsed($document, $root, $component_map, $personnel_and_cost);
+ $this->addSubsectionPost($document, $root, $component_map, $personnel_and_cost);
+
+ if ($backlog !== []) {
+ $this->addBacklog($document, $root, $backlog, $component_map);
+ }
+ if ($consultation_statistics !== []) {
+ $this->addProcessingStatistics($document, $root, $consultation_statistics, $component_map, 'ProcessedConsultationSection', 'PCN');
+ }
+ if ($oldest_pending_consultations !== []) {
+ $this->addOldestPendingItems($document, $root, $oldest_pending_consultations, $component_map, 'OldestPendingConsultationSection', 'OPC');
+ }
+
+ if ($request_statistics !== []) {
+ $this->addProcessingComparison($document, $root, $request_statistics, $component_map, 'ProcessedRequestComparisonSection', 'PRC');
+ }
+
+ if ($backlog !== []) {
+ $this->addBacklogComparison($document, $root, $backlog, $component_map, 'BackloggedRequestComparisonSection', 'BLR', 'requests');
+ }
+
+ if ($appeal_statistics !== []) {
+ $this->addProcessingComparison($document, $root, $appeal_statistics, $component_map, 'ProcessedAppealComparisonSection', 'APC');
+ }
+
+ if ($backlog !== []) {
+ $this->addBacklogComparison($document, $root, $backlog, $component_map, 'BackloggedAppealComparisonSection', 'ABC', 'appeals');
+ }
+
$xml = $document->saveXML();
if ($xml === FALSE) {
throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -239,7 +284,7 @@ private function addStatutes(\DOMDocument $document, \DOMElement $root, array $s
}
/**
- * Adds request or appeal counters and their organization references.
+ * Adds processing counters and their organization references.
*/
private function addProcessingStatistics(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map, string $section_name, string $prefix): void {
$section = $this->addTextElement($document, $root, 'foia', $section_name);
@@ -255,7 +300,7 @@ private function addProcessingStatistics(\DOMDocument $document, \DOMElement $ro
}
$organizations['ORG0'] = $statistics['overall'];
- // Match organization suffixes: PS/PA1 links to ORG1, PS/PA0 to ORG0.
+ // Match each statistics ID suffix to its corresponding organization.
foreach ($organizations as $organization_id => $counts) {
$entry = $this->addTextElement($document, $section, 'foia', 'ProcessingStatistics');
$entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
@@ -273,6 +318,67 @@ private function addProcessingStatistics(\DOMDocument $document, \DOMElement $ro
}
}
+ /**
+ * Adds request or appeal comparisons with zero counts for last year.
+ */
+ private function addProcessingComparison(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map, string $section_name, string $prefix): void {
+ $section = $this->addTextElement($document, $root, 'foia', $section_name);
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id];
+ }
+ $organizations['ORG0'] = $statistics['overall'];
+
+ foreach ($organizations as $organization_id => $counts) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'ProcessingComparison');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
+ // Reuse the inclusive fiscal-year counts already aggregated.
+ $fields = [
+ 'ItemsReceivedLastYearQuantity' => 0,
+ 'ItemsReceivedCurrentYearQuantity' => $counts['received'],
+ 'ItemsProcessedLastYearQuantity' => 0,
+ 'ItemsProcessedCurrentYearQuantity' => $counts['processed'],
+ ];
+ foreach ($fields as $name => $quantity) {
+ $this->addTextElement($document, $entry, 'foia', $name, (string) $quantity);
+ }
+ }
+ foreach ($organizations as $organization_id => $counts) {
+ $association = $this->addTextElement($document, $section, 'foia', 'ProcessingComparisonOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
+ /**
+ * Adds request or appeal backlog comparisons with zero for last year.
+ */
+ private function addBacklogComparison(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map, string $section_name, string $prefix, string $counter): void {
+ $section = $this->addTextElement($document, $root, 'foia', $section_name);
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id];
+ }
+ $organizations['ORG0'] = $statistics['overall'];
+
+ foreach ($organizations as $organization_id => $counts) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'BacklogComparison');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
+ // Reuse the matching count emitted in BacklogSection.
+ $this->addTextElement($document, $entry, 'foia', 'BacklogLastYearQuantity', '0');
+ $this->addTextElement($document, $entry, 'foia', 'BacklogCurrentYearQuantity', (string) $counts[$counter]);
+ }
+ foreach ($organizations as $organization_id => $counts) {
+ $association = $this->addTextElement($document, $section, 'foia', 'BacklogComparisonOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
/**
* Adds disposition counts, totals, and references to their organizations.
*/
@@ -652,6 +758,144 @@ private function addFeeWaivers(\DOMDocument $document, \DOMElement $root, array
}
}
+ /**
+ * Adds personnel and cost totals with their organization references.
+ */
+ private function addPersonnelAndCost(\DOMDocument $document, \DOMElement $root, array $component_map, array $statistics): void {
+ $section = $this->addTextElement($document, $root, 'foia', 'PersonnelAndCostSection');
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id] ?? [];
+ }
+ $organizations['ORG0'] = $statistics['overall'] ?? [];
+ $fields = [
+ 'FullTimeEmployeeQuantity',
+ 'EquivalentFullTimeEmployeeQuantity',
+ 'TotalFullTimeStaffQuantity',
+ 'ProcessingCostAmount',
+ 'LitigationCostAmount',
+ 'TotalCostAmount',
+ ];
+ foreach ($organizations as $organization_id => $values) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'PersonnelAndCost');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'PC' . substr($organization_id, 3));
+ foreach ($fields as $name) {
+ $this->addTextElement($document, $entry, 'foia', $name, $values[$name] ?? 'N/A');
+ }
+ }
+ foreach ($organizations as $organization_id => $values) {
+ $association = $this->addTextElement($document, $section, 'foia', 'PersonnelAndCostOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'PC' . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
+ /**
+ * Adds Section IX-XI fee totals and their ratios to total costs.
+ */
+ private function addFeesCollected(\DOMDocument $document, \DOMElement $root, array $fees, array $component_map): void {
+ $section = $this->addTextElement($document, $root, 'foia', 'FeesCollectedSection');
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $fees['components'][$component_id];
+ }
+ $organizations['ORG0'] = $fees['overall'];
+ foreach ($organizations as $organization_id => $values) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'FeesCollected');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'FC' . substr($organization_id, 3));
+ $this->addTextElement($document, $entry, 'foia', 'FeesCollectedAmount', $values['amount']);
+ $this->addTextElement($document, $entry, 'foia', 'FeesCollectedCostPercent', $values['ratio']);
+ }
+ foreach ($organizations as $organization_id => $values) {
+ $association = $this->addTextElement($document, $section, 'foia', 'FeesCollectedOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'FC' . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
+ /**
+ * Adds Section IX-XI subsection-use counts and organization references.
+ */
+ private function addSubsectionUsed(\DOMDocument $document, \DOMElement $root, array $component_map, array $statistics): void {
+ $section = $this->addTextElement($document, $root, 'foia', 'SubsectionUsedSection');
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id]['TimesUsedQuantity'] ?? '0';
+ }
+ $organizations['ORG0'] = $statistics['overall']['TimesUsedQuantity'] ?? '0';
+ foreach ($organizations as $organization_id => $quantity) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'SubsectionUsed');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'SU' . substr($organization_id, 3));
+ $this->addTextElement($document, $entry, 'foia', 'TimesUsedQuantity', $quantity);
+ }
+ foreach ($organizations as $organization_id => $quantity) {
+ $association = $this->addTextElement($document, $section, 'foia', 'SubsectionUsedOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'SU' . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
+ /**
+ * Adds Section IX-XI posting counts and their organization references.
+ */
+ private function addSubsectionPost(\DOMDocument $document, \DOMElement $root, array $component_map, array $statistics): void {
+ $section = $this->addTextElement($document, $root, 'foia', 'SubsectionPostSection');
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id] ?? [];
+ }
+ $organizations['ORG0'] = $statistics['overall'] ?? [];
+ foreach ($organizations as $organization_id => $values) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'Subsection');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'SP' . substr($organization_id, 3));
+ $this->addTextElement($document, $entry, 'foia', 'PostedbyFOIAQuantity', $values['PostedbyFOIAQuantity'] ?? '0');
+ $this->addTextElement($document, $entry, 'foia', 'PostedbyProgramQuantity', $values['PostedbyProgramQuantity'] ?? '0');
+ }
+ foreach ($organizations as $organization_id => $values) {
+ $association = $this->addTextElement($document, $section, 'foia', 'SubsectionPostOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'SP' . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
+ /**
+ * Adds request and appeal backlog counts with organization links.
+ */
+ private function addBacklog(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void {
+ $section = $this->addTextElement($document, $root, 'foia', 'BacklogSection');
+ $organizations = [];
+ foreach ($component_map as $component_id => $organization_id) {
+ $organizations[$organization_id] = $statistics['components'][$component_id];
+ }
+ $organizations['ORG0'] = $statistics['overall'];
+ $fields = [
+ 'requests' => 'BackloggedRequestQuantity',
+ 'appeals' => 'BackloggedAppealQuantity',
+ ];
+ foreach ($organizations as $organization_id => $counts) {
+ $entry = $this->addTextElement($document, $section, 'foia', 'Backlog');
+ $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'BK' . substr($organization_id, 3));
+ foreach ($fields as $key => $name) {
+ $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]);
+ }
+ }
+ foreach ($organizations as $organization_id => $counts) {
+ $association = $this->addTextElement($document, $section, 'foia', 'BacklogOrganizationAssociation');
+ $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+ $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'BK' . substr($organization_id, 3));
+ $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+ $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+ }
+ }
+
/**
* Appends a namespaced element and safely escapes optional text.
*/
diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php b/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php
index 661e50b99..f674a0bff 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php
@@ -9,12 +9,16 @@
use Drupal\Core\File\FileSystemInterface;
use Drupal\file\Entity\File;
use Drupal\foia_raw_data_to_report\UploadAssignments;
+use Drupal\foia_raw_data_to_report\SectionDataCsvValidator;
+use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
use Drupal\node\Entity\Node;
use Drupal\paragraphs\Entity\Paragraph;
use Drupal\system\FileDownloadController;
use Drupal\taxonomy\Entity\Term;
use Drupal\user\Entity\User;
use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpKernel\Event\RequestEvent;
+use Symfony\Component\HttpKernel\HttpKernelInterface;
use Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException;
if (getenv('IS_DDEV_PROJECT') !== 'true') {
@@ -31,7 +35,9 @@
$components = [];
$paragraphs = [];
$files = [];
+$section_files = [];
$report = NULL;
+$notification_store = NULL;
$suffix = bin2hex(random_bytes(6));
$node_storage = \Drupal::entityTypeManager()->getStorage('node');
$worker = \Drupal::service('plugin.manager.queue_worker')->createInstance('raw_data_to_report_processing');
@@ -72,6 +78,16 @@
]);
$paragraphs[] = $paragraph;
}
+ foreach ($paragraphs as $delta => $paragraph) {
+ $section_fees = $delta === 0 ? '908259' : '1816518';
+ $section_exclusions = $delta === 0 ? '7' : '0';
+ $section_foia_posts = $delta === 0 ? '145' : '0';
+ $section_program_posts = $delta === 0 ? '823' : '17';
+ $section_file = \Drupal::service('file.repository')->writeData(implode(',', SectionDataCsvValidator::HEADERS) . "\n39,1.85,7164103,1918485,$section_fees,$section_exclusions,$section_foia_posts,$section_program_posts", "private://section-check-$suffix-$delta.csv");
+ $section_file->setOwnerId($manager->id())->save();
+ $section_files[] = $section_file;
+ $paragraph->set('section_ix_xi_data', $section_file->id());
+ }
$report = Node::create([
'type' => 'raw_data_to_report',
'title' => "CSV report $suffix",
@@ -87,18 +103,34 @@
$check(count($paragraph->get('field_agency_component')->validate()) === 0, 'Component reference rejected on an unsaved report.');
}
$report->save();
- $process = static function () use ($worker, $report, $node_storage) {
- $worker->processItem(['nid' => (int) $report->id()]);
+ $notification_store = \Drupal::service('keyvalue.expirable')->get('foia_report_notifications.' . $manager->id());
+ $process = static function () use ($worker, $report, $node_storage, $manager, $suffix) {
+ $worker->processItem([
+ 'nid' => (int) $report->id(),
+ 'requester_uid' => (int) $manager->id(),
+ 'notification_id' => $suffix,
+ ]);
return $node_storage->loadUnchanged($report->id());
};
+ // Multiple failures in one upload must all reach the persisted messages.
+ file_put_contents($files[1]->getFileUri(), "\nshort,row", FILE_APPEND);
$report = $process();
$message = $report->get('field_messages')->value;
$check(str_contains($message, 'CSV validated.') && str_contains($message, '28 columns'), 'All files must be checked, with per-file results.');
+ $check(str_contains($message, 'CSV record 1 has 28 columns') && str_contains($message, 'CSV record 2 has 2 columns'), 'Messages lost errors from later records.');
foreach ($files as $file) {
$check(str_contains($message, $file->getFilename()), 'Messages must identify each filename.');
}
$check($report->get('field_request_data_xml')->isEmpty(), 'Invalid input generated XML.');
+ // The additional upload is required independently of its future contents.
+ $first_upload = $report->get('field_component_uploads')->get(0)->entity;
+ $section_id = $first_upload->get('section_ix_xi_data')->target_id;
+ $first_upload->set('section_ix_xi_data', []);
+ $check(count($first_upload->get('section_ix_xi_data')->validate()) > 0, 'Section IX-XI field was not required.');
+ $check(isset(UploadAssignments::validate($report)[0]), 'Missing Section IX-XI upload passed assignment validation.');
+ $first_upload->set('section_ix_xi_data', $section_id);
+
// Entity validation and queue processing must reject bad assignments.
$second = $report->get('field_component_uploads')->get(1)->entity;
$second->set('field_agency_component', $components[0]->id());
@@ -124,11 +156,61 @@
$check(!$selection->getReferenceableEntities(), 'Unselected agency must have no component choices.');
file_put_contents($files[1]->getFileUri(), implode(',', array_fill(0, 29, 'column')));
+ $valid_section = file_get_contents($section_files[1]->getFileUri());
+ file_put_contents($section_files[1]->getFileUri(), implode(',', SectionDataCsvValidator::HEADERS) . "\ninvalid,1.85,0,0,0,0,0,0");
+ $report = $process();
+ $check($report->get('field_request_data_xml')->isEmpty(), 'Invalid Section IX-XI data generated XML.');
+ $section_messages = $report->get('field_messages')->value;
+ $check(str_contains($section_messages, $section_files[1]->getFilename()) && str_contains($section_messages, 'Column A: Full-Time Employees must be an integer'), 'Section IX-XI error lacks file/column context.');
+ file_put_contents($section_files[1]->getFileUri(), $valid_section);
$report = $process();
$xml = $report->get('field_request_data_xml')->entity;
$check($xml && str_starts_with($xml->getFileUri(), 'private://'), 'Valid uploads did not produce private XML.');
$files[] = $xml;
- $check(substr_count($report->get('field_messages')->value, 'CSV validated.') === 2, 'Successful retry did not replace earlier messages.');
+ $document = new DOMDocument();
+ $check($document->loadXML(file_get_contents($xml->getFileUri())), 'Generated XML could not be parsed.');
+ $xpath = new DOMXPath($document);
+ $xpath->registerNamespace('foia', 'http://leisp.usdoj.gov/niem/FoiaAnnualReport/extension/1.03');
+ $xpath->registerNamespace('s', 'http://niem.gov/niem/structures/2.0');
+ $expected_totals = [
+ 1 => ['39', '1.85', '40.85', '7164103', '1918485', '9082588'],
+ 0 => ['78', '3.7', '81.7', '14328206', '3836970', '18165176'],
+ ];
+ foreach ($expected_totals as $id => $expected) {
+ $fields = [
+ 'FullTimeEmployeeQuantity',
+ 'EquivalentFullTimeEmployeeQuantity',
+ 'TotalFullTimeStaffQuantity',
+ 'ProcessingCostAmount',
+ 'LitigationCostAmount',
+ 'TotalCostAmount',
+ ];
+ foreach ($fields as $index => $field) {
+ $actual = $xpath->evaluate('string(//foia:PersonnelAndCost[@s:id="PC' . $id . '"]/foia:' . $field . ')');
+ $check($actual === $expected[$index], 'Incorrect personnel/cost XML value for ' . $field);
+ }
+ }
+
+ foreach ([1 => '7', 2 => '0', 0 => '7'] as $id => $expected) {
+ $actual = $xpath->evaluate('string(//foia:SubsectionUsed[@s:id="SU' . $id . '"]/foia:TimesUsedQuantity)');
+ $check($actual === $expected, 'Incorrect Section IX-XI subsection-use count.');
+ }
+ foreach ([1 => ['145', '823'], 2 => ['0', '17'], 0 => ['145', '840']] as $id => [$foia_posts, $program_posts]) {
+ $base = '//foia:Subsection[@s:id="SP' . $id . '"]';
+ $check($xpath->evaluate('string(' . $base . '/foia:PostedbyFOIAQuantity)') === $foia_posts, 'Incorrect FOIA posting count.');
+ $check($xpath->evaluate('string(' . $base . '/foia:PostedbyProgramQuantity)') === $program_posts, 'Incorrect program posting count.');
+ }
+ $expected_fees = [
+ 1 => ['908259.0000', '0.1000'],
+ 2 => ['1816518.0000', '0.2000'],
+ 0 => ['2724777.0000', '0.1500'],
+ ];
+ foreach ($expected_fees as $id => [$amount, $ratio]) {
+ $base = '//foia:FeesCollected[@s:id="FC' . $id . '"]';
+ $check($xpath->evaluate('string(' . $base . '/foia:FeesCollectedAmount)') === $amount, 'Incorrect Section IX-XI fees.');
+ $check($xpath->evaluate('string(' . $base . '/foia:FeesCollectedCostPercent)') === $ratio, 'Incorrect fee/cost ratio.');
+ }
+ $check(substr_count($report->get('field_messages')->value, 'CSV validated.') === 4, 'Successful retry did not replace earlier messages.');
// An aggregation exception must append to validation messages and retain XML.
$row = array_fill(0, 29, '');
$row[0] = 'Component';
@@ -138,16 +220,28 @@
$row[23] = '01/02/2026';
$contents = implode(',', array_fill(0, 29, 'column')) . "\n" . implode(',', $row);
file_put_contents($files[1]->getFileUri(), $contents);
+ // Appeal-only rows must not count as initial requests or pending requests.
+ $counts = (new RequestStatisticsAggregator())->aggregate([
+ ['component_id' => $components[1]->id(), 'uri' => $files[1]->getFileUri()],
+ ], 2026);
+ $check($counts['overall'] === ['pending_start' => 0, 'received' => 0, 'processed' => 0, 'pending_end' => 0], 'Appeal-only row contributed to request statistics.');
+ // An unknown appeal exemption exercises post-validation exception reporting.
+ $row[24] = '01/03/2026';
+ $row[25] = 'Affirmed on Appeal';
+ $row[28] = 'invalid-code';
+ $contents = implode(',', array_fill(0, 29, 'column')) . "\n" . implode(',', $row);
+ file_put_contents($files[1]->getFileUri(), $contents);
for ($attempt = 0; $attempt < 2; $attempt++) {
try {
$process();
- throw new LogicException('Expected the request-statistics date exception.');
+ throw new LogicException('Expected the appeal-exemption exception.');
}
catch (RuntimeException $exception) {
- $check(str_contains($exception->getMessage(), 'Column I'), 'Unexpected processing exception.');
+ $check(str_contains($exception->getMessage(), 'Column AC'), 'Unexpected processing exception.');
+ $check(str_starts_with($exception->getMessage(), 'Component ' . $components[1]->label() . ', CSV'), 'Processing exception must identify the component by name.');
$report = $node_storage->loadUnchanged($report->id());
$message = $report->get('field_messages')->value;
- $check(substr_count($message, 'CSV validated.') === 2, 'Exception lost validation messages.');
+ $check(substr_count($message, 'CSV validated.') === 4, 'Exception lost validation messages.');
$check(substr_count($message, 'XML report processing failed:') === 1, 'Retry duplicated exception messages.');
$check(str_ends_with($message, $exception->getMessage()), 'Exception details were not appended.');
$check($report->get('field_messages')->format === 'plain_text', 'Exception message must be plain text.');
@@ -171,7 +265,7 @@
foreach ([new AnonymousUserSession(), $manager] as $account) {
$switcher->switchTo($account);
try {
- foreach ($files as $file) {
+ foreach (array_merge($files, $section_files) as $file) {
try {
$response = $controller->download(new Request(['file' => substr($file->getFileUri(), 10)]));
$check(!$account->isAnonymous() && $response->getStatusCode() === 200, 'Anonymous download allowed.');
@@ -204,16 +298,64 @@
finally {
$switcher->switchBack();
}
- print "PASS: multi-file validation, retry, empty uploads, XML preservation, duplicate/agency constraints, component choices, private and detached downloads.\n";
+ // All worker outcomes are durable and addressed to the requesting user.
+ $notices = $notification_store->getAll();
+ $check(count($notices) === 3, 'Expected success, validation, and processing failure notices without retry duplicates.');
+ $messenger = \Drupal::messenger();
+ $messenger->deleteAll();
+ $notifications = \Drupal::service('foia_raw_data_to_report.notifications');
+ $notifications->deliver();
+ $check(!$messenger->all(), 'Notices leaked to a different account.');
+ $switcher->switchTo($manager);
+ try {
+ $subscriber = \Drupal::service('foia_raw_data_to_report.notification_subscriber');
+ $request = Request::create('/user');
+ $request->headers->set('X-Requested-With', 'XMLHttpRequest');
+ $event = new RequestEvent(\Drupal::service('http_kernel'), $request, HttpKernelInterface::MAIN_REQUEST);
+ $subscriber->onRequest($event);
+ $check(!$messenger->all(), 'AJAX consumed a notice.');
+ $request->headers->remove('X-Requested-With');
+ $request->setRequestFormat('json');
+ $subscriber->onRequest($event);
+ $check(!$messenger->all(), 'JSON consumed a notice.');
+ $request->setRequestFormat('html');
+ $subscriber->onRequest($event);
+ $messages = $messenger->deleteAll();
+ $check(count($messages['status'] ?? []) === 1 && count($messages['error'] ?? []) === 1, 'Missing completion notices or unread failure was not superseded.');
+ $check(str_contains((string) $messages['status'][0], 'label()), 'Completion notice lacks the report link/title.');
+ $notifications->deliver();
+ $check(!$messenger->all(), 'Delivered notices appeared twice.');
+ $notifications->record([
+ 'nid' => (int) $report->id(),
+ 'requester_uid' => (int) $manager->id(),
+ 'notification_id' => $suffix,
+ ], 'failed');
+ $notifications->deliver();
+ $check(!$messenger->all(), 'Retry repeated a delivered failure notice.');
+ $notifications->record(['nid' => (int) $report->id()], 'success');
+ $check(count($notification_store->getAll()) === 3, 'Legacy queue metadata created a notice.');
+ $notifications->record([
+ 'nid' => 0,
+ 'requester_uid' => (int) $manager->id(),
+ 'notification_id' => $suffix . '-missing',
+ ], 'success');
+ $notifications->deliver();
+ $check(!$messenger->all(), 'Missing report produced a broken notification.');
+ }
+ finally {
+ $switcher->switchBack();
+ }
+ print "PASS: multi-file validation, retry, empty uploads, XML preservation, duplicate/agency constraints, component choices, private and detached downloads, requester notifications and one-time delivery.\n";
}
finally {
+ $notification_store?->deleteAll();
if ($report) {
$report->delete();
}
foreach (array_reverse($entities) as $entity) {
$entity->delete();
}
- foreach ($files as $file) {
+ foreach (array_merge($files, $section_files) as $file) {
if ($stored = File::load($file->id())) {
$stored->delete();
}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
index b9c6000a9..b4acda3c8 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
@@ -28,12 +28,164 @@ private function validateContents(string $contents): array {
}
}
+ /**
+ * Requires exactly one receipt date and checks appeal dates and year-end.
+ */
+ public function testAppealReceivedRequiredAndFiscalYear(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ foreach (['', ' '] as $value) {
+ $row = $this->csvRow([8 => ' ', 23 => $value]);
+ $this->assertSame(['CSV record 2: Column X: Appeal Date Received cannot be blank'], $this->validateContents($header . $row));
+ }
+ $this->assertSame([], $this->validateContents($header . $this->csvRow()));
+ $this->assertSame([], $this->validateContents($header . $this->csvRow([2 => 'Y', 3 => ''])));
+ $both = $this->validateContents($header . $this->csvRow([23 => '01/02/2026']));
+ $this->assertCount(1, $both);
+ $this->assertStringContainsString('If there is data in Column X, Columns E through W must be empty', $both[0]);
+ foreach (['9/30/2026', '10/01/2025', '01/01/2020'] as $value) {
+ $this->assertSame([], $this->validateContents($header . $this->csvRow([8 => '', 23 => $value])));
+ }
+ $rows = $this->csvRow([8 => '', 23 => '10/01/2026']);
+ $rows .= $this->csvRow([1 => 'Request 2', 8 => '', 23 => '02/30/2026']);
+ $rows .= $this->csvRow([1 => 'Request 3', 8 => '', 23 => '']);
+ $this->assertSame([
+ 'CSV record 2: Column X: Appeal Date Received is later than the fiscal year',
+ 'CSV record 3: Column X: Appeal Date Received must be a valid date in MM/DD/YYYY format',
+ 'CSV record 4: Column X: Appeal Date Received cannot be blank',
+ ], $this->validateContents($header . $rows));
+ }
+
+ /**
+ * Checks appeal closure dates, dispositions, and independent row state.
+ */
+ public function testAppealClosedRules(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $base = [8 => '', 23 => '01/01/2025'];
+ foreach (['10/01/2025', '09/30/2026'] as $date) {
+ $row = $this->csvRow($base + [24 => $date, 25 => 'Affirmed on Appeal', 26 => 'No Records']);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+ $cases = [
+ [
+ [24 => '09/30/2025', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
+ 'Appeal Date Closed is outside the fiscal year',
+ ],
+ [
+ [24 => '10/01/2026', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
+ 'Appeal Date Closed is outside the fiscal year',
+ ],
+ [
+ [24 => '01/02/2026'],
+ 'Appeal Disposition is required if Appeal Closed Date is entered ',
+ ],
+ [
+ [24 => ' ', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
+ 'Appeal Closed Date is required if Appeal Disposition is entered',
+ ],
+ [
+ [23 => '01/03/2026', 24 => '01/02/2026', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
+ 'Appeal Date Closed cannot be prior to Appeal Date Received',
+ ],
+ [
+ [24 => '02/30/2026', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
+ 'Appeal Date Closed must be a valid date in MM/DD/YYYY format',
+ ],
+ ];
+ foreach ($cases as [$values, $message]) {
+ $row = $this->csvRow(array_replace($base, $values));
+ $this->assertSame(['CSV record 2: Column Y: ' . $message], $this->validateContents($header . $row));
+ }
+ // A previous row's received date must not leak into a blank X comparison.
+ $first = $this->csvRow([8 => '', 23 => '09/30/2026']);
+ $second = $this->csvRow([1 => 'Request 2', 24 => '01/02/2026', 25 => 'Affirmed on Appeal', 26 => 'No Records']);
+ $this->assertSame([], $this->validateContents($header . $first . $second));
+ $row = $this->csvRow([8 => '', 23 => '01/03/2026', 24 => '09/30/2025']);
+ $this->assertCount(3, $this->validateContents($header . $row));
+ }
+
+ /**
+ * Checks denial reasons and exemptions required by appeal dispositions.
+ */
+ public function testAppealDispositionReasons(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $base = [8 => '', 23 => '01/01/2026', 24 => '01/02/2026'];
+ foreach (['Affirmed', 'Affirmed on Appeal', 'Partially Affirmed & Partially Reversed/Remanded'] as $label) {
+ $row = $this->csvRow($base + [25 => $label, 26 => ' ', 28 => '']);
+ $this->assertSame([
+ 'CSV record 2: Column Z: If Column Z has "Affirmed" or "Partially Affirmed & Partially Reversed/Remanded", there must be an entry in Column AA and/or Column AC',
+ ], $this->validateContents($header . $row));
+ foreach ([[26 => 'No Records'], [28 => '3'], [26 => 'No Records', 28 => '3']] as $values) {
+ $row = $this->csvRow($base + [25 => ' ' . $label . ' '] + $values);
+ $expected = $label === 'Affirmed' && isset($values[28])
+ ? ['CSV record 2: Column AC: If AC has exemptions, Column Z must be Affirmed on Appeal or Partially Affirmed & Partially Reversed/Remanded']
+ : [];
+ $this->assertSame($expected, $this->validateContents($header . $row));
+ }
+ }
+ $row = $this->csvRow($base + [25 => 'Closed for Other Reasons']);
+ $this->assertSame([
+ 'CSV record 2: Column Z: If Column Z has appeal disposition "Closed for Other Reasons", then Column AA must have at least one reason entered',
+ ], $this->validateContents($header . $row));
+ $row = $this->csvRow($base + [25 => 'Closed for Other Reasons', 26 => 'No Records']);
+ $this->assertSame([], $this->validateContents($header . $row));
+ $row = $this->csvRow($base + [25 => 'Completely Reversed/Remanded']);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+
+ /**
+ * Requires explanation for the complete Other entry in appeal denial lists.
+ */
+ public function testAppealOtherReasonExplanation(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $base = [8 => '', 23 => '01/01/2026'];
+ foreach (['Other', 'No Records, Other', ' Other ,Other'] as $reasons) {
+ foreach (['', ' '] as $explanation) {
+ $row = $this->csvRow($base + [26 => $reasons, 27 => $explanation]);
+ $this->assertSame([
+ 'CSV record 2: Column AB: If Column AA has "Other" entered, there must be a value in Column AB',
+ ], $this->validateContents($header . $row));
+ }
+ $row = $this->csvRow($base + [26 => $reasons, 27 => 'Explanation']);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+ foreach (['', 'No Records', 'Improper Request for Other Reasons'] as $reasons) {
+ $row = $this->csvRow($base + [26 => $reasons]);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+ }
+
+ /**
+ * Restricts appeal exemptions to the two specified disposition labels.
+ */
+ public function testAppealExemptionDisposition(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $base = [8 => '', 23 => '01/01/2026', 24 => '01/02/2026', 26 => 'No Records'];
+ foreach (['Affirmed on Appeal', 'Partially Affirmed & Partially Reversed/Remanded'] as $label) {
+ $row = $this->csvRow($base + [25 => ' ' . $label . ' ', 28 => '3, 5']);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+ foreach (['Affirmed', 'Closed for Other Reasons', 'Completely Reversed/Remanded'] as $label) {
+ $row = $this->csvRow($base + [25 => $label, 28 => '3']);
+ $this->assertSame([
+ 'CSV record 2: Column AC: If AC has exemptions, Column Z must be Affirmed on Appeal or Partially Affirmed & Partially Reversed/Remanded',
+ ], $this->validateContents($header . $row));
+ }
+ $row = $this->csvRow([8 => '', 23 => '01/01/2026', 28 => '0']);
+ $this->assertSame([
+ 'CSV record 2: Column AC: If AC has exemptions, Column Z must be Affirmed on Appeal or Partially Affirmed & Partially Reversed/Remanded',
+ ], $this->validateContents($header . $row));
+ foreach (['', ' '] as $value) {
+ $row = $this->csvRow([8 => '', 23 => '01/01/2026', 28 => $value]);
+ $this->assertSame([], $this->validateContents($header . $row));
+ }
+ }
+
/**
* Accepts quoted commas, quotes, multiline values, and blank lines.
*/
public function testValidCsv(): void {
$header = implode(',', array_fill(0, 29, 'column'));
- $row = $this->csvRow([0 => 'A, B', 1 => "First line\nSecond line", 22 => 'A "quote"']);
+ $row = $this->csvRow([0 => 'A, B', 1 => "First line\nSecond line", 19 => '01/01/2026', 22 => 'A "quote"']);
$this->assertSame([], $this->validateContents("\xEF\xBB\xBF" . $header . "\r\n\r\n" . $row . "\r\n"));
}
@@ -61,7 +213,7 @@ private function csvRow(array $overrides = []): string {
public function testExpeditedReceivedDependency(): void {
$header = implode(',', array_fill(0, 29, 'column')) . "\n";
foreach ([
- [17 => '01/03/2026'],
+ [17 => '01/03/2026', 18 => 'D'],
[18 => 'D'],
[16 => ' ', 17 => '01/03/2026', 18 => 'D'],
] as $overrides) {
@@ -95,8 +247,8 @@ public function testAppealRequestDataExclusion(): void {
$overrides = [8 => '', 23 => '01/02/2026'];
// Zero is data too, even though PHP treats it as an empty value.
$overrides[$column] = '0';
- $this->assertSame(
- ['CSV record 2: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.'],
+ $this->assertContains(
+ 'CSV record 2: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.',
$this->validateContents($header . $this->csvRow($overrides)),
'Column index ' . $column,
);
@@ -120,10 +272,22 @@ public function testAppealExpeditedProcessing(): void {
$this->assertSame([], $this->validateContents($header . $this->csvRow($appeal + $values)));
}
$cases = [
- [[18 => 'D'], 'Column Q must contain value'],
- [[16 => 'bad-date'], 'Column Q: Request for EP - Date Received must be a valid date'],
- [[16 => '01/03/2026', 17 => '01/02/2026', 18 => 'D'], 'cannot be prior'],
- [[16 => '01/03/2026', 17 => '01/04/2026'], 'Must have G or D in Column S'],
+ [
+ [18 => 'D'],
+ 'Column Q must contain value',
+ ],
+ [
+ [16 => 'bad-date'],
+ 'Column Q: Request for EP - Date Received must be a valid date',
+ ],
+ [
+ [16 => '01/03/2026', 17 => '01/02/2026', 18 => 'D'],
+ 'cannot be prior',
+ ],
+ [
+ [16 => '01/03/2026', 17 => '01/04/2026'],
+ 'Must have G or D in Column S',
+ ],
];
foreach ($cases as [$values, $message]) {
$errors = $this->validateContents($header . $this->csvRow($appeal + $values));
@@ -132,15 +296,13 @@ public function testAppealExpeditedProcessing(): void {
}
/**
- * Allows blank I with or without X, and validates nonblank received dates.
+ * Allows blank I with X, and validates nonblank received dates.
*/
public function testOptionalInitiallyReceived(): void {
$header = implode(',', array_fill(0, 29, 'column')) . "\n";
foreach ([
- [8 => ''],
[8 => '', 23 => '01/02/2026'],
- [8 => ' ', 23 => '01/02/2026', 24 => '01/03/2026'],
- [8 => '', 9 => '01/02/2026', 12 => 'S'],
+ [8 => ' ', 23 => '01/02/2026', 24 => '01/03/2026', 25 => 'Affirmed on Appeal', 26 => 'No Records'],
[8 => '01/01/2026'],
] as $overrides) {
$this->assertSame([], $this->validateContents($header . $this->csvRow($overrides)));
@@ -156,7 +318,7 @@ public function testOptionalInitiallyReceived(): void {
// A blank I on the next row must not reuse the previous row's date.
$first = $this->csvRow([8 => '09/01/2026']);
$second = $this->csvRow([1 => 'Request 2', 8 => '', 9 => '01/02/2026', 12 => 'S']);
- $this->assertSame([], $this->validateContents($header . $first . $second));
+ $this->assertSame(['CSV record 3: Column X: Appeal Date Received cannot be blank'], $this->validateContents($header . $first . $second));
$row = $this->csvRow([8 => '01/03/2026', 9 => '01/02/2026', 12 => 'S']);
$this->assertSame(
['CSV record 2: Column J: Date Perfected cannot be prior to Date Initially Received'],
@@ -193,6 +355,65 @@ public function testInvalidDataRecord(): void {
$this->assertStringContainsString('record 3 has 30 columns; expected 29', $errors[0]);
}
+ /**
+ * Collects multiple failures per row and checks duplicates after errors.
+ */
+ public function testCollectsAllErrors(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $first = $this->csvRow([0 => '', 3 => '10']);
+ $second = $this->csvRow([0 => '', 16 => '', 18 => 'D']);
+ $this->assertSame([
+ 'CSV record 2: Column A: Component cannot be blank',
+ 'CSV record 2: Column D: Must complete Days Allowed with 20 or 30',
+ 'CSV record 3: Column A: Component cannot be blank',
+ 'CSV record 3: Column B: Request Number is duplicate',
+ 'CSV record 3: Column Q: Column Q must contain value if there is value in either Columns R or S',
+ ], $this->validateContents($header . $first . $second));
+ }
+
+ /**
+ * A malformed header or row must not suppress validation of later rows.
+ */
+ public function testContinuesAfterMalformedRecords(): void {
+ $contents = "bad,header\nshort,row\n" . $this->csvRow([0 => '']);
+ $errors = $this->validateContents($contents);
+ $this->assertCount(3, $errors);
+ $this->assertStringContainsString('record 1 has 2 columns', $errors[0]);
+ $this->assertStringContainsString('record 2 has 2 columns', $errors[1]);
+ $this->assertSame('CSV record 3: Column A: Component cannot be blank', $errors[2]);
+ }
+
+ /**
+ * Invalid dates cannot cause calculations or leak dates between rows.
+ */
+ public function testInvalidDatesDoNotPreventOtherChecks(): void {
+ $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+ $valid = $this->csvRow([9 => '01/02/2026', 12 => 'S']);
+ $invalid = $this->csvRow([
+ 1 => 'Request 2',
+ 8 => 'bad',
+ 9 => 'bad',
+ 10 => 'bad',
+ 11 => '1',
+ 12 => 'S',
+ 13 => '1',
+ 16 => 'bad',
+ 17 => 'bad',
+ 18 => 'D',
+ 19 => 'bad',
+ 20 => 'bad',
+ 21 => 'D',
+ ]);
+ $later = $this->csvRow([1 => 'Request 3', 0 => '']);
+ $errors = $this->validateContents($header . $valid . $invalid . $later);
+ $this->assertCount(8, $errors);
+ foreach (array_slice($errors, 0, 7) as $error) {
+ $this->assertStringContainsString('CSV record 3:', $error);
+ $this->assertStringContainsString('must be a valid date', $error);
+ }
+ $this->assertSame('CSV record 4: Column A: Component cannot be blank', $errors[7]);
+ }
+
/**
* Rejects empty input rather than silently producing an XML report.
*/
diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/SectionDataCsvValidatorTest.php b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/SectionDataCsvValidatorTest.php
new file mode 100644
index 000000000..bc38b731c
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/SectionDataCsvValidatorTest.php
@@ -0,0 +1,80 @@
+validate($path);
+ }
+ finally {
+ unlink($path);
+ }
+ }
+
+ /**
+ * Accepts the sample, zero values, signs, whitespace, BOM, and blank lines.
+ */
+ public function testValidData(): void {
+ $row = [39, '1.85', 7164103, 1918485, 0, 0, 145, 823];
+ $this->assertSame([], $this->validateCsv([SectionDataCsvValidator::HEADERS, $row], TRUE));
+ $row = ['-1', ' 0 ', '+1', 0, 0, 0, 0, 0];
+ $this->assertSame([], $this->validateCsv([[], SectionDataCsvValidator::HEADERS, $row, []]));
+ }
+
+ /**
+ * Requires exact headers and exactly one complete data record.
+ */
+ public function testStructure(): void {
+ $headers = SectionDataCsvValidator::HEADERS;
+ $this->assertNotEmpty($this->validateCsv([]));
+ $this->assertStringContainsString('found 0', implode(' ', $this->validateCsv([$headers])));
+ $row = array_fill(0, 8, '0');
+ $this->assertStringContainsString('found 2', implode(' ', $this->validateCsv([$headers, $row, $row])));
+ $errors = $this->validateCsv([$headers, array_fill(0, 7, '0')]);
+ $this->assertStringContainsString('record 2 has 7 columns', implode(' ', $errors));
+ $headers[0] .= ' ';
+ $headers[1] = 'equivalent full-time employees';
+ $errors = $this->validateCsv([$headers, $row]);
+ $this->assertCount(2, $errors);
+ $this->assertStringContainsString('record 1: Column A: Header', $errors[0]);
+ $this->assertStringContainsString('record 1: Column B: Header', $errors[1]);
+ }
+
+ /**
+ * Reports all invalid numeric cells without converting or rounding them.
+ */
+ public function testNumbers(): void {
+ $row = ['1.5', '1e3', '', '1,000', '$2', 'NaN', ' ', '2.0'];
+ $errors = $this->validateCsv([SectionDataCsvValidator::HEADERS, $row]);
+ $this->assertCount(8, $errors);
+ $this->assertStringContainsString('record 2: Column B: Equivalent Full-Time Employees must be a decimal number', $errors[1]);
+ foreach ([0, 2, 3, 4, 5, 6, 7] as $index) {
+ $this->assertStringContainsString('must be an integer', $errors[$index]);
+ }
+ }
+
+}
diff --git a/tests/behat/features/RawDataToReport.feature b/tests/behat/features/RawDataToReport.feature
index cd05a1497..ba8fbaff0 100644
--- a/tests/behat/features/RawDataToReport.feature
+++ b/tests/behat/features/RawDataToReport.feature
@@ -68,16 +68,21 @@ Feature: Raw data XML report action
And I am logged in as "raw_block_manager"
When I visit the entity of type "node" with the title "Component upload form"
And I click "Edit"
- And I press "Add Component CSV upload"
- And I fill in "Agency Component" with "Upload Fixture Component"
- And I press the "down" key in the "Agency Component" field
+ And I check "Update Upload Fixture Component"
+ And I press "Add placeholders for component data below"
+ And I wait for AJAX to finish
+ Then I should see "Added 1 component placeholder(s)."
+ And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_0_subform_field_request_data_csv_0]"
And I wait 2 seconds
- And I press the "down" key in the "Agency Component" field
- And I press the "enter" key in the "Agency Component" field
- And I attach the file "raw-data-valid.csv" to "CSV file"
+ And I wait for AJAX to finish
+ And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_0_subform_section_ix_xi_data_0]"
And I wait 2 seconds
And I wait for AJAX to finish
- And I press "Save"
+ And I check "Update Upload Fixture Component"
+ And I press "Add placeholders for component data below"
+ And I wait for AJAX to finish
+ Then I should see "Added 0 component placeholder(s)."
+ When I press "Save"
Then I should see "Upload Fixture Component"
And I should see "raw-data-valid"
And the URL should match "/node/[0-9]+$"
@@ -88,6 +93,9 @@ Feature: Raw data XML report action
And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_1_subform_field_request_data_csv_0]"
And I wait 2 seconds
And I wait for AJAX to finish
+ And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_1_subform_section_ix_xi_data_0]"
+ And I wait 2 seconds
+ And I wait for AJAX to finish
And I press "Save"
Then I should see "Each Agency Component may have only one CSV upload."
When I fill in "field_component_uploads[1][subform][field_agency_component][0][target_id]" with "Second Upload Component"
@@ -113,8 +121,14 @@ Feature: Raw data XML report action
And I press the "enter" key in the "Agency" field
And I press the "tab" key in the "Agency" field
And I wait for AJAX to finish
- And I fill in "Agency Component" with "New Report Component"
- And I attach the file "raw-data-valid.csv" to "CSV file"
+ And I check "Update New Report Component"
+ And I press "Add placeholders for component data below"
+ And I wait for AJAX to finish
+ Then I should see "Added 1 component placeholder(s)."
+ And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_0_subform_field_request_data_csv_0]"
+ And I wait 2 seconds
+ And I wait for AJAX to finish
+ And I attach the file "raw-data-valid.csv" to "files[field_component_uploads_0_subform_section_ix_xi_data_0]"
And I wait 2 seconds
And I wait for AJAX to finish
And I press "Save"
@@ -122,6 +136,34 @@ Feature: Raw data XML report action
And I should see "New Report Component"
And I should see "raw-data-valid"
+ @javascript
+ Scenario: Multiple placeholders can be added without uploading files first
+ Given agency_component content:
+ | title | field_agency | status |
+ | First Bulk Component | Raw Block Agency One | 1 |
+ | Second Bulk Component | Raw Block Agency One | 1 |
+ | Foreign Bulk Component | Raw Block Agency Two | 1 |
+ And raw_data_to_report content:
+ | title | field_agency | field_foia_annual_report_yr | field_agency_comp_abbreviation | status |
+ | Bulk placeholder form | Raw Block Agency One | 2026 | RBA1 | 1 |
+ And I am logged in as "raw_block_manager"
+ When I visit the entity of type "node" with the title "Bulk placeholder form"
+ And I click "Edit"
+ Then I should not see "Foreign Bulk Component"
+ When I check "Update First Bulk Component"
+ And I check "Update Second Bulk Component"
+ And I press "Add placeholders for component data below"
+ And I wait for AJAX to finish
+ Then I should see "Added 2 component placeholder(s)."
+ When I check "Update First Bulk Component"
+ And I check "Update Second Bulk Component"
+ And I press "Add placeholders for component data below"
+ And I wait for AJAX to finish
+ Then I should see "Added 0 component placeholder(s)."
+ When I press "Save"
+ Then I should see "CSV file field is required."
+ And I should see "Section IX-XI CSV file field is required."
+
@javascript
Scenario: Additional uploads autocomplete components on an unsaved report
Given agency_component content: