WS-472: raw data xml generation - all sections of XML should now be present

config/default/core.entity_form_display.paragraph.raw_data_component_upload.default.yml

@@ -5,6 +5,7 @@ dependencies:
   config:
     - field.field.paragraph.raw_data_component_upload.field_agency_component
     - field.field.paragraph.raw_data_component_upload.field_request_data_csv
+    - field.field.paragraph.raw_data_component_upload.section_ix_xi_data
     - paragraphs.paragraphs_type.raw_data_component_upload
   module:
     - file
@@ -30,6 +31,13 @@ content:
     settings:
       progress_indicator: throbber
     third_party_settings: {  }
+  section_ix_xi_data:
+    type: file_generic
+    weight: 2
+    region: content
+    settings:
+      progress_indicator: throbber
+    third_party_settings: {  }
 hidden:
   created: true
   status: true

config/default/core.entity_view_display.paragraph.raw_data_component_upload.default.yml

@@ -5,6 +5,7 @@ dependencies:
   config:
     - field.field.paragraph.raw_data_component_upload.field_agency_component
     - field.field.paragraph.raw_data_component_upload.field_request_data_csv
+    - field.field.paragraph.raw_data_component_upload.section_ix_xi_data
     - paragraphs.paragraphs_type.raw_data_component_upload
   module:
     - file
@@ -29,5 +30,13 @@ content:
     third_party_settings: {  }
     weight: 1
     region: content
+  section_ix_xi_data:
+    type: file_default
+    label: above
+    settings:
+      use_description_as_link_text: false
+    third_party_settings: {  }
+    weight: 2
+    region: content
 hidden:
   search_api_excerpt: true

config/default/field.field.paragraph.raw_data_component_upload.field_request_data_csv.yml

@@ -12,7 +12,7 @@ field_name: field_request_data_csv
 entity_type: paragraph
 bundle: raw_data_component_upload
 label: 'CSV file'
-description: ''
+description: 'Component Raw Data Report'
 required: true
 translatable: false
 default_value: {  }

config/default/field.field.paragraph.raw_data_component_upload.section_ix_xi_data.yml

@@ -0,0 +1,27 @@
+uuid: 36d4670e-76ee-47d2-bf44-c6cf284376f0
+langcode: en
+status: true
+dependencies:
+  config:
+    - field.storage.paragraph.section_ix_xi_data
+    - paragraphs.paragraphs_type.raw_data_component_upload
+  module:
+    - file
+id: paragraph.raw_data_component_upload.section_ix_xi_data
+field_name: section_ix_xi_data
+entity_type: paragraph
+bundle: raw_data_component_upload
+label: 'Section IX-XI CSV file'
+description: 'Section IX-XI Data'
+required: true
+translatable: false
+default_value: {  }
+default_value_callback: ''
+settings:
+  handler: 'default:file'
+  handler_settings: {  }
+  file_directory: 'request_data_tool/components/[date:custom:Y]-[date:custom:m]'
+  file_extensions: csv
+  max_filesize: ''
+  description_field: false
+field_type: file

config/default/field.storage.paragraph.section_ix_xi_data.yml

@@ -0,0 +1,23 @@
+uuid: 496fcb4b-f83e-46b1-baae-3a22a3dedee7
+langcode: en
+status: true
+dependencies:
+  module:
+    - file
+    - paragraphs
+id: paragraph.section_ix_xi_data
+field_name: section_ix_xi_data
+entity_type: paragraph
+type: file
+settings:
+  target_type: file
+  display_field: false
+  display_default: false
+  uri_scheme: private
+module: file
+locked: false
+cardinality: 1
+translatable: true
+indexes: {  }
+persist_with_no_fields: false
+custom_storage: false

docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.module

@@ -18,6 +18,7 @@ use Drupal\node\NodeInterface;
 use Drupal\file\FileInterface;
 use Drupal\file\FileReferenceResolver;
 use Drupal\foia_raw_data_to_report\Form\GenerateXmlReportForm;
+use Drupal\foia_raw_data_to_report\Form\ComponentPlaceholders;
 
 /**
  * Implements hook_node_view().
@@ -61,11 +62,12 @@ function foia_raw_data_to_report_file_access(FileInterface $file, $operation, Ac
   $access = AccessResult::neutral()->cachePerUser()->addCacheableDependency($file);
   $is_report_file = FALSE;
   $report_fields = ['field_request_data_csv', 'field_request_data_xml'];
+  $upload_fields = ['field_request_data_csv', 'section_ix_xi_data'];
   foreach ($resolver->getReferences($file) as $usage) {
     $entity = $resolver->loadEntityFromUsage($usage);
     $node = $entity;
     $parent_field = $usage->fieldName;
-    if ($entity instanceof ParagraphInterface && $entity->bundle() === 'raw_data_component_upload' && $usage->fieldName === 'field_request_data_csv') {
+    if ($entity instanceof ParagraphInterface && $entity->bundle() === 'raw_data_component_upload' && in_array($usage->fieldName, $upload_fields, TRUE)) {
       $is_report_file = TRUE;
       $access->addCacheableDependency($entity);
       $node = $entity->getParentEntity();
@@ -168,10 +170,11 @@ function foia_raw_data_to_report_form_node_form_alter(&$form, FormStateInterface
   }
   $form['field_component_uploads']['#prefix'] = '<div id="raw-data-component-uploads">';
   $form['field_component_uploads']['#suffix'] = '</div>';
+  ComponentPlaceholders::build($form, $form_state);
   $form['field_agency']['widget'][0]['target_id']['#ajax'] = [
     'callback' => 'foia_raw_data_to_report_refresh_components',
     'wrapper' => 'raw-data-component-uploads',
-    'event' => 'change',
+    'event' => 'autocompleteclose change',
   ];
 }
 

docroot/modules/custom/foia_raw_data_to_report/foia_raw_data_to_report.services.yml

@@ -1,3 +1,16 @@
 services:
   foia_raw_data_to_report.csv_validator:
     class: Drupal\foia_raw_data_to_report\CsvValidator
+  logger.channel.foia_raw_data_to_report:
+    parent: logger.channel_base
+    arguments: ['foia_raw_data_to_report']
+  foia_raw_data_to_report.notifications:
+    class: Drupal\foia_raw_data_to_report\ReportNotifications
+    arguments: ['@keyvalue.expirable', '@entity_type.manager', '@current_user', '@messenger', '@lock', '@logger.channel.foia_raw_data_to_report']
+  foia_raw_data_to_report.notification_subscriber:
+    class: Drupal\foia_raw_data_to_report\EventSubscriber\ReportNotificationSubscriber
+    arguments: ['@foia_raw_data_to_report.notifications']
+    tags:
+      - { name: event_subscriber }
+  foia_raw_data_to_report.section_csv_validator:
+    class: Drupal\foia_raw_data_to_report\SectionDataCsvValidator

docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php

@@ -21,7 +21,8 @@ final class AppealDispositionAggregator {
    * Counts each nonblank disposition once per row, then sums component counts.
    *
    * @param array $sources
-   *   Component/file pairs, each with component_id and uri keys.
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
    *
    * @return array
    *   Component counts keyed by entity ID and overall counts, keyed by outcome.
@@ -57,7 +58,7 @@ public function aggregate(array $sources): array {
           }
           // Reject unmapped values rather than silently omitting requests.
           if (!isset(self::DISPOSITIONS[$code])) {
-            throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $id, $record));
+            throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $source['component_label'] ?? $id, $record));
           }
           $components[$id][self::DISPOSITIONS[$code]]++;
         }

docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php

@@ -28,7 +28,8 @@ final class AppealNonExemptionDenialAggregator {
    * Counts each distinct reason once per row, then sums component counts.
    *
    * @param array $sources
-   *   Component/file pairs, each with component_id and uri keys.
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
    *
    * @return array
    *   Component and overall counts keyed by XML reason code.
@@ -67,7 +68,7 @@ public function aggregate(array $sources): array {
           foreach ($reasons as $reason) {
             // Unknown labels must not silently disappear from the report.
             if (!isset(self::REASONS[$reason])) {
-              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $id, $record, $reason));
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $source['component_label'] ?? $id, $record, $reason));
             }
             $components[$id][self::REASONS[$reason]]++;
           }

docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php

@@ -3,15 +3,16 @@
 namespace Drupal\foia_raw_data_to_report;
 
 /**
- * Streams Columns X and Y into calendar-day response time distributions.
+ * Streams Columns X and Y into working-day response time distributions.
  */
 final class AppealResponseTimeAggregator {
 
   /**
    * Calculates component and agency statistics without retaining CSV rows.
    *
    * @param array $sources
-   *   Component/file pairs, each with component_id and uri keys.
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
    * @param int $fiscal_year
    *   The validated report year.
    *
@@ -22,6 +23,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
     $timezone = new \DateTimeZone('UTC');
     $start = new \DateTimeImmutable(($fiscal_year - 1) . '-10-01', $timezone);
     $end = new \DateTimeImmutable($fiscal_year . '-09-30', $timezone);
+    $working_days = new WorkingDays();
     $histograms = [];
     foreach ($sources as $source) {
       $id = $source['component_id'];
@@ -45,7 +47,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
             $header = FALSE;
             continue;
           }
-          $context = sprintf('Component %s, CSV record %d', $id, $record);
+          $context = sprintf('Component %s, CSV record %d', $source['component_label'] ?? $id, $record);
           // Only rows with an appeal contribute to response time statistics.
           $received_text = trim($columns[23]);
           $completed = trim($columns[24]);
@@ -61,9 +63,9 @@ public function aggregate(array $sources, int $fiscal_year): array {
           if ($finish < $begin || $finish > $end) {
             throw new \RuntimeException($context . ': Response time dates must be in chronological order and within the report fiscal year.');
           }
-          // Elapsed calendar days exclude the starting day; same-day is zero.
+          // Working days exclude the starting day; same-day is zero.
           // UTC dates avoid daylight-saving changes affecting the calculation.
-          $days = (int) $begin->diff($finish)->days;
+          $days = $working_days->count($begin->format('m/d/Y'), $finish->format('m/d/Y'));
           $histograms[$id][$days] = ($histograms[$id][$days] ?? 0) + 1;
         }
         if (!feof($stream)) {

docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php

@@ -21,7 +21,8 @@ final class AppealStatisticsAggregator {
    * Returns component counters and agency totals without retaining CSV rows.
    *
    * @param array $sources
-   *   Component/file pairs, each with component_id and uri keys.
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
    * @param int $fiscal_year
    *   The validated report year.
    *
@@ -56,7 +57,7 @@ public function aggregate(array $sources, int $fiscal_year): array {
           // Rows without an appeal do not contribute to any of the counters.
           $received_text = trim($columns[23]);
           $completed = trim($columns[24]);
-          $context = sprintf('Component %s, CSV record %d', $source['component_id'], $record);
+          $context = sprintf('Component %s, CSV record %d', $source['component_label'] ?? $source['component_id'], $record);
           if ($received_text === '') {
             if ($completed !== '') {
               throw new \RuntimeException($context . ': Appeal Date Closed requires an Appeal Date Received.');

docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php

@@ -31,7 +31,8 @@ final class AppliedExemptionsAggregator {
    * Counts each distinct exemption once per row, then sums component counts.
    *
    * @param array $sources
-   *   Component/file pairs, each with component_id and uri keys.
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
    * @param int $column
    *   Zero-based CSV column: 15 for requests, 28 for appeals.
    *
@@ -77,7 +78,7 @@ public function aggregate(array $sources, int $column = 15): array {
           foreach ($codes as $code) {
             // Unknown codes must not silently disappear from the report.
             if (!isset(self::EXEMPTIONS[$code])) {
-              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column %s.', $id, $record, $code, $column_name));
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column %s.', $source['component_label'] ?? $id, $record, $code, $column_name));
             }
             $components[$id][$code]++;
           }

docroot/modules/custom/foia_raw_data_to_report/src/BacklogAggregator.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Counts requests and appeals exceeding their allowed working days.
+ */
+final class BacklogAggregator {
+
+  /**
+   * Counts requests and appeals, using fiscal year-end for open items.
+   *
+   * @param array $sources
+   *   Component/file pairs with component_id and uri keys, plus an optional
+   *   component_label for error messages.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component counts keyed by entity ID and overall counts, keyed by outcome.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $empty = ['requests' => 0, 'appeals' => 0];
+    $year_end = sprintf('09/30/%04d', $fiscal_year);
+    $working_days = new WorkingDays();
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for backlog counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $context = sprintf('Component %s, CSV %s, record %d', $source['component_label'] ?? $id, basename($source['uri']), $record);
+          // Column D supplies the threshold for both requests and appeals.
+          // Consultations leave Days Allowed blank and are excluded.
+          $days_allowed = trim($columns[3]);
+          if ($days_allowed === '') {
+            continue;
+          }
+          if (!in_array($days_allowed, ['20', '30'], TRUE)) {
+            throw new \RuntimeException($context . ': Column D: Expected Days Allowed to be 20 or 30.');
+          }
+          // Request age starts at J, falling back to I, without track filters.
+          $perfected = trim($columns[9]);
+          $request_start = $perfected !== '' ? $perfected : trim($columns[8]);
+          $intervals = [
+            'requests' => [$request_start, trim($columns[10]), 'J or I', 'K'],
+            'appeals' => [trim($columns[23]), trim($columns[24]), 'X', 'Y'],
+          ];
+          foreach ($intervals as $kind => [$start, $end, $start_column, $end_column]) {
+            // Rows without a received/perfected date do not identify an item.
+            if ($start === '') {
+              if ($end !== '') {
+                throw new \RuntimeException($context . ': Column ' . $end_column . ' requires a start date in Column ' . $start_column . '.');
+              }
+              continue;
+            }
+            $start_date = $this->calendarDate($start, $context . ', Column ' . $start_column);
+            $end = $end === '' ? $year_end : $end;
+            $end_date = $this->calendarDate($end, $context . ', Column ' . $end_column);
+            if ($end_date < $start_date) {
+              throw new \RuntimeException($context . ': End date cannot precede start date for ' . $kind . '.');
+            }
+            // Use actual receipt dates, including time before the fiscal year.
+            if ($working_days->count($start, $end) > (int) $days_allowed) {
+              $components[$id][$kind]++;
+            }
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for backlog counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+  /**
+   * Parses calendar dates defensively if a file changed after validation.
+   */
+  private function calendarDate(string $value, string $context): \DateTimeImmutable {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+    }
+    return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC'));
+  }
+
+}