From 0bfa2fa167c62772f3c43a2d520038e0c9b68cc8 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Fri, 18 Sep 2026 12:13:11 -0500
Subject: [PATCH 01/14] WS-472: XML now has Exemption 3 Statutes.

---
 .../custom/foia_raw_data_to_report/README.md  |  55 +++++-
 .../QueueWorker/RawDataToReportProcessing.php |  40 ++++-
 .../src/StatuteAggregator.php                 | 170 ++++++++++++++++++
 .../src/XmlReportBuilder.php                  | 100 ++++++++++-
 4 files changed, 357 insertions(+), 8 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/StatuteAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index f8e18e131..a6a872483 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -35,14 +35,31 @@ runners. Revisit the lease and server PHP resource limits when CSV conversion
 is implemented; a lease is not a processing timeout.
 
 `RawDataToReportProcessing::generateXmlReport()` uses `XmlReportBuilder` to
-create a **metadata-only XML stub** and attaches it to `field_request_data_xml`.
+create a **partial annual report XML document** and attaches it to `field_request_data_xml`.
 The document uses the example's `iepd:FoiaAnnualReport` root and namespaces,
 with `nc:DocumentApplicationName` set to `FOIA Annual Report Workbook`
 (application version `1.1`), `nc:DocumentCreationDate/nc:Date` set to the current
 generation date (`YYYY-MM-DD`, Drupal runtime timezone), and
 `nc:DocumentDescriptionText` set to `FOIA Annual Report`. It is well-formed XML,
-but is not yet a complete, schema-valid annual report. CSV-derived sections
+but is not yet a complete, schema-valid annual report. Additional CSV-derived sections
 will be added to this module's builder; `foia_export_xml` is not modified.
+The `nc:Organization` section uses the linked Agency taxonomy term's name and
+`field_agency_abbreviation`, with `s:id="ORG0"`. Its `nc:OrganizationSubUnit`
+children use the components from the uploaded paragraphs, in paragraph order,
+with IDs `ORG1`, `ORG2`, etc. Component abbreviations come from
+`field_agency_comp_abbreviation` on each component. Names and abbreviations are
+escaped as XML text. The report node's own abbreviation is not used.
+The Organization section is followed by `foia:DocumentFiscalYearDate`, using
+the report node's `field_foia_annual_report_yr`, also used for CSV validation.
+
+Each queue run compares attached CSV components with all `agency_component`
+nodes linked to the Agency (including unpublished components). It appends
+`Warning: no CSV has been attached for these Components: ` followed by missing
+component names. This warning does not block generation; existing validation
+errors still do. Components with an attached but invalid CSV receive validation
+errors rather than a missing-upload warning. Missing components are not added
+as XML subunits. Previous warnings are cleared with other processing messages.
+
 It uses the field's configured directory and storage
 scheme, with a unique filename, and replaces the current field reference.
 Each filename includes the generation timestamp in `YYYY-MM-DD-HH-MM-SS`
@@ -56,8 +73,8 @@ manage file permanence and usage on node save.
 Each click creates a separate job. The worker reads the latest node state when
 processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
-The CSV is validated before the XML stub is generated; CSV-derived report
-sections are not implemented yet.
+All CSVs are validated before aggregation and XML generation. Statute usage is
+aggregated; the remaining report sections are not implemented yet.
 
 ## CSV validation and messages
 
@@ -226,7 +243,7 @@ Import the exported configuration and rebuild caches before running the queue.
 The old node-level CSV field is replaced, with no migration of existing uploads.
 This follows the pre-launch assumption that existing upload content is disposable.
 The queue payload remains the parent node ID. Edits during processing are not
-locked or snapshotted. CSV conversion remains a stub, producing one metadata-only XML document
+locked or snapshotted. CSV conversion produces one partial annual report XML document
 only after every component upload passes validation.
 
 Local integration verification (creates and removes temporary fixtures):
@@ -240,3 +257,31 @@ Browser and regression verification:
 ```bash
 ddev behat -f RawDataToReport.feature
 ```
+
+## Exemption 3 statute aggregation
+
+After validation, `StatuteAggregator` streams each component CSV in a separate
+pass, one record at a time. Component identity comes from the paragraph, not
+Column A. Each distinct statute ID in Column E counts once per request row;
+agency totals sum the component counts. No request rows are retained.
+
+`StatuteAggregator::STATUTES` embeds all 77 labels from
+`ID_Statute_exemption_3.txt` verbatim, including its placeholder labels and
+punctuation. That file is not needed at runtime. Code 77 uses the trimmed
+Other Statute description in F; different descriptions produce separate statute
+entries. Other codes are grouped by their numeric ID.
+
+Distinct, nonblank Information Withheld values from G and Case Citation values
+from H are retained separately per statute and joined with newlines. G populates
+`foia:ReliedUponStatuteInformationWithheldText`; H populates
+`nc:Case/nc:CaseTitleText`. Memory grows with distinct statutes, components, and
+text values, rather than total CSV rows; retaining all unique text can still
+consume memory if every request has different text.
+
+`XmlReportBuilder` receives summaries, not CSV files. It adds
+`foia:Exemption3StatuteSection` after `foia:DocumentFiscalYearDate`, with `ES1`,
+`ES2`, etc. definitions followed by component usage associations and an `ORG0`
+agency total for each statute. Only components with nonzero usage get an
+association. No statutes produces an empty section. Names and citation text
+are escaped as XML text. Read failures abort generation before an existing XML
+file is replaced. The existing `foia_export_xml` module is unchanged.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index eb2f1561c..8c0791fa6 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -10,9 +10,11 @@
 use Drupal\file\FileRepositoryInterface;
 use Drupal\file\Plugin\Field\FieldType\FileItem;
 use Drupal\node\NodeInterface;
+use Drupal\taxonomy\TermInterface;
 use Drupal\foia_raw_data_to_report\CsvValidator;
 use Drupal\foia_raw_data_to_report\UploadAssignments;
 use Drupal\foia_raw_data_to_report\XmlReportBuilder;
+use Drupal\foia_raw_data_to_report\StatuteAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -72,6 +74,12 @@ public function processItem($data) {
     $assignment_errors = UploadAssignments::validate($node);
     $messages = [];
     $has_errors = FALSE;
+    $agency = $node->get('field_agency')->entity;
+    if (!$agency instanceof TermInterface) {
+      $messages[] = 'Select an Agency before generating a report.';
+      $has_errors = TRUE;
+    }
+    $uploaded_components = [];
     if ($node->get('field_component_uploads')->isEmpty()) {
       $messages[] = 'Add at least one Agency Component CSV upload before generating a report.';
       $has_errors = TRUE;
@@ -82,6 +90,9 @@ public function processItem($data) {
       $supported = $upload && $upload->bundle() === 'raw_data_component_upload';
       $component = $supported ? $upload->get('field_agency_component')->entity : NULL;
       $source = $supported ? $upload->get('field_request_data_csv')->entity : NULL;
+      if ($component && $source) {
+        $uploaded_components[$component->id()] = TRUE;
+      }
       $prefix = sprintf('Upload %d — Component: %s — File: %s', $delta + 1, $component?->label() ?? '(not selected)', $source?->getFilename() ?? '(not attached)');
       if (!$source) {
         $errors[] = 'No CSV file is attached. Please upload a CSV file and try again.';
@@ -96,6 +107,22 @@ public function processItem($data) {
       $has_errors = $has_errors || (bool) $errors;
       $messages[] = $prefix . "\n" . ($errors ? implode("\n", array_unique($errors)) : 'CSV validated.');
     }
+    // Compare against every linked component, regardless of publication/access.
+    // Missing uploads are a warning only; CSV validation still controls errors.
+    if ($agency instanceof TermInterface) {
+      $component_ids = $storage->getQuery()
+        ->accessCheck(FALSE)
+        ->condition('type', 'agency_component')
+        ->condition('field_agency.target_id', $agency->id())
+        ->sort('title')
+        ->sort('nid')
+        ->execute();
+      $missing = $storage->loadMultiple(array_diff($component_ids, array_keys($uploaded_components)));
+      if ($missing) {
+        $names = array_map(static fn(NodeInterface $component) => $component->label(), $missing);
+        $messages[] = 'Warning: no CSV has been attached for these Components: ' . implode(', ', $names);
+      }
+    }
     $node->set('field_messages', [
       'value' => ($has_errors ? "No new XML report was generated.\n\n" : '') . implode("\n\n", $messages),
       'format' => 'plain_text',
@@ -112,7 +139,18 @@ public function processItem($data) {
    * Builds and attaches report XML after all component CSVs pass validation.
    */
   protected function generateXmlReport(NodeInterface $node): void {
-    $xml = (new XmlReportBuilder())->build();
+    $components = [];
+    $sources = [];
+    foreach ($node->get('field_component_uploads') as $item) {
+      $component = $item->entity->get('field_agency_component')->entity;
+      $components[] = $component;
+      $sources[] = [
+        'component_id' => $component->id(),
+        'uri' => $item->entity->get('field_request_data_csv')->entity->getFileUri(),
+      ];
+    }
+    $statutes = (new StatuteAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, (int) $node->get('field_foia_annual_report_yr')->value, $statutes);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/StatuteAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/StatuteAggregator.php
new file mode 100644
index 000000000..eb2212a22
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/StatuteAggregator.php
@@ -0,0 +1,170 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated component CSVs into request counts for Exemption 3.
+ */
+final class StatuteAggregator {
+
+  /**
+   * Statute labels copied verbatim from ID_Statute_exemption_3.txt.
+   *
+   * The source file is not needed at runtime. Code 77 uses Column F instead.
+   */
+  public const STATUTES = [
+    1 => '52 U.S.C. § 30109(a)(12)(A) (formerly at 2 U.S.C. § 437g(a)(12)(A))',
+    2 => '5 U.S.C. §§ 7114(b)(4), 7132',
+    3 => '5 U.S.C. app. 4 § 107',
+    4 => '7 U.S.C. § 12',
+    5 => '7 U.S.C. § 136i-1',
+    6 => '8 U.S.C. § 1202(f)',
+    7 => '8 U.S.C. § 1158',
+    8 => '10 U.S.C. § 130(a)',
+    9 => '10 U.S.C. § 130b',
+    10 => '10 U.S.C. § 130c',
+    11 => '10 U.S.C. § 424',
+    12 => '10 U.S.C. § 1102(f)',
+    13 => '10 U.S.C. § 3309 (formerly at 10 U.S.C. § 2305(g))',
+    14 => '13 U.S.C. §§ 8(b), 9(a)',
+    15 => '13 U.S.C. § 301(g)',
+    16 => '15 U.S.C. §§ 46(f), 57b-2(f)',
+    17 => '15 U.S.C. §§ 2055(a)(2), 2055(b)(1), 2055(b)(5)',
+    18 => '15 U.S.C. § 6801',
+    19 => '15 U.S.C. § 3710a(c)',
+    20 => '15 U.S.C. § 18a(h)',
+    21 => '15 U.S.C. § 1314(g)',
+    22 => '15 U.S.C. § 4305(d)',
+    23 => '16 U.S.C. § 470hh',
+    24 => '54 U.S.C. § 100707 (formerly at 16 U.S.C. § 5937)',
+    25 => '18 U.S.C. § 701',
+    26 => '18 U.S.C. § 4208(c)',
+    27 => '18 U.S.C. § 798',
+    28 => '18 U.S.C. §§, 2510, et seq.',
+    29 => '18 U.S.C. § 3123(d)',
+    30 => '18 U.S.C. § 3153',
+    31 => '18 U.S.C. § 3509(d)',
+    32 => '18 U.S.C. § 3521(b)(1)(g)',
+    33 => '18 U.S.C. § 5038',
+    34 => '19 U.S.C. §§ 2605(h), 2605(i)',
+    35 => '19 U.S.C. § 1677f',
+    36 => '21 U.S.C. § 331(j)',
+    37 => '22 U.S.C. § 1644',
+    38 => '22 U.S.C. §§ 1461, 1461-1a',
+    39 => '22 U.S.C. § 2778(e)',
+    40 => '22 U.S.C. § 3104(c)',
+    41 => '26 U.S.C. §§ 6103, 6',
+    42 => '26 U.S.C. § 7123',
+    43 => '28 U.S.C. § 652(d)',
+    44 => '31 U.S.C. § 5311',
+    45 => '31 U.S.C. § 3730',
+    46 => '31 U.S.C. § 5319',
+    47 => '35 U.S.C. § 122',
+    48 => '38 U.S.C. § 5705',
+    49 => '38 U.S.C. § 7332',
+    50 => '39 U.S.C. § 410(c)(2)',
+    51 => '41 U.S.C. § 4702 (formerly at 41 U.S.C. § 253b(m)(1))',
+    52 => '41 U.S.C. § 2102 (formerly at 41 U.S.C. § 423(a)(1))',
+    53 => '42 U.S.C. § 300aa12(D)(4)(A)',
+    54 => '42 U.S.C. §§ 2000e-5b, 2008e-8(e)',
+    55 => '42 U.S.C. § 2000g-2b',
+    56 => '34 U.S.C. § 10231 (formerly at 42 U.S.C. § 3789g)',
+    57 => '34 U.S.C. § 12592 (formerly at 42 U.S.C. § 14132(b)(3))',
+    58 => '42 U.S.C. § 405(r)',
+    59 => '42 U.S.C. §§ 2162, 2167, 2168(a)(1)',
+    60 => '42 U.S.C. §§ 2286d(b), 2286d(h)(3)',
+    61 => '42 U.S.C. § 3610(d)',
+    62 => '45 U.S.C. § 362(d)',
+    63 => '47 U.S.C. § 605',
+    64 => '49 U.S.C. § 114(r) (formerly at 40 U.S.C. § 114(s))',
+    65 => '49 U.S.C. § 1114(c)',
+    66 => '[Statute repealed, cite to ID 64]',
+    67 => '50 U.S.C. § 3605 (formerly at 50 U.S.C. § 402 note)',
+    68 => '50 U.S.C. § 3507 (formerly at 50 U.S.C. § 403g)',
+    69 => '50 U.S.C. § 3024(i)(1) (formerly at 50 U.S.C. § 403-1(i)(1))',
+    70 => '50 U.S.C. § 3141(a) (formerly at 50 U.S.C. § 432 )',
+    71 => '50 U.S.C. § 3143 (formerly at 50 U.S.C. § 432a)',
+    72 => '50 U.S.C. § 1702(a)(1)',
+    73 => 'Pub. L. No. 115-232',
+    74 => 'Fed. R. Crim. P. 6(e)',
+    75 => '[Do not use]',
+    76 => 'Pub. L. No. 111-8, 123 Stat. 524',
+    77 => '[Statute not listed]',
+  ];
+
+  /**
+   * Aggregates one file at a time without retaining request rows.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Statute descriptions, withheld text, citations, and component counts.
+   */
+  public function aggregate(array $sources): array {
+    $statutes = [];
+    foreach ($sources as $source) {
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for statute aggregation.');
+      }
+      try {
+        $header = TRUE;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          if (trim($columns[4]) === '') {
+            continue;
+          }
+          // One request counts once per statute, even if its ID is repeated.
+          $ids = array_unique(array_map('intval', explode(',', $columns[4])));
+          foreach ($ids as $id) {
+            if (!isset(self::STATUTES[$id])) {
+              throw new \RuntimeException('CSV contains an unknown statute ID after validation.');
+            }
+            // Other statutes must remain distinct rather than sharing ID 77's
+            // generic label. Identical trimmed descriptions share one entry.
+            $description = $id === 77 ? trim($columns[5]) : self::STATUTES[$id];
+            $key = $id === 77 ? '77:' . $description : (string) $id;
+            if (!isset($statutes[$key])) {
+              $statutes[$key] = [
+                'description' => $description,
+                'information_withheld' => [],
+                'citations' => [],
+                'counts' => [],
+              ];
+            }
+            $component_id = $source['component_id'];
+            $statutes[$key]['counts'][$component_id] = ($statutes[$key]['counts'][$component_id] ?? 0) + 1;
+            // Preserve distinct G and H values without retaining their rows.
+            $withheld = trim($columns[6]);
+            if ($withheld !== '') {
+              $statutes[$key]['information_withheld'][$withheld] = $withheld;
+            }
+            $citation = trim($columns[7]);
+            if ($citation !== '') {
+              $statutes[$key]['citations'][$citation] = $citation;
+            }
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for statute aggregation.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    ksort($statutes, SORT_NATURAL);
+    return $statutes;
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 566fc9dbb..a3149aa07 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -2,6 +2,8 @@
 
 namespace Drupal\foia_raw_data_to_report;
 
+use Drupal\taxonomy\TermInterface;
+
 /**
  * Builds annual report XML independently of the node-based foia_export_xml.
  */
@@ -21,9 +23,21 @@ final class XmlReportBuilder {
   ];
 
   /**
-   * Builds the metadata stub; CSV-derived report sections will be added later.
+   * Builds report metadata, organizations, and aggregated statute usage.
+   *
+   * @param \Drupal\taxonomy\TermInterface $agency
+   *   The report's linked Agency term, supplying its name and abbreviation.
+   * @param \Drupal\node\NodeInterface[] $components
+   *   Components from the validated upload paragraphs, in paragraph order.
+   * @param int $fiscal_year
+   *   The report node's Year, already checked during CSV validation.
+   * @param array $statutes
+   *   Statute summaries returned by StatuteAggregator.
+   *
+   * @return string
+   *   The serialized report XML.
    */
-  public function build(): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -43,6 +57,27 @@ public function build(): string {
     $root->appendChild($creation_date);
     $root->appendChild($document->createElementNS(self::NAMESPACES['nc'], 'nc:DocumentDescriptionText', 'FOIA Annual Report'));
 
+    // Use the Agency term, not the abbreviation on the raw data report node.
+    $organization = $document->createElementNS(self::NAMESPACES['nc'], 'nc:Organization');
+    $organization->setAttributeNS(self::NAMESPACES['s'], 's:id', 'ORG0');
+    $root->appendChild($organization);
+    $this->addOrganizationText($document, $organization, (string) $agency->get('field_agency_abbreviation')->value, $agency->label());
+
+    // Only uploaded components are subunits, with IDs matching the example.
+    $component_map = [];
+    foreach (array_values($components) as $delta => $component) {
+      $component_map[$component->id()] = 'ORG' . ($delta + 1);
+      $subunit = $document->createElementNS(self::NAMESPACES['nc'], 'nc:OrganizationSubUnit');
+      $subunit->setAttributeNS(self::NAMESPACES['s'], 's:id', 'ORG' . ($delta + 1));
+      $organization->appendChild($subunit);
+      $this->addOrganizationText($document, $subunit, (string) $component->get('field_agency_comp_abbreviation')->value, $component->label());
+    }
+
+    // Follow the Organization section with the report's selected fiscal year.
+    $root->appendChild($document->createElementNS(self::NAMESPACES['foia'], 'foia:DocumentFiscalYearDate', (string) $fiscal_year));
+
+    $this->addStatutes($document, $root, $statutes, $component_map);
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -50,4 +85,65 @@ public function build(): string {
     return $xml;
   }
 
+  /**
+   * Adds statute definitions, component counts, and agency-wide totals.
+   */
+  private function addStatutes(\DOMDocument $document, \DOMElement $root, array $statutes, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'Exemption3StatuteSection');
+    $statutes = array_values($statutes);
+    foreach ($statutes as $delta => $statute) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'ReliedUponStatute');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'ES' . ($delta + 1));
+      $this->addTextElement($document, $entry, 'j', 'StatuteDescriptionText', $statute['description']);
+      // Keep distinct G and H values, separated by newlines across requests.
+      $citations = implode("\n", $statute['citations']);
+      $withheld = implode("\n", $statute['information_withheld']);
+      $this->addTextElement($document, $entry, 'foia', 'ReliedUponStatuteInformationWithheldText', $withheld);
+      $case = $this->addTextElement($document, $entry, 'nc', 'Case');
+      // Match the example report when no case citation was supplied.
+      $this->addTextElement($document, $case, 'nc', 'CaseTitleText', $citations === '' ? 'N/A' : $citations);
+    }
+    // Definitions precede associations, matching the existing annual exporter.
+    foreach ($statutes as $delta => $statute) {
+      $counts = [];
+      foreach ($component_map as $component_id => $organization_id) {
+        if (isset($statute['counts'][$component_id])) {
+          $counts[$organization_id] = $statute['counts'][$component_id];
+        }
+      }
+      $counts['ORG0'] = array_sum($counts);
+      foreach ($counts as $organization_id => $quantity) {
+        $association = $this->addTextElement($document, $section, 'foia', 'ReliedUponStatuteOrganizationAssociation');
+        $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+        $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'ES' . ($delta + 1));
+        $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+        $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+        $this->addTextElement($document, $association, 'foia', 'ReliedUponStatuteQuantity', (string) $quantity);
+      }
+    }
+  }
+
+  /**
+   * Appends a namespaced element and safely escapes optional text.
+   */
+  private function addTextElement(\DOMDocument $document, \DOMElement $parent, string $prefix, string $name, string $text = ''): \DOMElement {
+    $element = $document->createElementNS(self::NAMESPACES[$prefix], $prefix . ':' . $name);
+    if ($text !== '') {
+      $element->appendChild($document->createTextNode($text));
+    }
+    $parent->appendChild($element);
+    return $element;
+  }
+
+  /**
+   * Adds organization text safely, including names containing XML characters.
+   */
+  private function addOrganizationText(\DOMDocument $document, \DOMElement $parent, string $abbreviation, string $name): void {
+    foreach (['OrganizationAbbreviationText' => $abbreviation, 'OrganizationName' => $name] as $element => $value) {
+      $child = $document->createElementNS(self::NAMESPACES['nc'], 'nc:' . $element);
+      $child->appendChild($document->createTextNode($value));
+      $parent->appendChild($child);
+    }
+  }
+
 }

From 9d1f1f48daf82462fed19092147ddd463264e160 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Fri, 18 Sep 2026 17:54:00 -0500
Subject: [PATCH 02/14] WS-472: Added Request Statistics section to XML.

---
 .../custom/foia_raw_data_to_report/README.md  |  30 ++++-
 .../QueueWorker/RawDataToReportProcessing.php |   5 +-
 .../src/RequestStatisticsAggregator.php       | 124 ++++++++++++++++++
 .../src/XmlReportBuilder.php                  |  44 ++++++-
 4 files changed, 198 insertions(+), 5 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index a6a872483..ad583a83d 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -73,8 +73,9 @@ manage file permanence and usage on node save.
 Each click creates a separate job. The worker reads the latest node state when
 processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
-All CSVs are validated before aggregation and XML generation. Statute usage is
-aggregated; the remaining report sections are not implemented yet.
+All CSVs are validated before aggregation and XML generation. Statute usage and
+processed request statistics are aggregated; the remaining report sections are
+not implemented yet.
 
 ## CSV validation and messages
 
@@ -285,3 +286,28 @@ agency total for each statute. Only components with nonzero usage get an
 association. No statutes produces an empty section. Names and citation text
 are escaped as XML text. Read failures abort generation before an existing XML
 file is replaced. The existing `foia_export_xml` module is unchanged.
+
+## Processed request statistics
+
+`RequestStatisticsAggregator` makes a separate streaming pass after validation,
+retaining only four counters per component. Every data row counts, including
+consultations and rows without statute codes. Headers and blank records are
+skipped using the same rules as validation.
+
+- Pending at start: I is before October 1 of the previous year.
+- Received: I is within the fiscal year, including both boundaries.
+- Processed: K is within the fiscal year, including both boundaries.
+- Pending at end: K is blank (including whitespace-only cells).
+
+Agency totals sum each counter across components. Both component and overall
+counts must satisfy `pending start + received - processed = pending end`.
+A mismatch or read failure raises a processing exception before the existing
+XML is replaced. Normal CSV validation already enforces the date constraints
+that make this equation hold. Header-only uploads produce four zero counts.
+
+The builder adds `foia:ProcessedRequestSection` after the statute section.
+Each uploaded component gets `PS1`, `PS2`, etc., and the agency total gets
+`PS0`. All four quantities are emitted, including zeroes. Corresponding
+`foia:ProcessingStatisticsOrganizationAssociation` elements reference those
+statistics using `foia:ComponentDataReference` and link to `ORG1`, `ORG2`, etc.
+or agency `ORG0` using `nc:OrganizationReference`.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 8c0791fa6..e45c6f5dd 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -15,6 +15,7 @@
 use Drupal\foia_raw_data_to_report\UploadAssignments;
 use Drupal\foia_raw_data_to_report\XmlReportBuilder;
 use Drupal\foia_raw_data_to_report\StatuteAggregator;
+use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -150,7 +151,9 @@ protected function generateXmlReport(NodeInterface $node): void {
       ];
     }
     $statutes = (new StatuteAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, (int) $node->get('field_foia_annual_report_yr')->value, $statutes);
+    $fiscal_year = (int) $node->get('field_foia_annual_report_yr')->value;
+    $request_statistics = (new RequestStatisticsAggregator())->aggregate($sources, $fiscal_year);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php
new file mode 100644
index 000000000..af60bbd87
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php
@@ -0,0 +1,124 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated CSVs into fiscal-year request statistics.
+ */
+final class RequestStatisticsAggregator {
+
+  /**
+   * Counters initialized even for a component with no request rows.
+   */
+  private const EMPTY_COUNTS = [
+    'pending_start' => 0,
+    'received' => 0,
+    'processed' => 0,
+    'pending_end' => 0,
+  ];
+
+  /**
+   * Returns component counters and agency totals without retaining CSV rows.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component counts keyed by entity ID, plus overall totals.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $start = ($fiscal_year - 1) * 10000 + 1001;
+    $end = $fiscal_year * 10000 + 930;
+    $components = [];
+    foreach ($sources as $source) {
+      $counts = self::EMPTY_COUNTS;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for request statistics.');
+      }
+      try {
+        $header = TRUE;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          // Every data row counts, including consultations and rows without E.
+          $received = $this->calendarDate(trim($columns[8]));
+          if ($received < $start) {
+            $counts['pending_start']++;
+          }
+          elseif ($received <= $end) {
+            $counts['received']++;
+          }
+          else {
+            throw new \RuntimeException('CSV received date is after the report fiscal year.');
+          }
+
+          // Count end-of-year pending requests directly from blank Column K.
+          $completed = trim($columns[10]);
+          if ($completed === '') {
+            $counts['pending_end']++;
+          }
+          else {
+            $completed_date = $this->calendarDate($completed);
+            if ($completed_date >= $start && $completed_date <= $end) {
+              $counts['processed']++;
+            }
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for request statistics.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+      // The counts must reconcile before any XML file can be replaced.
+      $this->checkBalance($counts, (string) $source['component_id']);
+      $id = $source['component_id'];
+      $components[$id] ??= self::EMPTY_COUNTS;
+      foreach ($counts as $key => $value) {
+        $components[$id][$key] += $value;
+      }
+    }
+
+    $overall = self::EMPTY_COUNTS;
+    foreach ($components as $counts) {
+      foreach ($counts as $key => $value) {
+        $overall[$key] += $value;
+      }
+    }
+    $this->checkBalance($overall, 'agency overall');
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+  /**
+   * Converts a validated date to YYYYMMDD, guarding against changed CSV data.
+   */
+  private function calendarDate(string $value): int {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException('CSV contains an invalid date after validation.');
+    }
+    return (int) $parts[3] * 10000 + (int) $parts[1] * 100 + (int) $parts[2];
+  }
+
+  /**
+   * Checks pending start + received - processed = pending end.
+   */
+  private function checkBalance(array $counts, string $label): void {
+    if ($counts['pending_start'] + $counts['received'] - $counts['processed'] !== $counts['pending_end']) {
+      throw new \RuntimeException('Request statistics do not balance for ' . $label . '.');
+    }
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index a3149aa07..0b49f46ba 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -23,7 +23,7 @@ final class XmlReportBuilder {
   ];
 
   /**
-   * Builds report metadata, organizations, and aggregated statute usage.
+   * Builds metadata, organizations, statute usage, and request statistics.
    *
    * @param \Drupal\taxonomy\TermInterface $agency
    *   The report's linked Agency term, supplying its name and abbreviation.
@@ -33,11 +33,13 @@ final class XmlReportBuilder {
    *   The report node's Year, already checked during CSV validation.
    * @param array $statutes
    *   Statute summaries returned by StatuteAggregator.
+   * @param array $request_statistics
+   *   Component and overall summaries from RequestStatisticsAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -77,6 +79,9 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
     $root->appendChild($document->createElementNS(self::NAMESPACES['foia'], 'foia:DocumentFiscalYearDate', (string) $fiscal_year));
 
     $this->addStatutes($document, $root, $statutes, $component_map);
+    if ($request_statistics !== []) {
+      $this->addRequestStatistics($document, $root, $request_statistics, $component_map);
+    }
 
     $xml = $document->saveXML();
     if ($xml === FALSE) {
@@ -123,6 +128,41 @@ private function addStatutes(\DOMDocument $document, \DOMElement $root, array $s
     }
   }
 
+  /**
+   * Adds request counters and references to component/agency organizations.
+   */
+  private function addRequestStatistics(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'ProcessedRequestSection');
+    $fields = [
+      'pending_start' => 'ProcessingStatisticsPendingAtStartQuantity',
+      'received' => 'ProcessingStatisticsReceivedQuantity',
+      'processed' => 'ProcessingStatisticsProcessedQuantity',
+      'pending_end' => 'ProcessingStatisticsPendingAtEndQuantity',
+    ];
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $statistics['components'][$component_id];
+    }
+    $organizations['ORG0'] = $statistics['overall'];
+
+    // Keep matching suffixes: PS1 refers to ORG1, and PS0 to the agency ORG0.
+    foreach ($organizations as $organization_id => $counts) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'ProcessingStatistics');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'PS' . substr($organization_id, 3));
+      foreach ($fields as $key => $name) {
+        $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]);
+      }
+    }
+    // Statistics precede associations, matching the example and exporter.
+    foreach ($organizations as $organization_id => $counts) {
+      $association = $this->addTextElement($document, $section, 'foia', 'ProcessingStatisticsOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'PS' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From b1354f6a82a02e4c6631bf17431b59525d91a630 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sat, 19 Sep 2026 17:05:36 -0500
Subject: [PATCH 03/14] WS-472: Added RequestDispositionSection to XML.

---
 .../custom/foia_raw_data_to_report/README.md  |  27 ++++-
 .../src/DispositionAggregator.php             | 104 ++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |   4 +-
 .../src/XmlReportBuilder.php                  |  48 +++++++-
 4 files changed, 179 insertions(+), 4 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index ad583a83d..de133921c 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -74,7 +74,7 @@ Each click creates a separate job. The worker reads the latest node state when
 processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
 All CSVs are validated before aggregation and XML generation. Statute usage and
-processed request statistics are aggregated; the remaining report sections are
+processed request and disposition statistics are aggregated; other sections are
 not implemented yet.
 
 ## CSV validation and messages
@@ -311,3 +311,28 @@ Each uploaded component gets `PS1`, `PS2`, etc., and the agency total gets
 `foia:ProcessingStatisticsOrganizationAssociation` elements reference those
 statistics using `foia:ComponentDataReference` and link to `ORG1`, `ORG2`, etc.
 or agency `ORG0` using `nc:OrganizationReference`.
+
+## Request disposition statistics
+
+`DispositionAggregator` streams each validated CSV in a separate pass, keeping
+12 counters per component and summing them for the agency overall. Each data
+row with a nonblank Column N counts once for that code. Blank dispositions,
+headers, and blank records do not contribute. Date eligibility is enforced by
+existing CSV validation (a disposition requires a completion date in the report
+fiscal year). Header-only components retain all twelve zero counts.
+
+`DispositionAggregator::DISPOSITIONS` embeds the twelve labels from
+`ID_Disposition.txt`; the file is not needed at runtime. Codes 1–3 map to the
+FullGrant, PartialGrant, and FullExemptionDenial quantity elements. Codes 4–12
+map to `NonExemptionDenial` entries using the XML reason codes from the example
+and existing exporter: `NoRecords`, `Referred`, `Withdrawn`, `FeeRelated`,
+`NotDescribed`, `ImproperRequest`, `NotAgency`, `Duplicate`, and `Other`.
+These XML codes differ from the mapping file's human-readable labels.
+
+`foia:RequestDispositionSection` follows the processed request section. Each
+component receives `RD1`, `RD2`, etc., and the overall agency receives `RD0`.
+All twelve quantities, including zeroes, and their `RequestDispositionTotalQuantity`
+sum are output. `RequestDispositionOrganizationAssociation` links each RD ID
+through `ComponentDataReference` to its corresponding ORG ID through
+`OrganizationReference`. Unknown nonblank codes or read failures raise a
+processing exception before any existing XML is replaced.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php
new file mode 100644
index 000000000..0046c18bc
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/DispositionAggregator.php
@@ -0,0 +1,104 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated component CSVs into disposition counts from Column N.
+ */
+final class DispositionAggregator {
+
+  /**
+   * From Appendix B of "Agency Spreadsheet Guidance".
+   */
+  public const DISPOSITIONS = [
+    1 => 'Full Grant',
+    2 => 'Partial Grant / Partial Denial',
+    3 => 'Full Denial Based on Exemptions',
+    4 => 'No Records',
+    5 => 'All Records Referred to Another Component',
+    6 => 'Request Withdrawn',
+    7 => 'Fee-Related Reasons',
+    8 => 'Records Not Reasonably Described',
+    9 => 'Improper FOIA Request for Other Reason',
+    10 => 'Not Agency Record',
+    11 => 'Duplicate Request',
+    12 => 'Other',
+  ];
+
+  /**
+   * XML reason codes matching the example report and foia_export_xml.
+   */
+  public const NON_EXEMPTION_REASONS = [
+    4 => 'NoRecords',
+    5 => 'Referred',
+    6 => 'Withdrawn',
+    7 => 'FeeRelated',
+    8 => 'NotDescribed',
+    9 => 'ImproperRequest',
+    10 => 'NotAgency',
+    11 => 'Duplicate',
+    12 => 'Other',
+  ];
+
+  /**
+   * Counts each nonblank disposition once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component counts keyed by entity ID and overall counts, keyed by code.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_keys(self::DISPOSITIONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for disposition counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $code = trim($columns[13]);
+          if ($code === '') {
+            continue;
+          }
+          // Reject unmapped values rather than silently omitting requests.
+          if (!preg_match('/^(?:[1-9]|1[0-2])$/', $code)) {
+            throw new \RuntimeException(sprintf('Component %s, CSV record %d: Column N must contain a disposition code from 1 through 12.', $id, $record));
+          }
+          $components[$id][(int) $code]++;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for disposition counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index e45c6f5dd..c044d50d6 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -16,6 +16,7 @@
 use Drupal\foia_raw_data_to_report\XmlReportBuilder;
 use Drupal\foia_raw_data_to_report\StatuteAggregator;
 use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
+use Drupal\foia_raw_data_to_report\DispositionAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -153,7 +154,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $statutes = (new StatuteAggregator())->aggregate($sources);
     $fiscal_year = (int) $node->get('field_foia_annual_report_yr')->value;
     $request_statistics = (new RequestStatisticsAggregator())->aggregate($sources, $fiscal_year);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics);
+    $dispositions = (new DispositionAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 0b49f46ba..1c0c97008 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -23,7 +23,7 @@ final class XmlReportBuilder {
   ];
 
   /**
-   * Builds metadata, organizations, statute usage, and request statistics.
+   * Builds metadata, organizations, statute usage, and request summaries.
    *
    * @param \Drupal\taxonomy\TermInterface $agency
    *   The report's linked Agency term, supplying its name and abbreviation.
@@ -35,11 +35,13 @@ final class XmlReportBuilder {
    *   Statute summaries returned by StatuteAggregator.
    * @param array $request_statistics
    *   Component and overall summaries from RequestStatisticsAggregator.
+   * @param array $dispositions
+   *   Component and overall summaries from DispositionAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -82,6 +84,9 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
     if ($request_statistics !== []) {
       $this->addRequestStatistics($document, $root, $request_statistics, $component_map);
     }
+    if ($dispositions !== []) {
+      $this->addDispositions($document, $root, $dispositions, $component_map);
+    }
 
     $xml = $document->saveXML();
     if ($xml === FALSE) {
@@ -163,6 +168,45 @@ private function addRequestStatistics(\DOMDocument $document, \DOMElement $root,
     }
   }
 
+  /**
+   * Adds disposition counts, totals, and references to their organizations.
+   */
+  private function addDispositions(\DOMDocument $document, \DOMElement $root, array $dispositions, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'RequestDispositionSection');
+    $fields = [
+      1 => 'RequestDispositionFullGrantQuantity',
+      2 => 'RequestDispositionPartialGrantQuantity',
+      3 => 'RequestDispositionFullExemptionDenialQuantity',
+    ];
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $dispositions['components'][$component_id];
+    }
+    $organizations['ORG0'] = $dispositions['overall'];
+    foreach ($organizations as $organization_id => $counts) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'RequestDisposition');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'RD' . substr($organization_id, 3));
+      foreach ($fields as $code => $name) {
+        $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$code]);
+      }
+      // These values are XML reason codes, not the human-readable CSV labels.
+      foreach (DispositionAggregator::NON_EXEMPTION_REASONS as $code => $reason) {
+        $denial = $this->addTextElement($document, $entry, 'foia', 'NonExemptionDenial');
+        $this->addTextElement($document, $denial, 'foia', 'NonExemptionDenialReasonCode', $reason);
+        $this->addTextElement($document, $denial, 'foia', 'NonExemptionDenialQuantity', (string) $counts[$code]);
+      }
+      $this->addTextElement($document, $entry, 'foia', 'RequestDispositionTotalQuantity', (string) array_sum($counts));
+    }
+    // Keep RD/ORG suffixes aligned, including RD0 for the overall agency.
+    foreach ($organizations as $organization_id => $counts) {
+      $association = $this->addTextElement($document, $section, 'foia', 'RequestDispositionOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'RD' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From e2daf9b46c5e031fa44ef91df5dc60e6cae15650 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sat, 19 Sep 2026 17:40:48 -0500
Subject: [PATCH 04/14] WS-472: Added OtherDenialReason section of XML.

---
 .../custom/foia_raw_data_to_report/README.md  | 26 ++++++-
 .../src/OtherDenialReasonAggregator.php       | 74 +++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |  4 +-
 .../src/XmlReportBuilder.php                  | 38 +++++++++-
 4 files changed, 137 insertions(+), 5 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index de133921c..a404e4f2c 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -73,9 +73,9 @@ manage file permanence and usage on node save.
 Each click creates a separate job. The worker reads the latest node state when
 processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
-All CSVs are validated before aggregation and XML generation. Statute usage and
-processed request and disposition statistics are aggregated; other sections are
-not implemented yet.
+All CSVs are validated before aggregation and XML generation. Statute usage,
+processed request statistics, dispositions, and other denial reasons are
+aggregated; the remaining sections are not implemented yet.
 
 ## CSV validation and messages
 
@@ -336,3 +336,23 @@ sum are output. `RequestDispositionOrganizationAssociation` links each RD ID
 through `ComponentDataReference` to its corresponding ORG ID through
 `OrganizationReference`. Unknown nonblank codes or read failures raise a
 processing exception before any existing XML is replaced.
+
+## Other denial reasons
+
+`OtherDenialReasonAggregator` streams Column O, counting each nonblank value
+once per request. It trims surrounding whitespace but preserves case,
+punctuation, and internal whitespace, including multiline text. Numeric-looking
+reasons remain text. Distinct reasons are sorted by text for stable output.
+Agency counts sum each reason's usage across components. Memory grows with the
+number and length of distinct reasons, not with repeated request rows.
+
+`foia:RequestDenialOtherReasonSection` follows the disposition section. Each
+component has a `ComponentOtherDenialReason` (`CODR1`, `CODR2`, etc.) containing
+reason descriptions and usage counts. The agency entry (`CODR0`) includes the
+summed count for every distinct reason. Each entry has a
+`ComponentOtherDenialReasonQuantity` equal to the sum of its usage counts,
+matching the example (two reasons used 1 and 3 times give a total of 4).
+Components with no reasons have zero totals and no reason entries.
+`OtherDenialReasonOrganizationAssociation` links each CODR entry to its ORG
+organization. Text is escaped safely when written to XML. Blank records and
+headers are skipped; read failures abort generation before XML replacement.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php
new file mode 100644
index 000000000..108503e70
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php
@@ -0,0 +1,74 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated CSVs into counts of the free-text reasons in Column O.
+ */
+final class OtherDenialReasonAggregator {
+
+  /**
+   * Counts each nonblank reason once per row, then sums across components.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component and overall reason maps, with description and quantity entries.
+   */
+  public function aggregate(array $sources): array {
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= [];
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for other denial reasons.');
+      }
+      try {
+        $header = TRUE;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          // Match exact text after trimming only surrounding whitespace.
+          // Keep case, punctuation, and internal whitespace distinctions.
+          $reason = trim($columns[14]);
+          if ($reason === '') {
+            continue;
+          }
+          // A prefix preserves numeric-looking reasons as text keys.
+          $key = 'reason:' . $reason;
+          $components[$id][$key] ??= ['description' => $reason, 'quantity' => 0];
+          $components[$id][$key]['quantity']++;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for other denial reasons.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+
+    $overall = [];
+    foreach ($components as $id => $reasons) {
+      // Stable text order makes repeated report generation easy to compare.
+      ksort($components[$id], SORT_STRING);
+      foreach ($reasons as $key => $reason) {
+        $overall[$key] ??= ['description' => $reason['description'], 'quantity' => 0];
+        $overall[$key]['quantity'] += $reason['quantity'];
+      }
+    }
+    ksort($overall, SORT_STRING);
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index c044d50d6..88c0d2687 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -17,6 +17,7 @@
 use Drupal\foia_raw_data_to_report\StatuteAggregator;
 use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
 use Drupal\foia_raw_data_to_report\DispositionAggregator;
+use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -155,7 +156,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $fiscal_year = (int) $node->get('field_foia_annual_report_yr')->value;
     $request_statistics = (new RequestStatisticsAggregator())->aggregate($sources, $fiscal_year);
     $dispositions = (new DispositionAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions);
+    $other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 1c0c97008..81f33d4dd 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -37,11 +37,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from RequestStatisticsAggregator.
    * @param array $dispositions
    *   Component and overall summaries from DispositionAggregator.
+   * @param array $other_reasons
+   *   Component and overall summaries from OtherDenialReasonAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -87,6 +89,9 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
     if ($dispositions !== []) {
       $this->addDispositions($document, $root, $dispositions, $component_map);
     }
+    if ($other_reasons !== []) {
+      $this->addOtherDenialReasons($document, $root, $other_reasons, $component_map);
+    }
 
     $xml = $document->saveXML();
     if ($xml === FALSE) {
@@ -207,6 +212,37 @@ private function addDispositions(\DOMDocument $document, \DOMElement $root, arra
     }
   }
 
+  /**
+   * Adds free-text reason counts, totals, and organization references.
+   */
+  private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root, array $other_reasons, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'RequestDenialOtherReasonSection');
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $other_reasons['components'][$component_id];
+    }
+    $organizations['ORG0'] = $other_reasons['overall'];
+    foreach ($organizations as $organization_id => $reasons) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'ComponentOtherDenialReason');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'CODR' . substr($organization_id, 3));
+      foreach ($reasons as $reason) {
+        $item = $this->addTextElement($document, $entry, 'foia', 'OtherDenialReason');
+        $this->addTextElement($document, $item, 'foia', 'OtherDenialReasonDescriptionText', $reason['description']);
+        $this->addTextElement($document, $item, 'foia', 'OtherDenialReasonQuantity', (string) $reason['quantity']);
+      }
+      // The example sums usages, not the number of distinct reason texts.
+      $total = array_sum(array_column($reasons, 'quantity'));
+      $this->addTextElement($document, $entry, 'foia', 'ComponentOtherDenialReasonQuantity', (string) $total);
+    }
+    foreach ($organizations as $organization_id => $reasons) {
+      $association = $this->addTextElement($document, $section, 'foia', 'OtherDenialReasonOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'CODR' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From de0dcba2fcb1e13e2294339f107fb29942649559 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 00:22:07 -0500
Subject: [PATCH 05/14] WS-472: Added
 RequestDispositionAppliedExemptionsSection to the XML output.

---
 .../custom/foia_raw_data_to_report/README.md  | 25 ++++-
 .../src/AppliedExemptionsAggregator.php       | 96 +++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |  4 +-
 .../src/XmlReportBuilder.php                  | 39 +++++++-
 4 files changed, 160 insertions(+), 4 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index a404e4f2c..b281277dc 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -74,8 +74,8 @@ Each click creates a separate job. The worker reads the latest node state when
 processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
 All CSVs are validated before aggregation and XML generation. Statute usage,
-processed request statistics, dispositions, and other denial reasons are
-aggregated; the remaining sections are not implemented yet.
+processed request statistics, dispositions, other denial reasons, and applied
+exemptions are aggregated; the remaining sections are not implemented yet.
 
 ## CSV validation and messages
 
@@ -356,3 +356,24 @@ Components with no reasons have zero totals and no reason entries.
 `OtherDenialReasonOrganizationAssociation` links each CODR entry to its ORG
 organization. Text is escaped safely when written to XML. Blank records and
 headers are skipped; read failures abort generation before XML replacement.
+
+## Applied exemptions
+
+`AppliedExemptionsAggregator` streams Column P in a separate pass, splits each
+nonblank cell on commas, trims each code, and normalizes letters to uppercase.
+Each distinct exemption counts once per request, so `5,7a,7A` increments 5 and
+7(A) once each. Blank cells, headers, and blank records do not contribute.
+Fourteen counters per component cover 1–6, 7(A)–7(F), 8, and 9; agency counters
+sum the component values. Unknown codes raise a processing exception before
+existing XML is replaced rather than being silently omitted.
+
+`foia:RequestDispositionAppliedExemptionsSection` follows the Other denial
+reason section. Each `ComponentAppliedExemptions` has `RDE1`, `RDE2`, etc., or
+agency `RDE0`. `AppliedExemption` elements contain the example's labels such
+as `Ex. 5` and `Ex. 7(A)` plus `AppliedExemptionQuantity`. Zero-count exemptions
+are omitted; components with no exemptions retain their empty container and
+organization association.
+`ComponentAppliedExemptionsOrganizationAssociation` links each RDE entry to
+its corresponding ORG entry. No total across exemptions is emitted, because a
+single request may use several exemptions. Memory retains only counters, not
+request rows. Existing CSV validation continues to run before aggregation.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
new file mode 100644
index 000000000..11103ce12
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
@@ -0,0 +1,96 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated component CSVs into applied exemption counts from Column P.
+ */
+final class AppliedExemptionsAggregator {
+
+  /**
+   * CSV exemption codes mapped to the annual report's XML labels.
+   */
+  public const EXEMPTIONS = [
+    '1' => 'Ex. 1',
+    '2' => 'Ex. 2',
+    '3' => 'Ex. 3',
+    '4' => 'Ex. 4',
+    '5' => 'Ex. 5',
+    '6' => 'Ex. 6',
+    '7A' => 'Ex. 7(A)',
+    '7B' => 'Ex. 7(B)',
+    '7C' => 'Ex. 7(C)',
+    '7D' => 'Ex. 7(D)',
+    '7E' => 'Ex. 7(E)',
+    '7F' => 'Ex. 7(F)',
+    '8' => 'Ex. 8',
+    '9' => 'Ex. 9',
+  ];
+
+  /**
+   * Counts each distinct exemption once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component counts keyed by entity ID and overall counts, keyed by code.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_keys(self::EXEMPTIONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for applied exemption counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $value = trim($columns[15]);
+          if ($value === '') {
+            continue;
+          }
+          // Split the CSV cell, ignore surrounding whitespace and normalize
+          // letter case. Repeated codes count once for this request.
+          $codes = array_unique(array_map(static fn(string $code) => strtoupper(trim($code)), explode(',', $value)));
+          foreach ($codes as $code) {
+            // Unknown codes must not silently disappear from the report.
+            if (!isset(self::EXEMPTIONS[$code])) {
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column P.', $id, $record, $code));
+            }
+            $components[$id][$code]++;
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for applied exemption counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 88c0d2687..148c2f7be 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -18,6 +18,7 @@
 use Drupal\foia_raw_data_to_report\RequestStatisticsAggregator;
 use Drupal\foia_raw_data_to_report\DispositionAggregator;
 use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
+use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -157,7 +158,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $request_statistics = (new RequestStatisticsAggregator())->aggregate($sources, $fiscal_year);
     $dispositions = (new DispositionAggregator())->aggregate($sources);
     $other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons);
+    $applied_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 81f33d4dd..de760a15d 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -39,11 +39,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from DispositionAggregator.
    * @param array $other_reasons
    *   Component and overall summaries from OtherDenialReasonAggregator.
+   * @param array $applied_exemptions
+   *   Component and overall summaries from AppliedExemptionsAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -92,6 +94,9 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
     if ($other_reasons !== []) {
       $this->addOtherDenialReasons($document, $root, $other_reasons, $component_map);
     }
+    if ($applied_exemptions !== []) {
+      $this->addAppliedExemptions($document, $root, $applied_exemptions, $component_map);
+    }
 
     $xml = $document->saveXML();
     if ($xml === FALSE) {
@@ -243,6 +248,38 @@ private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root
     }
   }
 
+  /**
+   * Adds per-exemption counts and organization references, without a total.
+   */
+  private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root, array $exemptions, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'RequestDispositionAppliedExemptionsSection');
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $exemptions['components'][$component_id];
+    }
+    $organizations['ORG0'] = $exemptions['overall'];
+    foreach ($organizations as $organization_id => $counts) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'ComponentAppliedExemptions');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'RDE' . substr($organization_id, 3));
+      foreach (AppliedExemptionsAggregator::EXEMPTIONS as $code => $label) {
+        // Omit unused exemptions for components and the agency overall.
+        if ($counts[$code] === 0) {
+          continue;
+        }
+        $item = $this->addTextElement($document, $entry, 'foia', 'AppliedExemption');
+        $this->addTextElement($document, $item, 'foia', 'AppliedExemptionCode', $label);
+        $this->addTextElement($document, $item, 'foia', 'AppliedExemptionQuantity', (string) $counts[$code]);
+      }
+    }
+    foreach ($organizations as $organization_id => $counts) {
+      $association = $this->addTextElement($document, $section, 'foia', 'ComponentAppliedExemptionsOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'RDE' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From ad376e2a81ead7f5aa7e211d357b036b5d12e457 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 15:11:59 -0500
Subject: [PATCH 06/14] WS-472: Added Appeal Statistics section to XML.

---
 .../custom/foia_raw_data_to_report/README.md  |  29 +++-
 .../src/AppealStatisticsAggregator.php        | 137 ++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |   4 +-
 .../src/XmlReportBuilder.php                  |  22 ++-
 4 files changed, 182 insertions(+), 10 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index b281277dc..f15b14389 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -75,7 +75,8 @@ processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
 All CSVs are validated before aggregation and XML generation. Statute usage,
 processed request statistics, dispositions, other denial reasons, and applied
-exemptions are aggregated; the remaining sections are not implemented yet.
+exemptions and appeal processing statistics are aggregated; the remaining
+sections are not implemented yet.
 
 ## CSV validation and messages
 
@@ -377,3 +378,29 @@ organization association.
 its corresponding ORG entry. No total across exemptions is emitted, because a
 single request may use several exemptions. Memory retains only counters, not
 request rows. Existing CSV validation continues to run before aggregation.
+
+## Processed appeal statistics
+
+`AppealStatisticsAggregator` streams X (Appeal Date Received) and Y (Appeal
+Date Closed), retaining four counters per component and summing agency totals.
+Rows with both dates blank do not contribute. Dates use the existing
+month/day/four-digit-year format, including single-digit months and days.
+
+- Pending at start: X precedes October 1 of the previous year.
+- Received: X falls within the fiscal year, including both boundaries.
+- Processed: Y falls within the fiscal year, including both boundaries.
+- Pending at end: X is populated and Y is blank.
+
+Both component and agency counts must satisfy
+`pending start + received - processed = pending end`. Because X and Y do not
+have separate CSV validation rules yet, the accumulator rejects invalid dates,
+Y without X, Y preceding X, and X after year-end. A populated Y outside the
+fiscal year makes these counters fail the balance check; such rows must be
+reviewed rather than silently omitted or reported with adjusted totals.
+Errors raise processing exceptions before any existing XML is replaced.
+
+`foia:ProcessedAppealSection` follows the applied exemptions section. It uses
+the same four `ProcessingStatistics` quantities as requests, with IDs `PA1`,
+`PA2`, etc. and agency `PA0`. `ProcessingStatisticsOrganizationAssociation`
+links each PA ID to its corresponding ORG ID. Components with no appeals have
+four zero quantities. Request statistics continue to use distinct PS IDs.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php
new file mode 100644
index 000000000..0ffe9488b
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealStatisticsAggregator.php
@@ -0,0 +1,137 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams validated CSVs into fiscal-year appeal statistics.
+ */
+final class AppealStatisticsAggregator {
+
+  /**
+   * Counters initialized even for a component with no appeal rows.
+   */
+  private const EMPTY_COUNTS = [
+    'pending_start' => 0,
+    'received' => 0,
+    'processed' => 0,
+    'pending_end' => 0,
+  ];
+
+  /**
+   * Returns component counters and agency totals without retaining CSV rows.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component counts keyed by entity ID, plus overall totals.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $start = ($fiscal_year - 1) * 10000 + 1001;
+    $end = $fiscal_year * 10000 + 930;
+    $components = [];
+    foreach ($sources as $source) {
+      $counts = self::EMPTY_COUNTS;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal statistics.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          // Rows without an appeal do not contribute to any of the counters.
+          $received_text = trim($columns[23]);
+          $completed = trim($columns[24]);
+          $context = sprintf('Component %s, CSV record %d', $source['component_id'], $record);
+          if ($received_text === '') {
+            if ($completed !== '') {
+              throw new \RuntimeException($context . ': Appeal Date Closed requires an Appeal Date Received.');
+            }
+            continue;
+          }
+          $received = $this->calendarDate($received_text, $context . ', Column X');
+          if ($received < $start) {
+            $counts['pending_start']++;
+          }
+          elseif ($received <= $end) {
+            $counts['received']++;
+          }
+          else {
+            throw new \RuntimeException($context . ': Appeal Date Received is after the report fiscal year.');
+          }
+
+          // Count pending appeals directly from blank Column Y.
+          if ($completed === '') {
+            $counts['pending_end']++;
+          }
+          else {
+            $completed_date = $this->calendarDate($completed, $context . ', Column Y');
+            if ($completed_date < $received) {
+              throw new \RuntimeException($context . ': Appeal Date Closed cannot precede Appeal Date Received.');
+            }
+            if ($completed_date >= $start && $completed_date <= $end) {
+              $counts['processed']++;
+            }
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal statistics.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+      // The counts must reconcile before any XML file can be replaced.
+      $this->checkBalance($counts, (string) $source['component_id']);
+      $id = $source['component_id'];
+      $components[$id] ??= self::EMPTY_COUNTS;
+      foreach ($counts as $key => $value) {
+        $components[$id][$key] += $value;
+      }
+    }
+
+    $overall = self::EMPTY_COUNTS;
+    foreach ($components as $counts) {
+      foreach ($counts as $key => $value) {
+        $overall[$key] += $value;
+      }
+    }
+    $this->checkBalance($overall, 'agency overall');
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+  /**
+   * Parses appeal dates, which do not yet have separate CSV validation rules.
+   */
+  private function calendarDate(string $value, string $context): int {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+    }
+    return (int) $parts[3] * 10000 + (int) $parts[1] * 100 + (int) $parts[2];
+  }
+
+  /**
+   * Checks pending start + received - processed = pending end.
+   */
+  private function checkBalance(array $counts, string $label): void {
+    if ($counts['pending_start'] + $counts['received'] - $counts['processed'] !== $counts['pending_end']) {
+      throw new \RuntimeException('Appeal statistics do not balance for ' . $label . '.');
+    }
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 148c2f7be..1c10a3587 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -19,6 +19,7 @@
 use Drupal\foia_raw_data_to_report\DispositionAggregator;
 use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
 use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
+use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -159,7 +160,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $dispositions = (new DispositionAggregator())->aggregate($sources);
     $other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources);
     $applied_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions);
+    $appeal_statistics = (new AppealStatisticsAggregator())->aggregate($sources, $fiscal_year);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index de760a15d..cd9af6bc0 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -41,11 +41,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from OtherDenialReasonAggregator.
    * @param array $applied_exemptions
    *   Component and overall summaries from AppliedExemptionsAggregator.
+   * @param array $appeal_statistics
+   *   Component and overall summaries from AppealStatisticsAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -86,7 +88,7 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
 
     $this->addStatutes($document, $root, $statutes, $component_map);
     if ($request_statistics !== []) {
-      $this->addRequestStatistics($document, $root, $request_statistics, $component_map);
+      $this->addProcessingStatistics($document, $root, $request_statistics, $component_map, 'ProcessedRequestSection', 'PS');
     }
     if ($dispositions !== []) {
       $this->addDispositions($document, $root, $dispositions, $component_map);
@@ -98,6 +100,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addAppliedExemptions($document, $root, $applied_exemptions, $component_map);
     }
 
+    if ($appeal_statistics !== []) {
+      $this->addProcessingStatistics($document, $root, $appeal_statistics, $component_map, 'ProcessedAppealSection', 'PA');
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -144,10 +150,10 @@ private function addStatutes(\DOMDocument $document, \DOMElement $root, array $s
   }
 
   /**
-   * Adds request counters and references to component/agency organizations.
+   * Adds request or appeal counters and their organization references.
    */
-  private function addRequestStatistics(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void {
-    $section = $this->addTextElement($document, $root, 'foia', 'ProcessedRequestSection');
+  private function addProcessingStatistics(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map, string $section_name, string $prefix): void {
+    $section = $this->addTextElement($document, $root, 'foia', $section_name);
     $fields = [
       'pending_start' => 'ProcessingStatisticsPendingAtStartQuantity',
       'received' => 'ProcessingStatisticsReceivedQuantity',
@@ -160,10 +166,10 @@ private function addRequestStatistics(\DOMDocument $document, \DOMElement $root,
     }
     $organizations['ORG0'] = $statistics['overall'];
 
-    // Keep matching suffixes: PS1 refers to ORG1, and PS0 to the agency ORG0.
+    // Match organization suffixes: PS/PA1 links to ORG1, PS/PA0 to ORG0.
     foreach ($organizations as $organization_id => $counts) {
       $entry = $this->addTextElement($document, $section, 'foia', 'ProcessingStatistics');
-      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'PS' . substr($organization_id, 3));
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
       foreach ($fields as $key => $name) {
         $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]);
       }
@@ -172,7 +178,7 @@ private function addRequestStatistics(\DOMDocument $document, \DOMElement $root,
     foreach ($organizations as $organization_id => $counts) {
       $association = $this->addTextElement($document, $section, 'foia', 'ProcessingStatisticsOrganizationAssociation');
       $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
-      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'PS' . substr($organization_id, 3));
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3));
       $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
       $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
     }

From d190b6a18fb7f2a35f90e9c80f9dd5d9f2a67ac9 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 15:57:16 -0500
Subject: [PATCH 07/14] WS-472: Added Appeal Disposition section to XML report.

---
 .../custom/foia_raw_data_to_report/README.md  | 21 ++++-
 .../src/AppealDispositionAggregator.php       | 81 +++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |  4 +-
 .../src/XmlReportBuilder.php                  | 40 ++++++++-
 4 files changed, 143 insertions(+), 3 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index f15b14389..cc163d59c 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -75,7 +75,7 @@ processing; it does not snapshot the CSV selection. Deleted nodes are skipped.
 Processing exceptions leave the item available for retry after its lease expires.
 All CSVs are validated before aggregation and XML generation. Statute usage,
 processed request statistics, dispositions, other denial reasons, and applied
-exemptions and appeal processing statistics are aggregated; the remaining
+exemptions, appeal processing statistics, and appeal dispositions are aggregated; the remaining
 sections are not implemented yet.
 
 ## CSV validation and messages
@@ -404,3 +404,22 @@ the same four `ProcessingStatistics` quantities as requests, with IDs `PA1`,
 `PA2`, etc. and agency `PA0`. `ProcessingStatisticsOrganizationAssociation`
 links each PA ID to its corresponding ORG ID. Components with no appeals have
 four zero quantities. Request statistics continue to use distinct PS IDs.
+
+## Appeal dispositions
+
+`AppealDispositionAggregator` streams Column Z and counts the four specified
+outcomes: `Affirmed on Appeal`, `Partially Affirmed & Partially Reversed/Remanded`,
+`Completely Reversed/Remanded`, and `Closed for Other Reasons`. Matching is exact
+and case-sensitive after trimming surrounding whitespace. Blank cells, headers,
+and blank records do not contribute. Each populated row contributes once;
+agency counters sum the component counters. No additional date filter is applied
+by this accumulator. Existing validation and appeal date checks still run first.
+Unrecognized nonblank outcomes or read failures abort generation before the
+previous XML is replaced.
+
+`foia:AppealDispositionSection` follows the processed appeal section. It emits
+Affirmed, Partial, Reversed, and Other quantities and their
+`AppealDispositionTotalQuantity` sum, including zeroes for components without
+outcomes. IDs `AD1`, `AD2`, etc. and agency `AD0` link to their corresponding ORG
+IDs through `AppealDispositionOrganizationAssociation`. Only four counters per
+component are retained in memory.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php
new file mode 100644
index 000000000..b55259a92
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php
@@ -0,0 +1,81 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Column Z into appeal disposition counts per component.
+ */
+final class AppealDispositionAggregator {
+
+  /**
+   * Exact Column Z values mapped to their counters.
+   */
+  public const DISPOSITIONS = [
+    'Affirmed on Appeal' => 'affirmed',
+    'Partially Affirmed & Partially Reversed/Remanded' => 'partial',
+    'Completely Reversed/Remanded' => 'reversed',
+    'Closed for Other Reasons' => 'other',
+  ];
+
+  /**
+   * Counts each nonblank disposition once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component counts keyed by entity ID and overall counts, keyed by outcome.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_values(self::DISPOSITIONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal disposition counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $code = trim($columns[25]);
+          if ($code === '') {
+            continue;
+          }
+          // Reject unmapped values rather than silently omitting requests.
+          if (!isset(self::DISPOSITIONS[$code])) {
+            throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $id, $record));
+          }
+          $components[$id][self::DISPOSITIONS[$code]]++;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal disposition counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 1c10a3587..686bb1eda 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -20,6 +20,7 @@
 use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
 use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
 use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator;
+use Drupal\foia_raw_data_to_report\AppealDispositionAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -161,7 +162,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources);
     $applied_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources);
     $appeal_statistics = (new AppealStatisticsAggregator())->aggregate($sources, $fiscal_year);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics);
+    $appeal_dispositions = (new AppealDispositionAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index cd9af6bc0..8bef6b066 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -43,11 +43,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from AppliedExemptionsAggregator.
    * @param array $appeal_statistics
    *   Component and overall summaries from AppealStatisticsAggregator.
+   * @param array $appeal_dispositions
+   *   Component and overall summaries from AppealDispositionAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -103,6 +105,9 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
     if ($appeal_statistics !== []) {
       $this->addProcessingStatistics($document, $root, $appeal_statistics, $component_map, 'ProcessedAppealSection', 'PA');
     }
+    if ($appeal_dispositions !== []) {
+      $this->addAppealDispositions($document, $root, $appeal_dispositions, $component_map);
+    }
 
     $xml = $document->saveXML();
     if ($xml === FALSE) {
@@ -286,6 +291,39 @@ private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root,
     }
   }
 
+  /**
+   * Adds four appeal disposition counts, their total, and organization links.
+   */
+  private function addAppealDispositions(\DOMDocument $document, \DOMElement $root, array $dispositions, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'AppealDispositionSection');
+    $fields = [
+      'affirmed' => 'AppealDispositionAffirmedQuantity',
+      'partial' => 'AppealDispositionPartialQuantity',
+      'reversed' => 'AppealDispositionReversedQuantity',
+      'other' => 'AppealDispositionOtherQuantity',
+    ];
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $dispositions['components'][$component_id];
+    }
+    $organizations['ORG0'] = $dispositions['overall'];
+    foreach ($organizations as $organization_id => $counts) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'AppealDisposition');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'AD' . substr($organization_id, 3));
+      foreach ($fields as $key => $name) {
+        $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]);
+      }
+      $this->addTextElement($document, $entry, 'foia', 'AppealDispositionTotalQuantity', (string) array_sum($counts));
+    }
+    foreach ($organizations as $organization_id => $counts) {
+      $association = $this->addTextElement($document, $section, 'foia', 'AppealDispositionOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'AD' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From ca78a8467b5de32a6657c398d57124b88cab42e0 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 16:20:30 -0500
Subject: [PATCH 08/14] WS-472: Added AppealDispositionAppliedExemption section
 to XML report.

---
 .../custom/foia_raw_data_to_report/README.md  | 19 +++++++++++++++++++
 .../src/AppliedExemptionsAggregator.php       | 14 ++++++++++----
 .../QueueWorker/RawDataToReportProcessing.php |  4 +++-
 .../src/XmlReportBuilder.php                  | 18 ++++++++++++------
 4 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index cc163d59c..217cfa7a2 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -423,3 +423,22 @@ Affirmed, Partial, Reversed, and Other quantities and their
 outcomes. IDs `AD1`, `AD2`, etc. and agency `AD0` link to their corresponding ORG
 IDs through `AppealDispositionOrganizationAssociation`. Only four counters per
 component are retained in memory.
+
+## Appeal applied exemptions
+
+The queue reuses `AppliedExemptionsAggregator` with zero-based column 28 (AC)
+for appeal exemptions, independently of Column P's request counts. It splits
+commas, trims surrounding whitespace, normalizes letter case, and counts each
+exemption once per row. For example, `5,7a,7A` contributes one count each to
+5 and 7(A). Blank cells, headers, and blank records are skipped. Unknown codes
+(including malformed separators or empty list entries) stop generation before
+an existing XML attachment is replaced.
+
+`foia:AppealDispositionAppliedExemptionsSection` follows the appeal disposition
+section. `ComponentAppliedExemptions` IDs `ADE1`, `ADE2`, etc. and agency `ADE0`
+contain nonzero `AppliedExemption` entries with labels such as `Ex. 5` and
+`Ex. 7(A)` and their quantities. Agency counts sum the component counts.
+`ComponentAppliedExemptionsOrganizationAssociation` links each ADE entry to
+its corresponding ORG entry. Empty components retain their container and
+association, but no zero-count exemption entries. No combined total is emitted.
+Request exemptions continue to use a separate section and RDE IDs.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
index 11103ce12..3d8c2ed21 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppliedExemptionsAggregator.php
@@ -3,7 +3,7 @@
 namespace Drupal\foia_raw_data_to_report;
 
 /**
- * Streams validated component CSVs into applied exemption counts from Column P.
+ * Streams component CSVs into request or appeal exemption counts.
  */
 final class AppliedExemptionsAggregator {
 
@@ -32,11 +32,17 @@ final class AppliedExemptionsAggregator {
    *
    * @param array $sources
    *   Component/file pairs, each with component_id and uri keys.
+   * @param int $column
+   *   Zero-based CSV column: 15 for requests, 28 for appeals.
    *
    * @return array
    *   Component counts keyed by entity ID and overall counts, keyed by code.
    */
-  public function aggregate(array $sources): array {
+  public function aggregate(array $sources, int $column = 15): array {
+    if (!in_array($column, [15, 28], TRUE)) {
+      throw new \InvalidArgumentException('Exemption counts require Column P or AC.');
+    }
+    $column_name = $column === 15 ? 'P' : 'AC';
     $empty = array_fill_keys(array_keys(self::EXEMPTIONS), 0);
     $components = [];
     foreach ($sources as $source) {
@@ -61,7 +67,7 @@ public function aggregate(array $sources): array {
             $header = FALSE;
             continue;
           }
-          $value = trim($columns[15]);
+          $value = trim($columns[$column]);
           if ($value === '') {
             continue;
           }
@@ -71,7 +77,7 @@ public function aggregate(array $sources): array {
           foreach ($codes as $code) {
             // Unknown codes must not silently disappear from the report.
             if (!isset(self::EXEMPTIONS[$code])) {
-              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column P.', $id, $record, $code));
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown exemption code "%s" in Column %s.', $id, $record, $code, $column_name));
             }
             $components[$id][$code]++;
           }
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 686bb1eda..b4d6f87e9 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -163,7 +163,9 @@ protected function generateXmlReport(NodeInterface $node): void {
     $applied_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources);
     $appeal_statistics = (new AppealStatisticsAggregator())->aggregate($sources, $fiscal_year);
     $appeal_dispositions = (new AppealDispositionAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions);
+    // Column AC holds appeal exemptions; Column P remains request-only.
+    $appeal_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources, 28);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 8bef6b066..72991e9f6 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -45,11 +45,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from AppealStatisticsAggregator.
    * @param array $appeal_dispositions
    *   Component and overall summaries from AppealDispositionAggregator.
+   * @param array $appeal_exemptions
+   *   Column AC summaries from AppliedExemptionsAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -99,7 +101,7 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addOtherDenialReasons($document, $root, $other_reasons, $component_map);
     }
     if ($applied_exemptions !== []) {
-      $this->addAppliedExemptions($document, $root, $applied_exemptions, $component_map);
+      $this->addAppliedExemptions($document, $root, $applied_exemptions, $component_map, 'RequestDispositionAppliedExemptionsSection', 'RDE');
     }
 
     if ($appeal_statistics !== []) {
@@ -109,6 +111,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addAppealDispositions($document, $root, $appeal_dispositions, $component_map);
     }
 
+    if ($appeal_exemptions !== []) {
+      $this->addAppliedExemptions($document, $root, $appeal_exemptions, $component_map, 'AppealDispositionAppliedExemptionsSection', 'ADE');
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -262,8 +268,8 @@ private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root
   /**
    * Adds per-exemption counts and organization references, without a total.
    */
-  private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root, array $exemptions, array $component_map): void {
-    $section = $this->addTextElement($document, $root, 'foia', 'RequestDispositionAppliedExemptionsSection');
+  private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root, array $exemptions, array $component_map, string $section_name, string $prefix): void {
+    $section = $this->addTextElement($document, $root, 'foia', $section_name);
     $organizations = [];
     foreach ($component_map as $component_id => $organization_id) {
       $organizations[$organization_id] = $exemptions['components'][$component_id];
@@ -271,7 +277,7 @@ private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root,
     $organizations['ORG0'] = $exemptions['overall'];
     foreach ($organizations as $organization_id => $counts) {
       $entry = $this->addTextElement($document, $section, 'foia', 'ComponentAppliedExemptions');
-      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'RDE' . substr($organization_id, 3));
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
       foreach (AppliedExemptionsAggregator::EXEMPTIONS as $code => $label) {
         // Omit unused exemptions for components and the agency overall.
         if ($counts[$code] === 0) {
@@ -285,7 +291,7 @@ private function addAppliedExemptions(\DOMDocument $document, \DOMElement $root,
     foreach ($organizations as $organization_id => $counts) {
       $association = $this->addTextElement($document, $section, 'foia', 'ComponentAppliedExemptionsOrganizationAssociation');
       $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
-      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'RDE' . substr($organization_id, 3));
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3));
       $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
       $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
     }

From 9948a939412f38de8702fb24bd848d009ddae5af Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 17:16:03 -0500
Subject: [PATCH 09/14] WS-472: Added AppealNonExemptionDenialSection to the
 XML report.

---
 .../custom/foia_raw_data_to_report/README.md  | 14 +++
 .../AppealNonExemptionDenialAggregator.php    | 92 +++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |  4 +-
 .../src/XmlReportBuilder.php                  | 37 +++++++-
 4 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index 217cfa7a2..55934df2b 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -442,3 +442,17 @@ contain nonzero `AppliedExemption` entries with labels such as `Ex. 5` and
 its corresponding ORG entry. Empty components retain their container and
 association, but no zero-count exemption entries. No combined total is emitted.
 Request exemptions continue to use a separate section and RDE IDs.
+
+## Appeal non-exemption denials
+
+`AppealNonExemptionDenialAggregator` streams Column AA and maps the 11 CSV
+reason labels to XML codes using its `REASONS` constant. Comma-separated
+reasons are trimmed and counted once per distinct reason per row. Blank
+cells are ignored; unknown labels stop generation with the component and CSV
+record in the error message. Agency counts sum all component counts.
+
+`AppealNonExemptionDenialSection` follows the appeal exemptions section.
+Each component and the agency have an `AppealNonExemptionDenial` entry with
+all 11 reason counts, including zero counts, matching the example XML.
+`ANE1`, `ANE2`, etc. reference the component organizations; `ANE0` references
+`ORG0` for the agency. No combined total across reasons is emitted.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php
new file mode 100644
index 000000000..5dbbd64a7
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php
@@ -0,0 +1,92 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Column AA into appeal non-exemption denial counts.
+ */
+final class AppealNonExemptionDenialAggregator {
+
+  /**
+   * Column AA reason labels mapped to annual report XML codes.
+   */
+  public const REASONS = [
+    'No Records' => 'NoRecords',
+    'Records Referred at the Initial Request Level' => 'Referred',
+    'Request Withdrawn' => 'Withdrawn',
+    'Fee-Related Reason' => 'FeeRelated',
+    'Records not Reasonably Described' => 'NotDescribed',
+    'Improper Request for Other Reasons' => 'ImproperRequest',
+    'Not an Agency Record' => 'NotAgency',
+    'Duplicate Request or Appeal' => 'Duplicate',
+    'Request in Litigation' => 'InLitigation',
+    'Appeal Based Solely on Denial of Request for Expedited Processing' => 'ExpeditedDenial',
+    'Other' => 'Other',
+  ];
+
+  /**
+   * Counts each distinct reason once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component and overall counts keyed by XML reason code.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_values(self::REASONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal non-exemption denial counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $value = trim($columns[26]);
+          if ($value === '') {
+            continue;
+          }
+          // Split comma-separated reasons and count repeats once per appeal.
+          $reasons = array_unique(array_map('trim', explode(',', $value)));
+          foreach ($reasons as $reason) {
+            // Unknown labels must not silently disappear from the report.
+            if (!isset(self::REASONS[$reason])) {
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $id, $record, $reason));
+            }
+            $components[$id][self::REASONS[$reason]]++;
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal non-exemption denial counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index b4d6f87e9..6a88b8942 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -21,6 +21,7 @@
 use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
 use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator;
 use Drupal\foia_raw_data_to_report\AppealDispositionAggregator;
+use Drupal\foia_raw_data_to_report\AppealNonExemptionDenialAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
@@ -165,7 +166,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $appeal_dispositions = (new AppealDispositionAggregator())->aggregate($sources);
     // Column AC holds appeal exemptions; Column P remains request-only.
     $appeal_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources, 28);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions);
+    $appeal_denials = (new AppealNonExemptionDenialAggregator())->aggregate($sources);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 72991e9f6..633b630fe 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -47,11 +47,13 @@ final class XmlReportBuilder {
    *   Component and overall summaries from AppealDispositionAggregator.
    * @param array $appeal_exemptions
    *   Column AC summaries from AppliedExemptionsAggregator.
+   * @param array $appeal_denials
+   *   Column AA summaries from AppealNonExemptionDenialAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -115,6 +117,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addAppliedExemptions($document, $root, $appeal_exemptions, $component_map, 'AppealDispositionAppliedExemptionsSection', 'ADE');
     }
 
+    if ($appeal_denials !== []) {
+      $this->addAppealNonExemptionDenials($document, $root, $appeal_denials, $component_map);
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -330,6 +336,35 @@ private function addAppealDispositions(\DOMDocument $document, \DOMElement $root
     }
   }
 
+  /**
+   * Adds appeal denial reason counts and their organization associations.
+   */
+  private function addAppealNonExemptionDenials(\DOMDocument $document, \DOMElement $root, array $denials, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'AppealNonExemptionDenialSection');
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $denials['components'][$component_id];
+    }
+    $organizations['ORG0'] = $denials['overall'];
+    foreach ($organizations as $organization_id => $counts) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'AppealNonExemptionDenial');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'ANE' . substr($organization_id, 3));
+      // Include all reason codes, including zero counts, as in the example.
+      foreach (AppealNonExemptionDenialAggregator::REASONS as $code) {
+        $reason = $this->addTextElement($document, $entry, 'foia', 'NonExemptionDenial');
+        $this->addTextElement($document, $reason, 'foia', 'NonExemptionDenialReasonCode', $code);
+        $this->addTextElement($document, $reason, 'foia', 'NonExemptionDenialQuantity', (string) $counts[$code]);
+      }
+    }
+    foreach ($organizations as $organization_id => $counts) {
+      $association = $this->addTextElement($document, $section, 'foia', 'AppealNonExemptionDenialOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'ANE' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From 5544b93e388c976b269d09e4554d740919b2e32c Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 18:34:20 -0500
Subject: [PATCH 10/14] WS-472: Added AppealDenialOtherReason section to XML
 report.

---
 .../custom/foia_raw_data_to_report/README.md  | 14 +++++++++
 .../src/OtherDenialReasonAggregator.php       | 30 ++++++++++++-------
 .../QueueWorker/RawDataToReportProcessing.php |  3 +-
 .../src/XmlReportBuilder.php                  | 18 +++++++----
 4 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index 55934df2b..1ef98aa03 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -456,3 +456,17 @@ Each component and the agency have an `AppealNonExemptionDenial` entry with
 all 11 reason counts, including zero counts, matching the example XML.
 `ANE1`, `ANE2`, etc. reference the component organizations; `ANE0` references
 `ORG0` for the agency. No combined total across reasons is emitted.
+
+## Appeal other denial reasons
+
+`OtherDenialReasonAggregator` also streams Column AB (column index 27). Unlike
+Column O, appeal reasons are split on commas. Surrounding whitespace and empty
+entries are ignored; each distinct reason counts once per row. Text is retained
+without a code mapping, preserving case and internal whitespace. Agency counts
+sum the component counts, and reasons are sorted by text for stable output.
+
+`AppealDenialOtherReasonSection` follows the appeal non-exemption denial section.
+It includes reason descriptions and counts for each component and the agency,
+plus `ComponentOtherDenialReasonQuantity` as the sum of usage counts. `ADOR1`,
+`ADOR2`, etc. link to component organizations; `ADOR0` links to `ORG0`. Empty
+components retain a zero total and an organization association.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php
index 108503e70..d25246951 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/OtherDenialReasonAggregator.php
@@ -3,7 +3,7 @@
 namespace Drupal\foia_raw_data_to_report;
 
 /**
- * Streams validated CSVs into counts of the free-text reasons in Column O.
+ * Streams CSVs into request or appeal free-text denial reason counts.
  */
 final class OtherDenialReasonAggregator {
 
@@ -12,11 +12,16 @@ final class OtherDenialReasonAggregator {
    *
    * @param array $sources
    *   Component/file pairs, each with component_id and uri keys.
+   * @param int $column
+   *   Zero-based CSV column: 14 for requests, 27 for appeals.
    *
    * @return array
    *   Component and overall reason maps, with description and quantity entries.
    */
-  public function aggregate(array $sources): array {
+  public function aggregate(array $sources, int $column = 14): array {
+    if (!in_array($column, [14, 27], TRUE)) {
+      throw new \InvalidArgumentException('Other denial reasons require Column O or AB.');
+    }
     $components = [];
     foreach ($sources as $source) {
       $id = $source['component_id'];
@@ -38,16 +43,19 @@ public function aggregate(array $sources): array {
             $header = FALSE;
             continue;
           }
-          // Match exact text after trimming only surrounding whitespace.
-          // Keep case, punctuation, and internal whitespace distinctions.
-          $reason = trim($columns[14]);
-          if ($reason === '') {
-            continue;
+          // Column AB holds comma-separated reasons; Column O is one reason.
+          $values = $column === 27 ? explode(',', $columns[$column]) : [$columns[$column]];
+          // Count each reason once per row, trimming surrounding whitespace.
+          // Preserve case, punctuation, and internal whitespace distinctions.
+          foreach (array_unique(array_map('trim', $values)) as $reason) {
+            if ($reason === '') {
+              continue;
+            }
+            // A prefix preserves numeric-looking reasons as text keys.
+            $key = 'reason:' . $reason;
+            $components[$id][$key] ??= ['description' => $reason, 'quantity' => 0];
+            $components[$id][$key]['quantity']++;
           }
-          // A prefix preserves numeric-looking reasons as text keys.
-          $key = 'reason:' . $reason;
-          $components[$id][$key] ??= ['description' => $reason, 'quantity' => 0];
-          $components[$id][$key]['quantity']++;
         }
         if (!feof($stream)) {
           throw new \RuntimeException('Unable to finish reading a CSV for other denial reasons.');
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 6a88b8942..acb963bad 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -167,7 +167,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     // Column AC holds appeal exemptions; Column P remains request-only.
     $appeal_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources, 28);
     $appeal_denials = (new AppealNonExemptionDenialAggregator())->aggregate($sources);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials);
+    $appeal_other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources, 27);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 633b630fe..3a42f06b5 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -49,11 +49,13 @@ final class XmlReportBuilder {
    *   Column AC summaries from AppliedExemptionsAggregator.
    * @param array $appeal_denials
    *   Column AA summaries from AppealNonExemptionDenialAggregator.
+   * @param array $appeal_other_reasons
+   *   Column AB summaries from OtherDenialReasonAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -100,7 +102,7 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addDispositions($document, $root, $dispositions, $component_map);
     }
     if ($other_reasons !== []) {
-      $this->addOtherDenialReasons($document, $root, $other_reasons, $component_map);
+      $this->addOtherDenialReasons($document, $root, $other_reasons, $component_map, 'RequestDenialOtherReasonSection', 'CODR');
     }
     if ($applied_exemptions !== []) {
       $this->addAppliedExemptions($document, $root, $applied_exemptions, $component_map, 'RequestDispositionAppliedExemptionsSection', 'RDE');
@@ -121,6 +123,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addAppealNonExemptionDenials($document, $root, $appeal_denials, $component_map);
     }
 
+    if ($appeal_other_reasons !== []) {
+      $this->addOtherDenialReasons($document, $root, $appeal_other_reasons, $component_map, 'AppealDenialOtherReasonSection', 'ADOR');
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -243,8 +249,8 @@ private function addDispositions(\DOMDocument $document, \DOMElement $root, arra
   /**
    * Adds free-text reason counts, totals, and organization references.
    */
-  private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root, array $other_reasons, array $component_map): void {
-    $section = $this->addTextElement($document, $root, 'foia', 'RequestDenialOtherReasonSection');
+  private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root, array $other_reasons, array $component_map, string $section_name, string $prefix): void {
+    $section = $this->addTextElement($document, $root, 'foia', $section_name);
     $organizations = [];
     foreach ($component_map as $component_id => $organization_id) {
       $organizations[$organization_id] = $other_reasons['components'][$component_id];
@@ -252,7 +258,7 @@ private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root
     $organizations['ORG0'] = $other_reasons['overall'];
     foreach ($organizations as $organization_id => $reasons) {
       $entry = $this->addTextElement($document, $section, 'foia', 'ComponentOtherDenialReason');
-      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'CODR' . substr($organization_id, 3));
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3));
       foreach ($reasons as $reason) {
         $item = $this->addTextElement($document, $entry, 'foia', 'OtherDenialReason');
         $this->addTextElement($document, $item, 'foia', 'OtherDenialReasonDescriptionText', $reason['description']);
@@ -265,7 +271,7 @@ private function addOtherDenialReasons(\DOMDocument $document, \DOMElement $root
     foreach ($organizations as $organization_id => $reasons) {
       $association = $this->addTextElement($document, $section, 'foia', 'OtherDenialReasonOrganizationAssociation');
       $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
-      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'CODR' . substr($organization_id, 3));
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3));
       $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
       $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
     }

From 7e24eaa7a4303a8502b15fbd8fa045f48bbf79dd Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Sun, 20 Sep 2026 22:46:45 -0500
Subject: [PATCH 11/14] WS-472: Added AppealResponseTimeSection section to the
 XML output.

---
 .../custom/foia_raw_data_to_report/README.md  |  19 +++
 .../src/AppealResponseTimeAggregator.php      | 133 ++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |   4 +-
 .../src/XmlReportBuilder.php                  |  41 +++++-
 4 files changed, 195 insertions(+), 2 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index 1ef98aa03..1f86732f9 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -470,3 +470,22 @@ It includes reason descriptions and counts for each component and the agency,
 plus `ComponentOtherDenialReasonQuantity` as the sum of usage counts. `ADOR1`,
 `ADOR2`, etc. link to component organizations; `ADOR0` links to `ORG0`. Empty
 components retain a zero total and an organization association.
+
+## Appeal response times
+
+`AppealResponseTimeAggregator` uses Columns X and Y. Rows without an Appeal
+Date Received are excluded; a closed date without a received date is rejected.
+The start is the later of Appeal Date Received and October 1 of the previous
+year. The end is Appeal Date Closed, or September 30 of the report year when
+Y is blank.
+Elapsed calendar days exclude the starting day and include the ending day;
+same-day completion is zero. Weekends and holidays count.
+
+Day-frequency maps keep memory independent of the number of rows. They yield
+the exact median (averaging the middle pair for even counts), average rounded
+to two decimals, minimum, and maximum. The agency uses combined frequencies,
+not averages of component statistics. Empty components report zeros.
+
+`AppealResponseTimeSection` follows the appeal other-reasons section, using
+`ART1`, `ART2`, etc. for components and `ART0` for the agency, associated with
+their Organization entries. XML averages always display two decimal places.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php
new file mode 100644
index 000000000..4c042625d
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php
@@ -0,0 +1,133 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Columns X and Y into calendar-day response time distributions.
+ */
+final class AppealResponseTimeAggregator {
+
+  /**
+   * Calculates component and agency statistics without retaining CSV rows.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component and overall median, average, lowest, and highest values.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $timezone = new \DateTimeZone('UTC');
+    $start = new \DateTimeImmutable(($fiscal_year - 1) . '-10-01', $timezone);
+    $end = new \DateTimeImmutable($fiscal_year . '-09-30', $timezone);
+    $histograms = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $histograms[$id] ??= [];
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal response times.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $context = sprintf('Component %s, CSV record %d', $id, $record);
+          // Only rows with an appeal contribute to response time statistics.
+          $received_text = trim($columns[23]);
+          $completed = trim($columns[24]);
+          if ($received_text === '') {
+            if ($completed !== '') {
+              throw new \RuntimeException($context . ': Appeal Date Closed requires an Appeal Date Received.');
+            }
+            continue;
+          }
+          $received = $this->calendarDate($received_text, $context . ', Column X');
+          $finish = $completed === '' ? $end : $this->calendarDate($completed, $context . ', Column Y');
+          $begin = $received > $start ? $received : $start;
+          if ($finish < $begin || $finish > $end) {
+            throw new \RuntimeException($context . ': Response time dates must be in chronological order and within the report fiscal year.');
+          }
+          // Elapsed calendar days exclude the starting day; same-day is zero.
+          // UTC dates avoid daylight-saving changes affecting the calculation.
+          $days = (int) $begin->diff($finish)->days;
+          $histograms[$id][$days] = ($histograms[$id][$days] ?? 0) + 1;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal response times.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $components = [];
+    $overall = [];
+    foreach ($histograms as $id => $histogram) {
+      $components[$id] = $this->summarize($histogram);
+      // Combine frequencies so agency statistics are weighted by appeal count.
+      foreach ($histogram as $days => $count) {
+        $overall[$days] = ($overall[$days] ?? 0) + $count;
+      }
+    }
+    return ['components' => $components, 'overall' => $this->summarize($overall)];
+  }
+
+  /**
+   * Parses calendar dates defensively if a file changed after validation.
+   */
+  private function calendarDate(string $value, string $context): \DateTimeImmutable {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+    }
+    return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC'));
+  }
+
+  /**
+   * Finds exact medians and weighted averages from day frequencies.
+   */
+  private function summarize(array $histogram): array {
+    if ($histogram === []) {
+      return ['median' => 0, 'average' => 0, 'lowest' => 0, 'highest' => 0];
+    }
+    ksort($histogram, SORT_NUMERIC);
+    $count = array_sum($histogram);
+    $lower_position = intdiv($count + 1, 2);
+    $upper_position = intdiv($count, 2) + 1;
+    $seen = 0;
+    $sum = 0;
+    $lower = NULL;
+    $upper = NULL;
+    foreach ($histogram as $days => $frequency) {
+      $seen += $frequency;
+      $sum += $days * $frequency;
+      if ($lower === NULL && $seen >= $lower_position) {
+        $lower = $days;
+      }
+      if ($upper === NULL && $seen >= $upper_position) {
+        $upper = $days;
+      }
+    }
+    return [
+      'median' => ($lower + $upper) / 2,
+      'average' => round($sum / $count, 2),
+      'lowest' => array_key_first($histogram),
+      'highest' => array_key_last($histogram),
+    ];
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index acb963bad..09e3a5450 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -20,6 +20,7 @@
 use Drupal\foia_raw_data_to_report\OtherDenialReasonAggregator;
 use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
 use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator;
+use Drupal\foia_raw_data_to_report\AppealResponseTimeAggregator;
 use Drupal\foia_raw_data_to_report\AppealDispositionAggregator;
 use Drupal\foia_raw_data_to_report\AppealNonExemptionDenialAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
@@ -168,7 +169,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $appeal_exemptions = (new AppliedExemptionsAggregator())->aggregate($sources, 28);
     $appeal_denials = (new AppealNonExemptionDenialAggregator())->aggregate($sources);
     $appeal_other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources, 27);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons);
+    $appeal_response_times = (new AppealResponseTimeAggregator())->aggregate($sources, $fiscal_year);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 3a42f06b5..5da5bb740 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -51,11 +51,13 @@ final class XmlReportBuilder {
    *   Column AA summaries from AppealNonExemptionDenialAggregator.
    * @param array $appeal_other_reasons
    *   Column AB summaries from OtherDenialReasonAggregator.
+   * @param array $appeal_response_times
+   *   Component and overall summaries from AppealResponseTimeAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -127,6 +129,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addOtherDenialReasons($document, $root, $appeal_other_reasons, $component_map, 'AppealDenialOtherReasonSection', 'ADOR');
     }
 
+    if ($appeal_response_times !== []) {
+      $this->addAppealResponseTimes($document, $root, $appeal_response_times, $component_map);
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -371,6 +377,39 @@ private function addAppealNonExemptionDenials(\DOMDocument $document, \DOMElemen
     }
   }
 
+  /**
+   * Adds response time statistics and their organization references.
+   */
+  private function addAppealResponseTimes(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'AppealResponseTimeSection');
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $statistics['components'][$component_id];
+    }
+    $organizations['ORG0'] = $statistics['overall'];
+    $fields = [
+      'median' => 'ResponseTimeMedianDaysValue',
+      'average' => 'ResponseTimeAverageDaysValue',
+      'lowest' => 'ResponseTimeLowestDaysValue',
+      'highest' => 'ResponseTimeHighestDaysValue',
+    ];
+    foreach ($organizations as $organization_id => $values) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'ResponseTime');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'ART' . substr($organization_id, 3));
+      foreach ($fields as $key => $name) {
+        $value = $key === 'average' ? number_format($values[$key], 2, '.', '') : (string) $values[$key];
+        $this->addTextElement($document, $entry, 'foia', $name, $value);
+      }
+    }
+    foreach ($organizations as $organization_id => $values) {
+      $association = $this->addTextElement($document, $section, 'foia', 'ResponseTimeOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'ART' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From d481e454d27f45908264961c247c28edbdedd851 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Mon, 21 Sep 2026 00:15:43 -0500
Subject: [PATCH 12/14] WS-472: Added the OldestPendingAppeal section to the
 XML report.

---
 .../custom/foia_raw_data_to_report/README.md  | 14 +++
 .../src/OldestPendingAppealAggregator.php     | 98 +++++++++++++++++++
 .../QueueWorker/RawDataToReportProcessing.php |  4 +-
 .../src/XmlReportBuilder.php                  | 36 ++++++-
 4 files changed, 150 insertions(+), 2 deletions(-)
 create mode 100644 docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php

diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md
index 1f86732f9..b1862ec3c 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/README.md
+++ b/docroot/modules/custom/foia_raw_data_to_report/README.md
@@ -489,3 +489,17 @@ not averages of component statistics. Empty components report zeros.
 `AppealResponseTimeSection` follows the appeal other-reasons section, using
 `ART1`, `ART2`, etc. for components and `ART0` for the agency, associated with
 their Organization entries. XML averages always display two decimal places.
+
+## Oldest pending appeals
+
+`OldestPendingAppealAggregator` streams rows with a nonblank Column X and a
+blank Column Y. It retains only the ten earliest received dates per component
+and the ten earliest across the agency. Repeated dates remain separate items;
+fewer than ten pending appeals produce only the items available.
+
+Pending days are elapsed calendar days from the actual received date through
+September 30 of the report year, without clamping to the fiscal-year start.
+Same-day receipt is zero days. XML receipt dates use `YYYY-MM-DD`.
+`OldestPendingAppealSection` follows the appeal response times, with `OPA1`,
+`OPA2`, etc. linked to components and `OPA0` to the agency. Components without
+pending appeals retain an empty container and their organization association.
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php
new file mode 100644
index 000000000..9fd678d0d
--- /dev/null
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingAppealAggregator.php
@@ -0,0 +1,98 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Columns X and Y into the ten oldest open appeals.
+ */
+final class OldestPendingAppealAggregator {
+
+  /**
+   * Collects at most ten receipt dates per component and for the agency.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component and overall lists with receipt_date and pending_days entries.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $end = new \DateTimeImmutable($fiscal_year . '-09-30', new \DateTimeZone('UTC'));
+    $components = [];
+    $overall = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= [];
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for oldest pending appeals.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          // Only appeals with a received date and no closed date are pending.
+          $received_text = trim($columns[23]);
+          if ($received_text === '' || trim($columns[24]) !== '') {
+            continue;
+          }
+          $context = sprintf('Component %s, CSV record %d, Column X', $id, $record);
+          $received = $this->calendarDate($received_text, $context);
+          if ($received > $end) {
+            throw new \RuntimeException($context . ': Appeal Date Received is after the report fiscal year.');
+          }
+          // Use the full elapsed calendar time, including prior fiscal years.
+          $item = [
+            'receipt_date' => $received->format('Y-m-d'),
+            'pending_days' => (int) $received->diff($end)->days,
+          ];
+          $this->retainOldest($components[$id], $item);
+          $this->retainOldest($overall, $item);
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for oldest pending appeals.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+  /**
+   * Keeps the earliest ten dates, retaining repeated dates as separate rows.
+   */
+  private function retainOldest(array &$items, array $item): void {
+    $items[] = $item;
+    usort($items, static fn(array $a, array $b): int => strcmp($a['receipt_date'], $b['receipt_date']));
+    if (count($items) > 10) {
+      array_pop($items);
+    }
+  }
+
+  /**
+   * Parses calendar dates defensively if a file changed after validation.
+   */
+  private function calendarDate(string $value, string $context): \DateTimeImmutable {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+    }
+    return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC'));
+  }
+
+}
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
index 09e3a5450..d756bbfd5 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php
@@ -21,6 +21,7 @@
 use Drupal\foia_raw_data_to_report\AppliedExemptionsAggregator;
 use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator;
 use Drupal\foia_raw_data_to_report\AppealResponseTimeAggregator;
+use Drupal\foia_raw_data_to_report\OldestPendingAppealAggregator;
 use Drupal\foia_raw_data_to_report\AppealDispositionAggregator;
 use Drupal\foia_raw_data_to_report\AppealNonExemptionDenialAggregator;
 use Symfony\Component\DependencyInjection\ContainerInterface;
@@ -170,7 +171,8 @@ protected function generateXmlReport(NodeInterface $node): void {
     $appeal_denials = (new AppealNonExemptionDenialAggregator())->aggregate($sources);
     $appeal_other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources, 27);
     $appeal_response_times = (new AppealResponseTimeAggregator())->aggregate($sources, $fiscal_year);
-    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times);
+    $oldest_pending_appeals = (new OldestPendingAppealAggregator())->aggregate($sources, $fiscal_year);
+    $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times, $oldest_pending_appeals);
     $field = $node->get('field_request_data_xml');
     $previous_file = $field->entity;
     $item = $field->first() ?? $field->appendItem();
diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
index 5da5bb740..e2e9c0140 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php
@@ -53,11 +53,13 @@ final class XmlReportBuilder {
    *   Column AB summaries from OtherDenialReasonAggregator.
    * @param array $appeal_response_times
    *   Component and overall summaries from AppealResponseTimeAggregator.
+   * @param array $oldest_pending_appeals
+   *   Component and overall lists from OldestPendingAppealAggregator.
    *
    * @return string
    *   The serialized report XML.
    */
-  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = []): string {
+  public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = [], array $oldest_pending_appeals = []): string {
     $document = new \DOMDocument('1.0', 'UTF-8');
     $document->formatOutput = TRUE;
     $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport');
@@ -133,6 +135,10 @@ public function build(TermInterface $agency, array $components, int $fiscal_year
       $this->addAppealResponseTimes($document, $root, $appeal_response_times, $component_map);
     }
 
+    if ($oldest_pending_appeals !== []) {
+      $this->addOldestPendingAppeals($document, $root, $oldest_pending_appeals, $component_map);
+    }
+
     $xml = $document->saveXML();
     if ($xml === FALSE) {
       throw new \RuntimeException('Unable to serialize the raw data report XML.');
@@ -410,6 +416,34 @@ private function addAppealResponseTimes(\DOMDocument $document, \DOMElement $roo
     }
   }
 
+  /**
+   * Adds oldest open appeals and their organization references.
+   */
+  private function addOldestPendingAppeals(\DOMDocument $document, \DOMElement $root, array $appeals, array $component_map): void {
+    $section = $this->addTextElement($document, $root, 'foia', 'OldestPendingAppealSection');
+    $organizations = [];
+    foreach ($component_map as $component_id => $organization_id) {
+      $organizations[$organization_id] = $appeals['components'][$component_id];
+    }
+    $organizations['ORG0'] = $appeals['overall'];
+    foreach ($organizations as $organization_id => $items) {
+      $entry = $this->addTextElement($document, $section, 'foia', 'OldestPendingItems');
+      $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'OPA' . substr($organization_id, 3));
+      foreach ($items as $item) {
+        $old_item = $this->addTextElement($document, $entry, 'foia', 'OldItem');
+        $this->addTextElement($document, $old_item, 'foia', 'OldItemReceiptDate', $item['receipt_date']);
+        $this->addTextElement($document, $old_item, 'foia', 'OldItemPendingDaysQuantity', (string) $item['pending_days']);
+      }
+    }
+    foreach ($organizations as $organization_id => $items) {
+      $association = $this->addTextElement($document, $section, 'foia', 'OldestPendingItemsOrganizationAssociation');
+      $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference');
+      $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'OPA' . substr($organization_id, 3));
+      $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference');
+      $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id);
+    }
+  }
+
   /**
    * Appends a namespaced element and safely escapes optional text.
    */

From 22ee93d49c94c007dae9e4e14002d5e9ddee0ce3 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Mon, 21 Sep 2026 10:12:15 -0500
Subject: [PATCH 13/14] WS-472: CSV validation, Column Q must contain value if
 there is value in either Column R or S.

---
 .../src/CsvValidator.php                      |  6 +-
 .../tests/src/Unit/CsvValidatorTest.php       | 55 ++++++++++++++++++-
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
index ee870a2b0..61f73596f 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
@@ -536,10 +536,10 @@ public function validate(string $uri, int $fiscal_year): array {
           }
         }
 
-        // Column R: A determination date is optional unless Q, S or T has data.
+        // Column Q: A received date is required when R or S has data.
         $ep_determined = trim($columns[17]);
-        if ($ep_determined === '' && ($ep_received !== '' || trim($columns[18]) !== '' || trim($columns[19]) !== '')) {
-          return [sprintf('CSV record %d: Column R must contain valure if there is value in either Columns Q, S, or T', $record)];
+        if ($ep_received === '' && ($ep_determined !== '' || trim($columns[18]) !== '')) {
+          return [sprintf('CSV record %d: Column Q must contain value if there is value in either Columns R or S', $record)];
         }
 
         if ($ep_determined !== '') {
diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
index 4dabd6e90..26730e6e0 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
@@ -8,7 +8,7 @@
 use PHPUnit\Framework\Attributes\Group;
 
 /**
- * Tests streaming validation of CSV column counts.
+ * Tests CSV structure and expedited processing field dependencies.
  */
 #[CoversClass(CsvValidator::class)]
 #[Group('foia_raw_data_to_report')]
@@ -21,7 +21,7 @@ private function validateContents(string $contents): array {
     $path = tempnam(sys_get_temp_dir(), 'foia-csv-');
     try {
       file_put_contents($path, $contents);
-      return (new CsvValidator())->validate($path);
+      return (new CsvValidator())->validate($path, 2026);
     }
     finally {
       unlink($path);
@@ -33,10 +33,59 @@ private function validateContents(string $contents): array {
    */
   public function testValidCsv(): void {
     $header = implode(',', array_fill(0, 29, 'column'));
-    $row = '"A, B","First line' . "\n" . 'Second line","A ""quote""",' . implode(',', array_fill(0, 26, 'value'));
+    $row = $this->csvRow([0 => 'A, B', 1 => "First line\nSecond line", 22 => 'A "quote"']);
     $this->assertSame([], $this->validateContents("\xEF\xBB\xBF" . $header . "\r\n\r\n" . $row . "\r\n"));
   }
 
+  /**
+   * Serializes a valid request row with selected column overrides.
+   */
+  private function csvRow(array $overrides = []): string {
+    $row = array_fill(0, 29, '');
+    $row[0] = 'Component';
+    $row[1] = 'Request 1';
+    $row[2] = 'N';
+    $row[3] = '20';
+    $row[8] = '01/01/2026';
+    $stream = fopen('php://temp', 'w+');
+    fputcsv($stream, array_replace($row, $overrides), ',', '"', '');
+    rewind($stream);
+    $contents = stream_get_contents($stream);
+    fclose($stream);
+    return $contents;
+  }
+
+  /**
+   * Requires Q for R or S, but does not require R for Q, S, or T.
+   */
+  public function testExpeditedReceivedDependency(): void {
+    $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+    foreach ([
+      [17 => '01/03/2026'],
+      [18 => 'D'],
+      [16 => '  ', 17 => '01/03/2026', 18 => 'D'],
+    ] as $overrides) {
+      $this->assertSame(
+        ['CSV record 2: Column Q must contain value if there is value in either Columns R or S'],
+        $this->validateContents($header . $this->csvRow($overrides)),
+      );
+    }
+    foreach ([
+      [],
+      [16 => '01/02/2026'],
+      [19 => '01/02/2026'],
+      [16 => '01/02/2026', 18 => 'D'],
+      [16 => '01/02/2026', 17 => '01/03/2026', 18 => 'D'],
+    ] as $overrides) {
+      $this->assertSame([], $this->validateContents($header . $this->csvRow($overrides)));
+    }
+    // The existing R date-order and S granted/denied rules still apply.
+    $row = $this->csvRow([16 => '01/03/2026', 17 => '01/02/2026', 18 => 'D']);
+    $this->assertStringContainsString('cannot be prior', $this->validateContents($header . $row)[0]);
+    $row = $this->csvRow([16 => '01/02/2026', 17 => '01/03/2026']);
+    $this->assertSame(['CSV record 2: Must have G or D in Column S'], $this->validateContents($header . $row));
+  }
+
   /**
    * Rejects an incorrect header column count.
    */

From 4de8f0b5a76a5868f78309db6b2ffbb978ce9a71 Mon Sep 17 00:00:00 2001
From: Rob Ristroph <rgristroph@gmail.com>
Date: Mon, 21 Sep 2026 10:56:12 -0500
Subject: [PATCH 14/14] WS-472: Update to CSV validation, if Column X has data,
 Columns E through W must be empty.

---
 .../src/CsvValidator.php                      | 44 +++++++++-------
 .../tests/src/Unit/CsvValidatorTest.php       | 50 +++++++++++++++++++
 2 files changed, 77 insertions(+), 17 deletions(-)

diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
index 61f73596f..5c1c045f6 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php
@@ -268,6 +268,16 @@ public function validate(string $uri, int $fiscal_year): array {
           continue;
         }
 
+        // Column X: Appeal rows must not contain request data in E through W.
+        $appeal_received = trim($columns[23]);
+        if ($appeal_received !== '') {
+          foreach (array_slice($columns, 4, 19) as $value) {
+            if (trim($value) !== '') {
+              return [sprintf('CSV record %d: If there is data in Column X, Columns E through W must be empty.', $record)];
+            }
+          }
+        }
+
         // Column A: Component is required, including for consultation rows.
         if (trim($columns[0]) === '') {
           return [sprintf('CSV record %d: Component cannot be blank', $record)];
@@ -367,24 +377,24 @@ public function validate(string $uri, int $fiscal_year): array {
           return [sprintf('CSV record %d: Ex. 3 Code 77 must appear in Column E if there is data in Column H', $record)];
         }
 
-        // Column I: Date Initially Received is required for all rows.
+        // Column I: Optional, but the Column X check requires it to be blank
+        // for appeal rows. Validate date contents only when present.
         $initially_received = trim($columns[8]);
-        if ($initially_received === '') {
-          return [sprintf('CSV record %d: Data initially Received cannot be blank', $record)];
-        }
-
-        // Column I: Require a real calendar date in month/day/four-digit year
-        // order. Allow single-digit months/days as used in the reference CSV.
-        if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $initially_received, $date_parts)
-          || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
-          return [sprintf('CSV record %d: Date Initially Received must be a valid date in MM/DD/YYYY format', $record)];
-        }
+        $received_date = NULL;
+        if ($initially_received !== '') {
+          // Column I: Require a real calendar date in month/day/four-digit year
+          // order. Allow single-digit months/days as used in the reference CSV.
+          if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $initially_received, $date_parts)
+            || !checkdate((int) $date_parts[1], (int) $date_parts[2], (int) $date_parts[3])) {
+            return [sprintf('CSV record %d: Date Initially Received must be a valid date in MM/DD/YYYY format', $record)];
+          }
 
-        // Column I: September 30 of the report year is the latest allowed date.
-        // Compare calendar dates without time zones; earlier years are allowed.
-        $received_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
-        if ($received_date > $fiscal_year * 10000 + 930) {
-          return [sprintf('CSV record %d: Date Initially Received is later than the fiscal year', $record)];
+          // Column I: September 30 of the report year is the latest date.
+          // Compare dates without time zones; earlier years are allowed.
+          $received_date = (int) $date_parts[3] * 10000 + (int) $date_parts[1] * 100 + (int) $date_parts[2];
+          if ($received_date > $fiscal_year * 10000 + 930) {
+            return [sprintf('CSV record %d: Date Initially Received is later than the fiscal year', $record)];
+          }
         }
 
         // Column J: Date Perfected is optional; validate only nonblank values.
@@ -409,7 +419,7 @@ public function validate(string $uri, int $fiscal_year): array {
           }
 
           // Column J: Perfection cannot precede the initially received date.
-          if ($perfected_date < $received_date) {
+          if ($received_date !== NULL && $perfected_date < $received_date) {
             return [sprintf('CSV record %d: Date Perfected cannot be prior to Date Initially Received', $record)];
           }
         }
diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
index 26730e6e0..2d54f62df 100644
--- a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
+++ b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php
@@ -86,6 +86,56 @@ public function testExpeditedReceivedDependency(): void {
     $this->assertSame(['CSV record 2: Must have G or D in Column S'], $this->validateContents($header . $row));
   }
 
+  /**
+   * Rejects every request-data column E through W when X contains data.
+   */
+  public function testAppealRequestDataExclusion(): void {
+    $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+    for ($column = 4; $column <= 22; $column++) {
+      $overrides = [8 => '', 23 => '01/02/2026'];
+      // Zero is data too, even though PHP treats it as an empty value.
+      $overrides[$column] = '0';
+      $this->assertSame(
+        ['CSV record 2: If there is data in Column X, Columns E through W must be empty.'],
+        $this->validateContents($header . $this->csvRow($overrides)),
+        'Column index ' . $column,
+      );
+    }
+  }
+
+  /**
+   * Allows blank I with or without X, and validates nonblank received dates.
+   */
+  public function testOptionalInitiallyReceived(): void {
+    $header = implode(',', array_fill(0, 29, 'column')) . "\n";
+    foreach ([
+      [8 => ''],
+      [8 => '', 23 => '01/02/2026'],
+      [8 => '  ', 23 => '01/02/2026', 24 => '01/03/2026'],
+      [8 => '', 9 => '01/02/2026', 12 => 'S'],
+      [8 => '01/01/2026'],
+    ] as $overrides) {
+      $this->assertSame([], $this->validateContents($header . $this->csvRow($overrides)));
+    }
+    $this->assertSame(
+      ['CSV record 2: Date Initially Received must be a valid date in MM/DD/YYYY format'],
+      $this->validateContents($header . $this->csvRow([8 => '02/30/2026'])),
+    );
+    $this->assertSame(
+      ['CSV record 2: Date Initially Received is later than the fiscal year'],
+      $this->validateContents($header . $this->csvRow([8 => '10/01/2026'])),
+    );
+    // A blank I on the next row must not reuse the previous row's date.
+    $first = $this->csvRow([8 => '09/01/2026']);
+    $second = $this->csvRow([1 => 'Request 2', 8 => '', 9 => '01/02/2026', 12 => 'S']);
+    $this->assertSame([], $this->validateContents($header . $first . $second));
+    $row = $this->csvRow([8 => '01/03/2026', 9 => '01/02/2026', 12 => 'S']);
+    $this->assertSame(
+      ['CSV record 2: Date Perfected cannot be prior to Date Initially Received'],
+      $this->validateContents($header . $row),
+    );
+  }
+
   /**
    * Rejects an incorrect header column count.
    */