WS-472: XML now has Exemption 3 Statutes.

docroot/modules/custom/foia_raw_data_to_report/src/AppealDispositionAggregator.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Column Z into appeal disposition counts per component.
+ */
+final class AppealDispositionAggregator {
+
+  /**
+   * Exact Column Z values mapped to their counters.
+   */
+  public const DISPOSITIONS = [
+    'Affirmed on Appeal' => 'affirmed',
+    'Partially Affirmed & Partially Reversed/Remanded' => 'partial',
+    'Completely Reversed/Remanded' => 'reversed',
+    'Closed for Other Reasons' => 'other',
+  ];
+
+  /**
+   * Counts each nonblank disposition once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component counts keyed by entity ID and overall counts, keyed by outcome.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_values(self::DISPOSITIONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal disposition counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $code = trim($columns[25]);
+          if ($code === '') {
+            continue;
+          }
+          // Reject unmapped values rather than silently omitting requests.
+          if (!isset(self::DISPOSITIONS[$code])) {
+            throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown Appeal Disposition in Column Z.', $id, $record));
+          }
+          $components[$id][self::DISPOSITIONS[$code]]++;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal disposition counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}

docroot/modules/custom/foia_raw_data_to_report/src/AppealNonExemptionDenialAggregator.php

@@ -0,0 +1,92 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Column AA into appeal non-exemption denial counts.
+ */
+final class AppealNonExemptionDenialAggregator {
+
+  /**
+   * Column AA reason labels mapped to annual report XML codes.
+   */
+  public const REASONS = [
+    'No Records' => 'NoRecords',
+    'Records Referred at the Initial Request Level' => 'Referred',
+    'Request Withdrawn' => 'Withdrawn',
+    'Fee-Related Reason' => 'FeeRelated',
+    'Records not Reasonably Described' => 'NotDescribed',
+    'Improper Request for Other Reasons' => 'ImproperRequest',
+    'Not an Agency Record' => 'NotAgency',
+    'Duplicate Request or Appeal' => 'Duplicate',
+    'Request in Litigation' => 'InLitigation',
+    'Appeal Based Solely on Denial of Request for Expedited Processing' => 'ExpeditedDenial',
+    'Other' => 'Other',
+  ];
+
+  /**
+   * Counts each distinct reason once per row, then sums component counts.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   *
+   * @return array
+   *   Component and overall counts keyed by XML reason code.
+   */
+  public function aggregate(array $sources): array {
+    $empty = array_fill_keys(array_values(self::REASONS), 0);
+    $components = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $components[$id] ??= $empty;
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal non-exemption denial counts.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $value = trim($columns[26]);
+          if ($value === '') {
+            continue;
+          }
+          // Split comma-separated reasons and count repeats once per appeal.
+          $reasons = array_unique(array_map('trim', explode(',', $value)));
+          foreach ($reasons as $reason) {
+            // Unknown labels must not silently disappear from the report.
+            if (!isset(self::REASONS[$reason])) {
+              throw new \RuntimeException(sprintf('Component %s, CSV record %d: Unknown appeal denial reason "%s" in Column AA.', $id, $record, $reason));
+            }
+            $components[$id][self::REASONS[$reason]]++;
+          }
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal non-exemption denial counts.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $overall = $empty;
+    foreach ($components as $counts) {
+      foreach ($counts as $code => $quantity) {
+        $overall[$code] += $quantity;
+      }
+    }
+    return ['components' => $components, 'overall' => $overall];
+  }
+
+}

docroot/modules/custom/foia_raw_data_to_report/src/AppealResponseTimeAggregator.php

@@ -0,0 +1,133 @@
+<?php
+
+namespace Drupal\foia_raw_data_to_report;
+
+/**
+ * Streams Columns X and Y into calendar-day response time distributions.
+ */
+final class AppealResponseTimeAggregator {
+
+  /**
+   * Calculates component and agency statistics without retaining CSV rows.
+   *
+   * @param array $sources
+   *   Component/file pairs, each with component_id and uri keys.
+   * @param int $fiscal_year
+   *   The validated report year.
+   *
+   * @return array
+   *   Component and overall median, average, lowest, and highest values.
+   */
+  public function aggregate(array $sources, int $fiscal_year): array {
+    $timezone = new \DateTimeZone('UTC');
+    $start = new \DateTimeImmutable(($fiscal_year - 1) . '-10-01', $timezone);
+    $end = new \DateTimeImmutable($fiscal_year . '-09-30', $timezone);
+    $histograms = [];
+    foreach ($sources as $source) {
+      $id = $source['component_id'];
+      $histograms[$id] ??= [];
+      $stream = @fopen($source['uri'], 'rb');
+      if ($stream === FALSE) {
+        throw new \RuntimeException('Unable to reopen a validated CSV for appeal response times.');
+      }
+      try {
+        $header = TRUE;
+        $record = 0;
+        while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) {
+          $record++;
+          if ($columns === [NULL]) {
+            continue;
+          }
+          if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) {
+            throw new \RuntimeException('CSV column count changed after validation.');
+          }
+          if ($header) {
+            $header = FALSE;
+            continue;
+          }
+          $context = sprintf('Component %s, CSV record %d', $id, $record);
+          // Only rows with an appeal contribute to response time statistics.
+          $received_text = trim($columns[23]);
+          $completed = trim($columns[24]);
+          if ($received_text === '') {
+            if ($completed !== '') {
+              throw new \RuntimeException($context . ': Appeal Date Closed requires an Appeal Date Received.');
+            }
+            continue;
+          }
+          $received = $this->calendarDate($received_text, $context . ', Column X');
+          $finish = $completed === '' ? $end : $this->calendarDate($completed, $context . ', Column Y');
+          $begin = $received > $start ? $received : $start;
+          if ($finish < $begin || $finish > $end) {
+            throw new \RuntimeException($context . ': Response time dates must be in chronological order and within the report fiscal year.');
+          }
+          // Elapsed calendar days exclude the starting day; same-day is zero.
+          // UTC dates avoid daylight-saving changes affecting the calculation.
+          $days = (int) $begin->diff($finish)->days;
+          $histograms[$id][$days] = ($histograms[$id][$days] ?? 0) + 1;
+        }
+        if (!feof($stream)) {
+          throw new \RuntimeException('Unable to finish reading a CSV for appeal response times.');
+        }
+      }
+      finally {
+        fclose($stream);
+      }
+    }
+    $components = [];
+    $overall = [];
+    foreach ($histograms as $id => $histogram) {
+      $components[$id] = $this->summarize($histogram);
+      // Combine frequencies so agency statistics are weighted by appeal count.
+      foreach ($histogram as $days => $count) {
+        $overall[$days] = ($overall[$days] ?? 0) + $count;
+      }
+    }
+    return ['components' => $components, 'overall' => $this->summarize($overall)];
+  }
+
+  /**
+   * Parses calendar dates defensively if a file changed after validation.
+   */
+  private function calendarDate(string $value, string $context): \DateTimeImmutable {
+    if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts)
+      || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) {
+      throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.');
+    }
+    return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC'));
+  }
+
+  /**
+   * Finds exact medians and weighted averages from day frequencies.
+   */
+  private function summarize(array $histogram): array {
+    if ($histogram === []) {
+      return ['median' => 0, 'average' => 0, 'lowest' => 0, 'highest' => 0];
+    }
+    ksort($histogram, SORT_NUMERIC);
+    $count = array_sum($histogram);
+    $lower_position = intdiv($count + 1, 2);
+    $upper_position = intdiv($count, 2) + 1;
+    $seen = 0;
+    $sum = 0;
+    $lower = NULL;
+    $upper = NULL;
+    foreach ($histogram as $days => $frequency) {
+      $seen += $frequency;
+      $sum += $days * $frequency;
+      if ($lower === NULL && $seen >= $lower_position) {
+        $lower = $days;
+      }
+      if ($upper === NULL && $seen >= $upper_position) {
+        $upper = $days;
+      }
+    }
+    return [
+      'median' => ($lower + $upper) / 2,
+      'average' => round($sum / $count, 2),
+      'lowest' => array_key_first($histogram),
+      'highest' => array_key_last($histogram),
+    ];
+  }
+
+}