diff --git a/docroot/modules/custom/foia_raw_data_to_report/README.md b/docroot/modules/custom/foia_raw_data_to_report/README.md index b1862ec3c..ee25a933e 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/README.md +++ b/docroot/modules/custom/foia_raw_data_to_report/README.md @@ -173,7 +173,7 @@ header; its names are not validated. Data records currently require: date, in which case uppercase `G` or `D` is required. Surrounding whitespace is ignored. -`CsvValidator::FEDERAL_HOLIDAYS` embeds all 205 dates supplied in +`WorkingDays::FEDERAL_HOLIDAYS` embeds all 205 dates supplied in `federal-holidays.txt`, covering 2008–2026. The file is not read at runtime. Maintain the constant for future years; dates outside its coverage currently exclude weekends only. Working-day arithmetic counts whole weeks and a short @@ -503,3 +503,124 @@ Same-day receipt is zero days. XML receipt dates use `YYYY-MM-DD`. `OldestPendingAppealSection` follows the appeal response times, with `OPA1`, `OPA2`, etc. linked to components and `OPA0` to the agency. Components without pending appeals retain an empty container and their organization association. + +## Processed request response times + +`ProcessedResponseTimeAggregator` uses completed rows (K) with S, C, or E in M. +The start is J when present, otherwise I. No fiscal-year clamping or Days Tolled +subtraction is applied. A counted row without either start date, or with an end +before its start, stops generation with a contextual error. + +`WorkingDays` shares the existing federal holiday constant and calculation with +CSV validation: exclude the start day, include the end day, skip weekends and +listed holidays. The calendar currently covers 2008–2026 and must be maintained. +Day-frequency maps provide exact medians, averages, minima and maxima without +retaining individual rows. Agency statistics use combined frequencies per track. + +`ProcessedResponseTimeSection` follows oldest pending appeals. `PRT1`, `PRT2`, +etc. reference component organizations; `PRT0` references the agency. Each has +Simple, Complex and Expedited response-time elements, empty for unused tracks. +Statistics below one use the corresponding `DaysCode` element with `LT1`; +otherwise `DaysValue` is used, with averages formatted to two decimal places. +The less-than-one comparison occurs before average rounding. + +Report-generation exceptions are appended after validation messages in the +node's Messages field as plain text. The worker reloads the stored node before +saving the message so unsaved file-field changes are not persisted accidentally. +The exception is rethrown to preserve Drush logging and queue retry behavior. +Each new attempt clears previous messages as before. + +## Information granted response times + +`InformationGrantedResponseTimeSection` uses the same accumulator and XML writer +as processed response times, with `information_granted_only` enabled to select +Column N codes 1 and 2. Completed dates, track selection, working days, empty +tracks, agency weighting, two-decimal averages, and `LT1` handling are unchanged. +The section follows processed response times and uses `IGRT1`, `IGRT2`, etc. +for components and `IGRT0` for the agency. Associations retain the example's +`ProcessedResponseTimeOrganizationAssociation` element name. + +## Simple response-time increments + +`aggregateSimpleIncrements()` shares the processed-response CSV reader and +working-day calculation. It includes completed requests with Track S and +Disposition 1 or 2. Zero-day completions count in 1–20. The thirteen bins +include both endpoints, followed by the open-ended 401+ bin. Agency counts +sum component counts. + +`SimpleResponseTimeIncrementsSection` follows information-granted response +times. Every component and the agency have all thirteen `TimeIncrement` +entries (including zero counts), followed by `TimeIncrementTotalQuantity`. +`SRT1`, `SRT2`, etc. refer to components and `SRT0` refers to the agency. + +## Complex response-time increments + +`aggregateComplexIncrements()` uses the same thirteen bins and working-day +calculation as simple increments, selecting Track C instead of S. Completed +requests with disposition 1 or 2 contribute; zero-day responses count in 1–20. +All bins, including zero counts, and their sum are emitted for every component +and the agency. `ComplexResponseTimeIncrementsSection` follows the simple +section and uses `CRT1`, `CRT2`, etc. and agency `CRT0` organization references. + +## Expedited response-time increments + +`aggregateExpeditedIncrements()` selects Track E, using the same calculation, +disposition 1/2 filter, thirteen bins, zero counts, and totals as simple and +complex increments. Zero-day responses count in 1–20. +`ExpeditedResponseTimeIncrementsSection` follows the complex section, with +`ERT1`, `ERT2`, etc. linked to components and `ERT0` linked to the agency. + +## Pending perfected requests + +`PendingPerfectedRequestsAggregator` includes rows with J populated and K blank, +separated by Track S, C, and E. There is no disposition filter. Age is working +days from J through September 30 of the report year, excluding the start day +and including the end day, without fiscal-start clamping or toll subtraction. +The shared `WorkingDays` calendar excludes weekends and federal holidays. + +Frequency maps provide counts, exact medians, and averages per component and +across all agency rows. Empty tracks emit quantity 0 and literal `N/A` in both +median/average value elements; populated averages have two decimals. Same-day +ages remain numeric zero. `PendingPerfectedRequestsSection` follows expedited +increments and uses `PPR1`, `PPR2`, etc. and agency `PPR0` organization links. + +## Oldest pending requests + +`OldestPendingRequestAggregator` selects rows with I populated and K blank, +without track or disposition filters. Pending age uses the shared working-day +calendar from the actual received date through fiscal year-end, excluding the +start day and including the end day. Prior-year time is included. + +At most ten items are retained per component and for the entire agency, sorted +by descending pending days, then ascending receipt date for ties. Duplicate +rows remain separate entries. Empty components retain their empty container +and association. `OldestPendingRequestSection` follows pending perfected +requests, with ISO receipt dates and `OPR1`, `OPR2`, etc.; `OPR0` is the agency. +The existing oldest-appeal section continues using calendar days. + +## Expedited processing + +`ExpeditedProcessingAggregator` counts uppercase G and D in Column S as granted +and denied. With both Q and R present, adjudications taking at most ten working +days also increment the timely counter. The shared calendar excludes the start +day, includes the end day, and skips weekends and listed federal holidays. +Same-day determinations qualify. Blank R still counts the outcome, but not a +timely adjudication. Invalid or reversed dates produce contextual exceptions. + +`ExpeditedProcessingSection` follows oldest pending requests. All three counts, +including zeros, appear for each component and the agency. `EP1`, `EP2`, etc. +and `EP0` link to their Organization entries. Agency counts sum components. + +## Fee waivers + +`FeeWaiverAggregator` counts uppercase G and D in Column V for every component +and sums those counts for the agency. Other values and blank cells do not +contribute. No date, track, or disposition filter is applied. +`FeeWaiverSection` follows expedited processing, emitting both granted and +denied quantities even when zero. `FW1`, `FW2`, etc. reference components; +`FW0` references the agency through `FeeWaiverOrganizationAssociation`. + +For appeal rows (Column X populated), CSV validation requires E–P and T–W +to be blank. Q, R, and S may contain expedited-processing data and retain +their date and dependency checks. Appeal rows are exempt from the rule that +S = G requires M = E, since M must remain blank on these rows. diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php index d5fcb5079..7773388b5 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php +++ b/docroot/modules/custom/foia_raw_data_to_report/src/CsvValidator.php @@ -12,219 +12,6 @@ final class CsvValidator { */ public const EXPECTED_COLUMNS = 29; - /** - * Holiday dates covering 2008 through 2026. - * - * Keep this list updated for future reporting years. - */ - private const FEDERAL_HOLIDAYS = [ - '2008-01-01', - '2008-01-21', - '2008-02-18', - '2008-05-26', - '2008-07-04', - '2008-09-01', - '2008-10-13', - '2008-11-11', - '2008-11-27', - '2008-12-25', - '2009-01-01', - '2009-01-19', - '2009-01-20', - '2009-02-16', - '2009-05-25', - '2009-07-03', - '2009-09-07', - '2009-10-12', - '2009-11-11', - '2009-11-26', - '2009-12-25', - '2010-01-01', - '2010-01-18', - '2010-02-15', - '2010-05-31', - '2010-07-05', - '2010-09-06', - '2010-10-11', - '2010-11-11', - '2010-11-25', - '2010-12-24', - '2010-12-31', - '2011-01-17', - '2011-02-21', - '2011-05-30', - '2011-07-04', - '2011-09-05', - '2011-10-10', - '2011-11-11', - '2011-11-24', - '2011-12-26', - '2012-01-02', - '2012-01-16', - '2012-02-20', - '2012-05-28', - '2012-07-04', - '2012-09-03', - '2012-10-08', - '2012-11-12', - '2012-11-22', - '2012-12-25', - '2013-01-01', - '2013-01-21', - '2013-02-18', - '2013-05-27', - '2013-07-04', - '2013-09-02', - '2013-10-14', - '2013-11-11', - '2013-11-28', - '2013-12-25', - '2014-01-01', - '2014-01-20', - '2014-02-17', - '2014-05-26', - '2014-07-04', - '2014-09-01', - '2014-10-13', - '2014-11-11', - '2014-11-27', - '2014-12-24', - '2014-12-25', - '2015-01-01', - '2015-01-19', - '2015-02-16', - '2015-05-25', - '2015-07-03', - '2015-09-07', - '2015-10-12', - '2015-11-11', - '2015-11-26', - '2015-12-25', - '2016-01-01', - '2016-01-18', - '2016-02-15', - '2016-05-30', - '2016-07-04', - '2016-09-05', - '2016-10-10', - '2016-11-11', - '2016-11-24', - '2016-12-26', - '2017-01-02', - '2017-01-16', - '2017-02-20', - '2017-05-29', - '2017-07-04', - '2017-09-04', - '2017-10-09', - '2017-11-10', - '2017-11-23', - '2017-12-25', - '2018-01-01', - '2018-01-15', - '2018-02-19', - '2018-05-28', - '2018-07-04', - '2018-09-03', - '2018-10-08', - '2018-11-12', - '2018-11-22', - '2018-12-24', - '2018-12-25', - '2019-01-01', - '2019-01-21', - '2019-02-18', - '2019-05-27', - '2019-07-04', - '2019-09-02', - '2019-10-14', - '2019-11-11', - '2019-11-28', - '2019-12-24', - '2019-12-25', - '2020-01-01', - '2020-01-20', - '2020-02-17', - '2020-04-03', - '2020-05-25', - '2020-09-07', - '2020-10-12', - '2020-11-11', - '2020-11-26', - '2020-12-25', - '2021-01-01', - '2021-01-18', - '2021-01-20', - '2021-02-15', - '2021-05-31', - '2021-06-18', - '2021-07-05', - '2021-09-06', - '2021-10-11', - '2021-11-11', - '2021-11-25', - '2021-12-24', - '2021-12-31', - '2022-01-17', - '2022-02-21', - '2022-05-30', - '2022-06-20', - '2022-07-04', - '2022-09-05', - '2022-10-10', - '2022-11-11', - '2022-11-24', - '2022-12-26', - '2023-01-02', - '2023-01-16', - '2023-02-20', - '2023-05-29', - '2023-06-19', - '2023-07-04', - '2023-09-04', - '2023-10-09', - '2023-11-10', - '2023-11-23', - '2023-12-25', - '2024-01-01', - '2024-01-15', - '2024-02-19', - '2024-05-27', - '2024-06-19', - '2024-07-04', - '2024-09-02', - '2024-10-14', - '2024-11-11', - '2024-11-28', - '2024-12-24', - '2024-12-25', - '2025-01-01', - '2025-01-09', - '2025-01-20', - '2025-02-17', - '2025-05-26', - '2025-06-19', - '2025-07-04', - '2025-09-01', - '2025-10-13', - '2025-11-11', - '2025-11-27', - '2025-12-24', - '2025-12-25', - '2025-12-26', - '2026-01-01', - '2026-01-19', - '2026-02-16', - '2026-05-25', - '2026-06-19', - '2026-07-03', - '2026-09-07', - '2026-10-12', - '2026-11-11', - '2026-11-26', - '2026-12-25', - ]; - /** * Returns human-readable errors, or an empty array when validation passes. * @@ -268,12 +55,12 @@ public function validate(string $uri, int $fiscal_year): array { continue; } - // Column X: Appeal rows must not contain request data in E through W. + // Column X: Appeal rows require E-P and T-W blank, but allow Q, R, S. $appeal_received = trim($columns[23]); if ($appeal_received !== '') { - foreach (array_slice($columns, 4, 19) as $value) { + foreach (array_merge(array_slice($columns, 4, 12), array_slice($columns, 19, 4)) as $value) { if (trim($value) !== '') { - return [sprintf('CSV record %d: If there is data in Column X, Columns E through W must be empty.', $record)]; + return [sprintf('CSV record %d: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.', $record)]; } } } @@ -473,8 +260,8 @@ public function validate(string $uri, int $fiscal_year): array { // Column M: Track may be blank. The Column J check above already // requires uppercase S, C or E whenever a perfected date is present. - // Independently, an uppercase G in S requires an uppercase E in M. - if (trim($columns[18]) === 'G' && trim($columns[12]) !== 'E') { + // Non-appeal rows with G in S require E in M. Appeal rows keep M blank. + if ($appeal_received === '' && trim($columns[18]) === 'G' && trim($columns[12]) !== 'E') { return [sprintf('CSV record %d: Column M: If Column S contains a G, Column M must contain an E', $record)]; } @@ -639,29 +426,7 @@ public function validate(string $uri, int $fiscal_year): array { * effects. Whole weeks avoid iterating over every day of a long request. */ private function countWorkingDays(string $start, string $end): int { - $timezone = new \DateTimeZone('UTC'); - $start_date = \DateTimeImmutable::createFromFormat('!m/d/Y', $start, $timezone); - $end_date = \DateTimeImmutable::createFromFormat('!m/d/Y', $end, $timezone); - $days = (int) $start_date->diff($end_date)->days; - $working_days = intdiv($days, 7) * 5; - $start_weekday = (int) $start_date->format('N'); - for ($offset = 1; $offset <= $days % 7; $offset++) { - if (($start_weekday - 1 + $offset) % 7 < 5) { - $working_days++; - } - } - - $first = $start_date->format('Y-m-d'); - $last = $end_date->format('Y-m-d'); - foreach (self::FEDERAL_HOLIDAYS as $holiday) { - if ($holiday > $last) { - break; - } - if ($holiday > $first && (int) (new \DateTimeImmutable($holiday, $timezone))->format('N') <= 5) { - $working_days--; - } - } - return $working_days; + return (new WorkingDays())->count($start, $end); } } diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php new file mode 100644 index 000000000..87ce2ea5d --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/ExpeditedProcessingAggregator.php @@ -0,0 +1,101 @@ + 'granted', 'D' => 'denied']; + + /** + * Counts granted, denied, and adjudications within ten working days. + * + * @param array $sources + * Component/file pairs, each with component_id and uri keys. + * + * @return array + * Component counts keyed by entity ID and overall counts, keyed by outcome. + */ + public function aggregate(array $sources): array { + $empty = ['granted' => 0, 'denied' => 0, 'within_ten' => 0]; + $working_days = new WorkingDays(); + $components = []; + foreach ($sources as $source) { + $id = $source['component_id']; + $components[$id] ??= $empty; + $stream = @fopen($source['uri'], 'rb'); + if ($stream === FALSE) { + throw new \RuntimeException('Unable to reopen a validated CSV for expedited processing counts.'); + } + try { + $header = TRUE; + $record = 0; + while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) { + $record++; + if ($columns === [NULL]) { + continue; + } + if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) { + throw new \RuntimeException('CSV column count changed after validation.'); + } + if ($header) { + $header = FALSE; + continue; + } + // Only uppercase G and D outcomes in Column S contribute. + $outcome = trim($columns[18]); + if (!isset(self::OUTCOMES[$outcome])) { + continue; + } + $components[$id][self::OUTCOMES[$outcome]]++; + $received = trim($columns[16]); + $determined = trim($columns[17]); + $context = sprintf('Component %s, CSV %s, record %d', $id, basename($source['uri']), $record); + $start = $this->calendarDate($received, $context . ', Column Q'); + // R may be blank: count the outcome, but no adjudication interval. + if ($determined === '') { + continue; + } + $end = $this->calendarDate($determined, $context . ', Column R'); + if ($end < $start) { + throw new \RuntimeException($context . ': Column R cannot precede Column Q.'); + } + // Exclude the start day and include the end day; ten is inclusive. + if ($working_days->count($received, $determined) <= 10) { + $components[$id]['within_ten']++; + } + } + if (!feof($stream)) { + throw new \RuntimeException('Unable to finish reading a CSV for expedited processing counts.'); + } + } + finally { + fclose($stream); + } + } + $overall = $empty; + foreach ($components as $counts) { + foreach ($counts as $code => $quantity) { + $overall[$code] += $quantity; + } + } + return ['components' => $components, 'overall' => $overall]; + } + + /** + * Parses calendar dates defensively if a file changed after validation. + */ + private function calendarDate(string $value, string $context): \DateTimeImmutable { + if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts) + || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) { + throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.'); + } + return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC')); + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/FeeWaiverAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/FeeWaiverAggregator.php new file mode 100644 index 000000000..ad29c8bac --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/FeeWaiverAggregator.php @@ -0,0 +1,70 @@ + 'granted', 'D' => 'denied']; + + /** + * Counts each G or D once per row, then sums component counts. + * + * @param array $sources + * Component/file pairs, each with component_id and uri keys. + * + * @return array + * Component counts keyed by entity ID and overall counts, keyed by outcome. + */ + public function aggregate(array $sources): array { + $empty = array_fill_keys(array_values(self::OUTCOMES), 0); + $components = []; + foreach ($sources as $source) { + $id = $source['component_id']; + $components[$id] ??= $empty; + $stream = @fopen($source['uri'], 'rb'); + if ($stream === FALSE) { + throw new \RuntimeException('Unable to reopen a validated CSV for fee-waiver counts.'); + } + try { + $header = TRUE; + while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) { + if ($columns === [NULL]) { + continue; + } + if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) { + throw new \RuntimeException('CSV column count changed after validation.'); + } + if ($header) { + $header = FALSE; + continue; + } + // Count only G and D in V, independently of dates and other outcomes. + $outcome = trim($columns[21]); + if (isset(self::OUTCOMES[$outcome])) { + $components[$id][self::OUTCOMES[$outcome]]++; + } + } + if (!feof($stream)) { + throw new \RuntimeException('Unable to finish reading a CSV for fee-waiver counts.'); + } + } + finally { + fclose($stream); + } + } + $overall = $empty; + foreach ($components as $counts) { + foreach ($counts as $code => $quantity) { + $overall[$code] += $quantity; + } + } + return ['components' => $components, 'overall' => $overall]; + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php new file mode 100644 index 000000000..a5f5df04b --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/OldestPendingRequestAggregator.php @@ -0,0 +1,100 @@ +calendarDate($received_text, $context); + if ($received > $end) { + throw new \RuntimeException($context . ': Date Initially Received is after the report fiscal year.'); + } + // Use elapsed working days, including time in prior fiscal years. + $item = [ + 'receipt_date' => $received->format('Y-m-d'), + 'pending_days' => $working_days->count($received_text, $end->format('m/d/Y')), + ]; + $this->retainOldest($components[$id], $item); + $this->retainOldest($overall, $item); + } + if (!feof($stream)) { + throw new \RuntimeException('Unable to finish reading a CSV for oldest pending requests.'); + } + } + finally { + fclose($stream); + } + } + return ['components' => $components, 'overall' => $overall]; + } + + /** + * Keeps the ten highest ages, breaking ties by the earliest receipt date. + */ + private function retainOldest(array &$items, array $item): void { + $items[] = $item; + // Preserve duplicate rows; equal working-day ages can span weekends. + usort($items, static fn(array $a, array $b): int => ($b['pending_days'] <=> $a['pending_days']) ?: strcmp($a['receipt_date'], $b['receipt_date'])); + if (count($items) > 10) { + array_pop($items); + } + } + + /** + * Parses calendar dates defensively if a file changed after validation. + */ + private function calendarDate(string $value, string $context): \DateTimeImmutable { + if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts) + || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) { + throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.'); + } + return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC')); + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php new file mode 100644 index 000000000..781a6fbde --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/PendingPerfectedRequestsAggregator.php @@ -0,0 +1,143 @@ + 'SimplePendingRequestStatistics', + 'C' => 'ComplexPendingRequestStatistics', + 'E' => 'ExpeditedPendingRequestStatistics', + ]; + + /** + * Calculates component and agency counts, medians, and averages. + * + * @param array $sources + * Component/file pairs, each with component_id and uri keys. + * @param int $fiscal_year + * The validated report year, ending September 30. + * + * @return array + * Component and overall statistics keyed by track. + */ + public function aggregate(array $sources, int $fiscal_year): array { + $end = new \DateTimeImmutable($fiscal_year . '-09-30', new \DateTimeZone('UTC')); + $working_days = new WorkingDays(); + $empty = array_fill_keys(array_keys(self::TRACKS), []); + $histograms = []; + foreach ($sources as $source) { + $id = $source['component_id']; + $histograms[$id] ??= $empty; + $stream = @fopen($source['uri'], 'rb'); + if ($stream === FALSE) { + throw new \RuntimeException('Unable to reopen a validated CSV for pending perfected requests.'); + } + try { + $header = TRUE; + $record = 0; + while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) { + $record++; + if ($columns === [NULL]) { + continue; + } + if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) { + throw new \RuntimeException('CSV column count changed after validation.'); + } + if ($header) { + $header = FALSE; + continue; + } + // Include only perfected requests that have not been completed. + $perfected = trim($columns[9]); + if ($perfected === '' || trim($columns[10]) !== '') { + continue; + } + $context = sprintf('Component %s, CSV %s, record %d', $id, basename($source['uri']), $record); + $track = trim($columns[12]); + if (!isset(self::TRACKS[$track])) { + throw new \RuntimeException($context . ': Column M must contain S, C, or E for a perfected request.'); + } + $start = $this->calendarDate($perfected, $context . ', Column J'); + if ($start > $end) { + throw new \RuntimeException($context . ': Column J Date Perfected is later than the fiscal year.'); + } + // Measure from the actual perfected date, including prior years. + $days = $working_days->count($perfected, $end->format('m/d/Y')); + $histograms[$id][$track][$days] = ($histograms[$id][$track][$days] ?? 0) + 1; + } + if (!feof($stream)) { + throw new \RuntimeException('Unable to finish reading a CSV for pending perfected requests.'); + } + } + finally { + fclose($stream); + } + } + $components = []; + $overall = $empty; + foreach ($histograms as $id => $tracks) { + foreach ($tracks as $track => $histogram) { + $components[$id][$track] = $this->summarize($histogram); + // Combine frequencies, rather than averaging component statistics. + foreach ($histogram as $days => $count) { + $overall[$track][$days] = ($overall[$track][$days] ?? 0) + $count; + } + } + } + foreach ($overall as $track => $histogram) { + $overall[$track] = $this->summarize($histogram); + } + return ['components' => $components, 'overall' => $overall]; + } + + /** + * Parses calendar dates defensively if a file changed after validation. + */ + private function calendarDate(string $value, string $context): \DateTimeImmutable { + if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts) + || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) { + throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.'); + } + return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC')); + } + + /** + * Finds exact medians and weighted averages from day frequencies. + */ + private function summarize(array $histogram): array { + if ($histogram === []) { + return ['quantity' => 0, 'median' => NULL, 'average' => NULL]; + } + ksort($histogram, SORT_NUMERIC); + $count = array_sum($histogram); + $lower_position = intdiv($count + 1, 2); + $upper_position = intdiv($count, 2) + 1; + $seen = 0; + $sum = 0; + $lower = NULL; + $upper = NULL; + foreach ($histogram as $days => $frequency) { + $seen += $frequency; + $sum += $days * $frequency; + if ($lower === NULL && $seen >= $lower_position) { + $lower = $days; + } + if ($upper === NULL && $seen >= $upper_position) { + $upper = $days; + } + } + return [ + 'quantity' => $count, + 'median' => ($lower + $upper) / 2, + 'average' => $sum / $count, + ]; + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php index d756bbfd5..0e7f13486 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php +++ b/docroot/modules/custom/foia_raw_data_to_report/src/Plugin/QueueWorker/RawDataToReportProcessing.php @@ -22,6 +22,11 @@ use Drupal\foia_raw_data_to_report\AppealStatisticsAggregator; use Drupal\foia_raw_data_to_report\AppealResponseTimeAggregator; use Drupal\foia_raw_data_to_report\OldestPendingAppealAggregator; +use Drupal\foia_raw_data_to_report\OldestPendingRequestAggregator; +use Drupal\foia_raw_data_to_report\ExpeditedProcessingAggregator; +use Drupal\foia_raw_data_to_report\FeeWaiverAggregator; +use Drupal\foia_raw_data_to_report\ProcessedResponseTimeAggregator; +use Drupal\foia_raw_data_to_report\PendingPerfectedRequestsAggregator; use Drupal\foia_raw_data_to_report\AppealDispositionAggregator; use Drupal\foia_raw_data_to_report\AppealNonExemptionDenialAggregator; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -141,7 +146,24 @@ public function processItem($data) { // Invalid input is a completed task, not a retryable exception. return; } - $this->generateXmlReport($node); + try { + $this->generateXmlReport($node); + } + catch (\Throwable $exception) { + // Reload persisted state: generation may have changed the XML field in + // memory before failing. Save only messages alongside the stored file. + $stored_node = $storage->loadUnchanged($node->id()); + if ($stored_node instanceof NodeInterface) { + $validation_messages = (string) $stored_node->get('field_messages')->value; + $stored_node->set('field_messages', [ + 'value' => $validation_messages . "\n\nXML report processing failed: " . $exception->getMessage(), + 'format' => 'plain_text', + ]); + $stored_node->save(); + } + // Preserve Drush logging and the queue's existing retry behavior. + throw $exception; + } } /** @@ -172,7 +194,16 @@ protected function generateXmlReport(NodeInterface $node): void { $appeal_other_reasons = (new OtherDenialReasonAggregator())->aggregate($sources, 27); $appeal_response_times = (new AppealResponseTimeAggregator())->aggregate($sources, $fiscal_year); $oldest_pending_appeals = (new OldestPendingAppealAggregator())->aggregate($sources, $fiscal_year); - $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times, $oldest_pending_appeals); + $processed_response_times = (new ProcessedResponseTimeAggregator())->aggregate($sources); + $information_granted_response_times = (new ProcessedResponseTimeAggregator())->aggregate($sources, TRUE); + $simple_response_increments = (new ProcessedResponseTimeAggregator())->aggregateSimpleIncrements($sources); + $complex_response_increments = (new ProcessedResponseTimeAggregator())->aggregateComplexIncrements($sources); + $expedited_response_increments = (new ProcessedResponseTimeAggregator())->aggregateExpeditedIncrements($sources); + $pending_perfected_requests = (new PendingPerfectedRequestsAggregator())->aggregate($sources, $fiscal_year); + $oldest_pending_requests = (new OldestPendingRequestAggregator())->aggregate($sources, $fiscal_year); + $expedited_processing = (new ExpeditedProcessingAggregator())->aggregate($sources); + $fee_waivers = (new FeeWaiverAggregator())->aggregate($sources); + $xml = (new XmlReportBuilder())->build($node->get('field_agency')->entity, $components, $fiscal_year, $statutes, $request_statistics, $dispositions, $other_reasons, $applied_exemptions, $appeal_statistics, $appeal_dispositions, $appeal_exemptions, $appeal_denials, $appeal_other_reasons, $appeal_response_times, $oldest_pending_appeals, $processed_response_times, $information_granted_response_times, $simple_response_increments, $complex_response_increments, $expedited_response_increments, $pending_perfected_requests, $oldest_pending_requests, $expedited_processing, $fee_waivers); $field = $node->get('field_request_data_xml'); $previous_file = $field->entity; $item = $field->first() ?? $field->appendItem(); diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php new file mode 100644 index 000000000..5543f81e2 --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/ProcessedResponseTimeAggregator.php @@ -0,0 +1,223 @@ + 'SimpleResponseTime', + 'C' => 'ComplexResponseTime', + 'E' => 'ExpeditedResponseTime', + ]; + + /** + * Returns component and agency summaries without retaining individual rows. + * + * @param array $sources + * Component/file pairs, each with component_id and uri keys. + * @param bool $information_granted_only + * Whether to include only disposition codes 1 and 2 in Column N. + * + * @return array + * Component and overall statistics keyed by track, empty for unused tracks. + */ + public function aggregate(array $sources, bool $information_granted_only = FALSE): array { + $histograms = $this->collectHistograms($sources, $information_granted_only); + $empty = array_fill_keys(array_keys(self::TRACKS), []); + $components = []; + $overall = $empty; + foreach ($histograms as $id => $tracks) { + foreach ($tracks as $track => $histogram) { + $components[$id][$track] = $this->summarize($histogram); + foreach ($histogram as $days => $count) { + $overall[$track][$days] = ($overall[$track][$days] ?? 0) + $count; + } + } + } + foreach ($overall as $track => $histogram) { + $overall[$track] = $this->summarize($histogram); + } + return ['components' => $components, 'overall' => $overall]; + } + + /** + * Upper bounds for the ordered response-time bins; the last is unbounded. + */ + public const INCREMENTS = [ + '1-20' => 20, + '21-40' => 40, + '41-60' => 60, + '61-80' => 80, + '81-100' => 100, + '101-120' => 120, + '121-140' => 140, + '141-160' => 160, + '161-180' => 180, + '181-200' => 200, + '201-300' => 300, + '301-400' => 400, + '401+' => PHP_INT_MAX, + ]; + + /** + * Counts simple information-granted requests in all thirteen day ranges. + */ + public function aggregateSimpleIncrements(array $sources): array { + return $this->aggregateIncrements($sources, 'S'); + } + + /** + * Counts complex information-granted requests in all thirteen day ranges. + */ + public function aggregateComplexIncrements(array $sources): array { + return $this->aggregateIncrements($sources, 'C'); + } + + /** + * Counts expedited information-granted requests in all thirteen day ranges. + */ + public function aggregateExpeditedIncrements(array $sources): array { + return $this->aggregateIncrements($sources, 'E'); + } + + /** + * Bins completed information-granted requests for the selected track. + */ + private function aggregateIncrements(array $sources, string $track): array { + $histograms = $this->collectHistograms($sources, TRUE, $track); + $empty = array_fill_keys(array_keys(self::INCREMENTS), 0); + $components = []; + $overall = $empty; + foreach ($histograms as $id => $tracks) { + $components[$id] = $empty; + // Same-day completions belong in 1-20 so every selected row counts. + foreach ($tracks[$track] as $days => $count) { + foreach (self::INCREMENTS as $code => $upper) { + if ($days <= $upper) { + $components[$id][$code] += $count; + $overall[$code] += $count; + break; + } + } + } + } + return ['components' => $components, 'overall' => $overall]; + } + + /** + * Shares CSV selection and working-day calculation across both outputs. + */ + private function collectHistograms(array $sources, bool $information_granted_only, ?string $only_track = NULL): array { + $empty = array_fill_keys(array_keys(self::TRACKS), []); + $histograms = []; + $working_days = new WorkingDays(); + foreach ($sources as $source) { + $id = $source['component_id']; + $histograms[$id] ??= $empty; + $stream = @fopen($source['uri'], 'rb'); + if ($stream === FALSE) { + throw new \RuntimeException('Unable to reopen a validated CSV for processed response times.'); + } + try { + $header = TRUE; + $record = 0; + while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) { + $record++; + if ($columns === [NULL]) { + continue; + } + if (count($columns) !== CsvValidator::EXPECTED_COLUMNS) { + throw new \RuntimeException('CSV column count changed after validation.'); + } + if ($header) { + $header = FALSE; + continue; + } + // Information-granted statistics include only full/partial grants. + if ($information_granted_only && !in_array(trim($columns[13]), ['1', '2'], TRUE)) { + continue; + } + // Only completed requests in one of the three tracks contribute. + $completed = trim($columns[10]); + $track = trim($columns[12]); + if ($completed === '' || !isset(self::TRACKS[$track]) || ($only_track !== NULL && $track !== $only_track)) { + continue; + } + // Prefer J; fall back to I without clamping to the fiscal year. + $perfected = trim($columns[9]); + $start = $perfected !== '' ? $perfected : trim($columns[8]); + $context = sprintf('Component %s, CSV record %d', $id, $record); + if ($start === '') { + throw new \RuntimeException($context . ': Completed requests require a date in Column J or I to calculate response time.'); + } + $start_date = $this->calendarDate($start, $context . ', Column J or I'); + $end_date = $this->calendarDate($completed, $context . ', Column K'); + if ($end_date < $start_date) { + throw new \RuntimeException($context . ': Date Completed cannot precede the response time start date.'); + } + // Share the Days Tolled calendar without subtracting tolled days. + $days = $working_days->count($start, $completed); + $histograms[$id][$track][$days] = ($histograms[$id][$track][$days] ?? 0) + 1; + } + if (!feof($stream)) { + throw new \RuntimeException('Unable to finish reading a CSV for processed response times.'); + } + } + finally { + fclose($stream); + } + } + return $histograms; + } + + /** + * Parses calendar dates defensively if a file changed after validation. + */ + private function calendarDate(string $value, string $context): \DateTimeImmutable { + if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts) + || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) { + throw new \RuntimeException($context . ': Expected a valid date in MM/DD/YYYY format.'); + } + return new \DateTimeImmutable(sprintf('%04d-%02d-%02d', $parts[3], $parts[1], $parts[2]), new \DateTimeZone('UTC')); + } + + /** + * Finds exact medians and weighted averages from day frequencies. + */ + private function summarize(array $histogram): array { + if ($histogram === []) { + return []; + } + ksort($histogram, SORT_NUMERIC); + $count = array_sum($histogram); + $lower_position = intdiv($count + 1, 2); + $upper_position = intdiv($count, 2) + 1; + $seen = 0; + $sum = 0; + $lower = NULL; + $upper = NULL; + foreach ($histogram as $days => $frequency) { + $seen += $frequency; + $sum += $days * $frequency; + if ($lower === NULL && $seen >= $lower_position) { + $lower = $days; + } + if ($upper === NULL && $seen >= $upper_position) { + $upper = $days; + } + } + return [ + 'median' => ($lower + $upper) / 2, + 'average' => $sum / $count, + 'lowest' => array_key_first($histogram), + 'highest' => array_key_last($histogram), + ]; + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php index af60bbd87..d4110c167 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php +++ b/docroot/modules/custom/foia_raw_data_to_report/src/RequestStatisticsAggregator.php @@ -40,7 +40,9 @@ public function aggregate(array $sources, int $fiscal_year): array { } try { $header = TRUE; + $record = 0; while (($columns = fgetcsv($stream, 0, ',', '"', '')) !== FALSE) { + $record++; if ($columns === [NULL]) { continue; } @@ -52,7 +54,8 @@ public function aggregate(array $sources, int $fiscal_year): array { continue; } // Every data row counts, including consultations and rows without E. - $received = $this->calendarDate(trim($columns[8])); + $context = sprintf('Component %s, CSV %s, record %d', $source['component_id'], basename($source['uri']), $record); + $received = $this->calendarDate(trim($columns[8]), $context . ', Column I (Date Initially Received)'); if ($received < $start) { $counts['pending_start']++; } @@ -69,7 +72,7 @@ public function aggregate(array $sources, int $fiscal_year): array { $counts['pending_end']++; } else { - $completed_date = $this->calendarDate($completed); + $completed_date = $this->calendarDate($completed, $context . ', Column K (Date Completed)'); if ($completed_date >= $start && $completed_date <= $end) { $counts['processed']++; } @@ -104,10 +107,10 @@ public function aggregate(array $sources, int $fiscal_year): array { /** * Converts a validated date to YYYYMMDD, guarding against changed CSV data. */ - private function calendarDate(string $value): int { + private function calendarDate(string $value, string $context): int { if (!preg_match('/^([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{4})$/', $value, $parts) || !checkdate((int) $parts[1], (int) $parts[2], (int) $parts[3])) { - throw new \RuntimeException('CSV contains an invalid date after validation.'); + throw new \RuntimeException($context . ': CSV contains an invalid date after validation: ' . ($value === '' ? '[blank]' : $value) . '. Expected MM/DD/YYYY.'); } return (int) $parts[3] * 10000 + (int) $parts[1] * 100 + (int) $parts[2]; } diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/WorkingDays.php b/docroot/modules/custom/foia_raw_data_to_report/src/WorkingDays.php new file mode 100644 index 000000000..bc9a502c7 --- /dev/null +++ b/docroot/modules/custom/foia_raw_data_to_report/src/WorkingDays.php @@ -0,0 +1,254 @@ +diff($end_date)->days; + $working_days = intdiv($days, 7) * 5; + $start_weekday = (int) $start_date->format('N'); + for ($offset = 1; $offset <= $days % 7; $offset++) { + if (($start_weekday - 1 + $offset) % 7 < 5) { + $working_days++; + } + } + + $first = $start_date->format('Y-m-d'); + $last = $end_date->format('Y-m-d'); + foreach (self::FEDERAL_HOLIDAYS as $holiday) { + if ($holiday > $last) { + break; + } + if ($holiday > $first && (int) (new \DateTimeImmutable($holiday, $timezone))->format('N') <= 5) { + $working_days--; + } + } + return $working_days; + } + +} diff --git a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php index e2e9c0140..73268515d 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php +++ b/docroot/modules/custom/foia_raw_data_to_report/src/XmlReportBuilder.php @@ -55,11 +55,29 @@ final class XmlReportBuilder { * Component and overall summaries from AppealResponseTimeAggregator. * @param array $oldest_pending_appeals * Component and overall lists from OldestPendingAppealAggregator. + * @param array $processed_response_times + * Component and overall summaries from ProcessedResponseTimeAggregator. + * @param array $information_granted_response_times + * Disposition-filtered summaries from ProcessedResponseTimeAggregator. + * @param array $simple_response_increments + * Simple-track bin counts from ProcessedResponseTimeAggregator. + * @param array $complex_response_increments + * Complex-track bin counts from ProcessedResponseTimeAggregator. + * @param array $expedited_response_increments + * Expedited-track bin counts from ProcessedResponseTimeAggregator. + * @param array $pending_perfected_requests + * Component and agency summaries from PendingPerfectedRequestsAggregator. + * @param array $oldest_pending_requests + * Component and overall lists from OldestPendingRequestAggregator. + * @param array $expedited_processing + * Component and agency counters from ExpeditedProcessingAggregator. + * @param array $fee_waivers + * Component and agency counters from FeeWaiverAggregator. * * @return string * The serialized report XML. */ - public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = [], array $oldest_pending_appeals = []): string { + public function build(TermInterface $agency, array $components, int $fiscal_year, array $statutes = [], array $request_statistics = [], array $dispositions = [], array $other_reasons = [], array $applied_exemptions = [], array $appeal_statistics = [], array $appeal_dispositions = [], array $appeal_exemptions = [], array $appeal_denials = [], array $appeal_other_reasons = [], array $appeal_response_times = [], array $oldest_pending_appeals = [], array $processed_response_times = [], array $information_granted_response_times = [], array $simple_response_increments = [], array $complex_response_increments = [], array $expedited_response_increments = [], array $pending_perfected_requests = [], array $oldest_pending_requests = [], array $expedited_processing = [], array $fee_waivers = []): string { $document = new \DOMDocument('1.0', 'UTF-8'); $document->formatOutput = TRUE; $root = $document->createElementNS(self::NAMESPACES['iepd'], 'iepd:FoiaAnnualReport'); @@ -136,7 +154,43 @@ public function build(TermInterface $agency, array $components, int $fiscal_year } if ($oldest_pending_appeals !== []) { - $this->addOldestPendingAppeals($document, $root, $oldest_pending_appeals, $component_map); + $this->addOldestPendingItems($document, $root, $oldest_pending_appeals, $component_map, 'OldestPendingAppealSection', 'OPA'); + } + + if ($processed_response_times !== []) { + $this->addProcessedResponseTimes($document, $root, $processed_response_times, $component_map, 'ProcessedResponseTimeSection', 'PRT'); + } + + if ($information_granted_response_times !== []) { + $this->addProcessedResponseTimes($document, $root, $information_granted_response_times, $component_map, 'InformationGrantedResponseTimeSection', 'IGRT'); + } + + if ($simple_response_increments !== []) { + $this->addResponseTimeIncrements($document, $root, $simple_response_increments, $component_map, 'SimpleResponseTimeIncrementsSection', 'SRT'); + } + + if ($complex_response_increments !== []) { + $this->addResponseTimeIncrements($document, $root, $complex_response_increments, $component_map, 'ComplexResponseTimeIncrementsSection', 'CRT'); + } + + if ($expedited_response_increments !== []) { + $this->addResponseTimeIncrements($document, $root, $expedited_response_increments, $component_map, 'ExpeditedResponseTimeIncrementsSection', 'ERT'); + } + + if ($pending_perfected_requests !== []) { + $this->addPendingPerfectedRequests($document, $root, $pending_perfected_requests, $component_map); + } + + if ($oldest_pending_requests !== []) { + $this->addOldestPendingItems($document, $root, $oldest_pending_requests, $component_map, 'OldestPendingRequestSection', 'OPR'); + } + + if ($expedited_processing !== []) { + $this->addExpeditedProcessing($document, $root, $expedited_processing, $component_map); + } + + if ($fee_waivers !== []) { + $this->addFeeWaivers($document, $root, $fee_waivers, $component_map); } $xml = $document->saveXML(); @@ -417,18 +471,18 @@ private function addAppealResponseTimes(\DOMDocument $document, \DOMElement $roo } /** - * Adds oldest open appeals and their organization references. + * Adds oldest pending items and their organization references. */ - private function addOldestPendingAppeals(\DOMDocument $document, \DOMElement $root, array $appeals, array $component_map): void { - $section = $this->addTextElement($document, $root, 'foia', 'OldestPendingAppealSection'); + private function addOldestPendingItems(\DOMDocument $document, \DOMElement $root, array $pending, array $component_map, string $section_name, string $prefix): void { + $section = $this->addTextElement($document, $root, 'foia', $section_name); $organizations = []; foreach ($component_map as $component_id => $organization_id) { - $organizations[$organization_id] = $appeals['components'][$component_id]; + $organizations[$organization_id] = $pending['components'][$component_id]; } - $organizations['ORG0'] = $appeals['overall']; + $organizations['ORG0'] = $pending['overall']; foreach ($organizations as $organization_id => $items) { $entry = $this->addTextElement($document, $section, 'foia', 'OldestPendingItems'); - $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'OPA' . substr($organization_id, 3)); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3)); foreach ($items as $item) { $old_item = $this->addTextElement($document, $entry, 'foia', 'OldItem'); $this->addTextElement($document, $old_item, 'foia', 'OldItemReceiptDate', $item['receipt_date']); @@ -438,7 +492,161 @@ private function addOldestPendingAppeals(\DOMDocument $document, \DOMElement $ro foreach ($organizations as $organization_id => $items) { $association = $this->addTextElement($document, $section, 'foia', 'OldestPendingItemsOrganizationAssociation'); $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); - $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'OPA' . substr($organization_id, 3)); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3)); + $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); + $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); + } + } + + /** + * Adds working-day statistics by track and organization associations. + */ + private function addProcessedResponseTimes(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map, string $section_name, string $prefix): void { + $section = $this->addTextElement($document, $root, 'foia', $section_name); + $organizations = []; + foreach ($component_map as $component_id => $organization_id) { + $organizations[$organization_id] = $statistics['components'][$component_id]; + } + $organizations['ORG0'] = $statistics['overall']; + foreach ($organizations as $organization_id => $tracks) { + $entry = $this->addTextElement($document, $section, 'foia', 'ProcessedResponseTime'); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3)); + foreach (ProcessedResponseTimeAggregator::TRACKS as $track => $name) { + $bin = $this->addTextElement($document, $entry, 'foia', $name); + foreach ($tracks[$track] as $key => $value) { + // Test the unrounded statistic before formatting the average. + $suffix = $value < 1 ? 'Code' : 'Value'; + $text = $value < 1 ? 'LT1' : ($key === 'average' ? number_format($value, 2, '.', '') : (string) $value); + $this->addTextElement($document, $bin, 'foia', 'ResponseTime' . ucfirst($key) . 'Days' . $suffix, $text); + } + } + } + foreach ($organizations as $organization_id => $tracks) { + $association = $this->addTextElement($document, $section, 'foia', 'ProcessedResponseTimeOrganizationAssociation'); + $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3)); + $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); + $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); + } + } + + /** + * Adds all thirteen response-time bins and organization references. + */ + private function addResponseTimeIncrements(\DOMDocument $document, \DOMElement $root, array $increments, array $component_map, string $section_name, string $prefix): void { + $section = $this->addTextElement($document, $root, 'foia', $section_name); + $organizations = []; + foreach ($component_map as $component_id => $organization_id) { + $organizations[$organization_id] = $increments['components'][$component_id]; + } + $organizations['ORG0'] = $increments['overall']; + foreach ($organizations as $organization_id => $counts) { + $entry = $this->addTextElement($document, $section, 'foia', 'ComponentResponseTimeIncrements'); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', $prefix . substr($organization_id, 3)); + foreach (ProcessedResponseTimeAggregator::INCREMENTS as $code => $upper) { + $increment = $this->addTextElement($document, $entry, 'foia', 'TimeIncrement'); + $this->addTextElement($document, $increment, 'foia', 'TimeIncrementCode', $code); + $this->addTextElement($document, $increment, 'foia', 'TimeIncrementProcessedQuantity', (string) $counts[$code]); + } + $this->addTextElement($document, $entry, 'foia', 'TimeIncrementTotalQuantity', (string) array_sum($counts)); + } + foreach ($organizations as $organization_id => $counts) { + $association = $this->addTextElement($document, $section, 'foia', 'ResponseTimeIncrementsOrganizationAssociation'); + $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', $prefix . substr($organization_id, 3)); + $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); + $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); + } + } + + /** + * Adds pending counts and ages by track, including N/A for empty tracks. + */ + private function addPendingPerfectedRequests(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void { + $section = $this->addTextElement($document, $root, 'foia', 'PendingPerfectedRequestsSection'); + $organizations = []; + foreach ($component_map as $component_id => $organization_id) { + $organizations[$organization_id] = $statistics['components'][$component_id]; + } + $organizations['ORG0'] = $statistics['overall']; + foreach ($organizations as $organization_id => $tracks) { + $entry = $this->addTextElement($document, $section, 'foia', 'PendingPerfectedRequests'); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'PPR' . substr($organization_id, 3)); + foreach (PendingPerfectedRequestsAggregator::TRACKS as $track => $name) { + $values = $tracks[$track]; + $bin = $this->addTextElement($document, $entry, 'foia', $name); + $this->addTextElement($document, $bin, 'foia', 'PendingRequestQuantity', (string) $values['quantity']); + $median = $values['quantity'] === 0 ? 'N/A' : (string) $values['median']; + $average = $values['quantity'] === 0 ? 'N/A' : number_format($values['average'], 2, '.', ''); + $this->addTextElement($document, $bin, 'foia', 'PendingRequestMedianDaysValue', $median); + $this->addTextElement($document, $bin, 'foia', 'PendingRequestAverageDaysValue', $average); + } + } + foreach ($organizations as $organization_id => $tracks) { + $association = $this->addTextElement($document, $section, 'foia', 'PendingPerfectedRequestsOrganizationAssociation'); + $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'PPR' . substr($organization_id, 3)); + $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); + $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); + } + } + + /** + * Adds expedited outcomes and timely adjudications with organization links. + */ + private function addExpeditedProcessing(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void { + $section = $this->addTextElement($document, $root, 'foia', 'ExpeditedProcessingSection'); + $organizations = []; + foreach ($component_map as $component_id => $organization_id) { + $organizations[$organization_id] = $statistics['components'][$component_id]; + } + $organizations['ORG0'] = $statistics['overall']; + $fields = [ + 'granted' => 'RequestGrantedQuantity', + 'denied' => 'RequestDeniedQuantity', + 'within_ten' => 'AdjudicationWithinTenDaysQuantity', + ]; + foreach ($organizations as $organization_id => $counts) { + $entry = $this->addTextElement($document, $section, 'foia', 'ExpeditedProcessing'); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'EP' . substr($organization_id, 3)); + foreach ($fields as $key => $name) { + $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]); + } + } + foreach ($organizations as $organization_id => $counts) { + $association = $this->addTextElement($document, $section, 'foia', 'ExpeditedProcessingOrganizationAssociation'); + $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'EP' . substr($organization_id, 3)); + $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); + $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); + } + } + + /** + * Adds fee-waiver outcomes, including zeros, and organization links. + */ + private function addFeeWaivers(\DOMDocument $document, \DOMElement $root, array $statistics, array $component_map): void { + $section = $this->addTextElement($document, $root, 'foia', 'FeeWaiverSection'); + $organizations = []; + foreach ($component_map as $component_id => $organization_id) { + $organizations[$organization_id] = $statistics['components'][$component_id]; + } + $organizations['ORG0'] = $statistics['overall']; + $fields = [ + 'granted' => 'RequestGrantedQuantity', + 'denied' => 'RequestDeniedQuantity', + ]; + foreach ($organizations as $organization_id => $counts) { + $entry = $this->addTextElement($document, $section, 'foia', 'FeeWaiver'); + $entry->setAttributeNS(self::NAMESPACES['s'], 's:id', 'FW' . substr($organization_id, 3)); + foreach ($fields as $key => $name) { + $this->addTextElement($document, $entry, 'foia', $name, (string) $counts[$key]); + } + } + foreach ($organizations as $organization_id => $counts) { + $association = $this->addTextElement($document, $section, 'foia', 'FeeWaiverOrganizationAssociation'); + $reference = $this->addTextElement($document, $association, 'foia', 'ComponentDataReference'); + $reference->setAttributeNS(self::NAMESPACES['s'], 's:ref', 'FW' . substr($organization_id, 3)); $organization = $this->addTextElement($document, $association, 'nc', 'OrganizationReference'); $organization->setAttributeNS(self::NAMESPACES['s'], 's:ref', $organization_id); } diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php b/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php index 2920b0c04..661e50b99 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php +++ b/docroot/modules/custom/foia_raw_data_to_report/tests/integration/component_uploads.php @@ -129,6 +129,31 @@ $check($xml && str_starts_with($xml->getFileUri(), 'private://'), 'Valid uploads did not produce private XML.'); $files[] = $xml; $check(substr_count($report->get('field_messages')->value, 'CSV validated.') === 2, 'Successful retry did not replace earlier messages.'); + // An aggregation exception must append to validation messages and retain XML. + $row = array_fill(0, 29, ''); + $row[0] = 'Component'; + $row[1] = 'Exception fixture'; + $row[2] = 'N'; + $row[3] = '20'; + $row[23] = '01/02/2026'; + $contents = implode(',', array_fill(0, 29, 'column')) . "\n" . implode(',', $row); + file_put_contents($files[1]->getFileUri(), $contents); + for ($attempt = 0; $attempt < 2; $attempt++) { + try { + $process(); + throw new LogicException('Expected the request-statistics date exception.'); + } + catch (RuntimeException $exception) { + $check(str_contains($exception->getMessage(), 'Column I'), 'Unexpected processing exception.'); + $report = $node_storage->loadUnchanged($report->id()); + $message = $report->get('field_messages')->value; + $check(substr_count($message, 'CSV validated.') === 2, 'Exception lost validation messages.'); + $check(substr_count($message, 'XML report processing failed:') === 1, 'Retry duplicated exception messages.'); + $check(str_ends_with($message, $exception->getMessage()), 'Exception details were not appended.'); + $check($report->get('field_messages')->format === 'plain_text', 'Exception message must be plain text.'); + $check($report->get('field_request_data_xml')->target_id === $xml->id(), 'Exception replaced existing XML.'); + } + } file_put_contents($files[1]->getFileUri(), 'bad,csv'); $report = $process(); $check($report->get('field_request_data_xml')->target_id === $xml->id(), 'Failed retry replaced existing XML.'); diff --git a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php index daf6e2874..b9c6000a9 100644 --- a/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php +++ b/docroot/modules/custom/foia_raw_data_to_report/tests/src/Unit/CsvValidatorTest.php @@ -87,22 +87,50 @@ public function testExpeditedReceivedDependency(): void { } /** - * Rejects every request-data column E through W when X contains data. + * Rejects E-P and T-W when X contains data, while allowing Q, R, and S. */ public function testAppealRequestDataExclusion(): void { $header = implode(',', array_fill(0, 29, 'column')) . "\n"; - for ($column = 4; $column <= 22; $column++) { + foreach (array_merge(range(4, 15), range(19, 22)) as $column) { $overrides = [8 => '', 23 => '01/02/2026']; // Zero is data too, even though PHP treats it as an empty value. $overrides[$column] = '0'; $this->assertSame( - ['CSV record 2: If there is data in Column X, Columns E through W must be empty.'], + ['CSV record 2: If there is data in Column X, Columns E through W must be empty, except for Columns Q, R, and S; those may optionally have data, but Columns E through P and Columns T through W must be blank.'], $this->validateContents($header . $this->csvRow($overrides)), 'Column index ' . $column, ); } } + /** + * Allows expedited-processing data on appeal rows with its usual validation. + */ + public function testAppealExpeditedProcessing(): void { + $header = implode(',', array_fill(0, 29, 'column')) . "\n"; + $appeal = [8 => '', 23 => '01/02/2026']; + foreach ([ + [], + [16 => '01/03/2026'], + [16 => '01/03/2026', 18 => 'D'], + [16 => '01/03/2026', 17 => '01/04/2026', 18 => 'D'], + [16 => '01/03/2026', 18 => 'G'], + [16 => '01/03/2026', 17 => '01/04/2026', 18 => 'G'], + ] as $values) { + $this->assertSame([], $this->validateContents($header . $this->csvRow($appeal + $values))); + } + $cases = [ + [[18 => 'D'], 'Column Q must contain value'], + [[16 => 'bad-date'], 'Column Q: Request for EP - Date Received must be a valid date'], + [[16 => '01/03/2026', 17 => '01/02/2026', 18 => 'D'], 'cannot be prior'], + [[16 => '01/03/2026', 17 => '01/04/2026'], 'Must have G or D in Column S'], + ]; + foreach ($cases as [$values, $message]) { + $errors = $this->validateContents($header . $this->csvRow($appeal + $values)); + $this->assertStringContainsString($message, $errors[0]); + } + } + /** * Allows blank I with or without X, and validates nonblank received dates. */