From 533b80db87d8760a299b2fb76f15ffa30d67d51d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 18:01:13 +0000 Subject: [PATCH 1/3] chore(data-raw): measure fiscal_year_end coverage before building on it (#77) data-raw/measure_fiscal_year_end.R reads a corpus's long partitions and reports, by survey year and government type, how many government-years have a missing, zero, implausible or internally conflicting fiscal_year_end; the most common values; how often a government's value changes between consecutive years; and every state government's and the District of Columbia's value. On the bundled fixture (FY2011, FY2012, FY2019, FY2020; full published partitions from the 2026-08-03 build) 0.1%-0.9% of government-years are missing a value, none is zero, implausible or conflicting, and every state government keeps the same value in all four years. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qbe6u2h83wgZdTgJgE4Xh5 --- data-raw/measure_fiscal_year_end.R | 101 +++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 data-raw/measure_fiscal_year_end.R diff --git a/data-raw/measure_fiscal_year_end.R b/data-raw/measure_fiscal_year_end.R new file mode 100644 index 0000000..dd60f4c --- /dev/null +++ b/data-raw/measure_fiscal_year_end.R @@ -0,0 +1,101 @@ +# data-raw/measure_fiscal_year_end.R +# +# Measures how usable `long.fiscal_year_end` is before anything is built on it +# (uscogdata#77, step 1). Reads the long partitions of a corpus directly and +# reports, by survey year and government type (0-3): +# +# * government-years with no fiscal_year_end on any row (missing), +# * values of 0, and values that are not a valid MMDD date (implausible), +# * government-years whose rows disagree on the value, +# * the most common values, +# * how often a government's value changes between consecutive years, +# * every state government's value, and the District of Columbia's. +# +# A "government-year" is one (canonical_govid, year). fiscal_year_end is a +# header field, so every row of a government-year should carry the same value. +# +# Usage, from the package root: +# Rscript data-raw/measure_fiscal_year_end.R +# Rscript data-raw/measure_fiscal_year_end.R /path/to/corpus +# +# The default is the bundled fixture, which holds only FY2011, FY2012, FY2019 +# and FY2020. Do not generalize its results to other years. + +measure_fiscal_year_end <- function( + corpus = file.path("inst", "extdata", "fixture_corpus")) { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + run <- function(sql) DBI::dbExecute(con, sql) + q <- function(sql) DBI::dbGetQuery(con, sql) + + run(sprintf( + "CREATE VIEW l AS SELECT * FROM read_parquet('%s/data/long/*/*.parquet', + hive_partitioning = true)", + normalizePath(corpus, mustWork = TRUE) + )) + run("CREATE MACRO valid_mmdd(x) AS x IS NOT NULL + AND x // 100 BETWEEN 1 AND 12 + AND x % 100 BETWEEN 1 AND CASE x // 100 + WHEN 2 THEN 29 WHEN 4 THEN 30 WHEN 6 THEN 30 + WHEN 9 THEN 30 WHEN 11 THEN 30 ELSE 31 END") + run("CREATE TABLE gy AS + SELECT year, type, canonical_govid, + any_value(gov_name) AS gov_name, + any_value(fips_state) AS fips_state, + COUNT(DISTINCT fiscal_year_end) AS n_values, + COUNT(*) FILTER (WHERE fiscal_year_end IS NULL) AS n_null_rows, + COUNT(*) AS n_rows, + any_value(fiscal_year_end) + FILTER (WHERE fiscal_year_end IS NOT NULL) AS fye + FROM l GROUP BY 1, 2, 3") + + list( + partitions = q("SELECT year, COUNT(*) AS rows FROM l GROUP BY 1 ORDER BY 1"), + by_year_type = q( + "SELECT year, type, COUNT(*) AS gov_years, + SUM((fye IS NULL)::INT) AS missing, + SUM((fye = 0)::INT) AS zero, + SUM((fye IS NOT NULL AND fye <> 0 AND NOT valid_mmdd(fye))::INT) + AS implausible, + SUM((n_values > 1)::INT) AS conflicting_within_year, + SUM((n_null_rows > 0 AND n_null_rows < n_rows)::INT) + AS partly_missing + FROM gy GROUP BY 1, 2 ORDER BY 1, 2"), + row_null_share = q( + "SELECT year, COUNT(*) AS rows, + AVG((fiscal_year_end IS NULL)::INT) AS null_share + FROM l GROUP BY 1 ORDER BY 1"), + top_values = q( + "WITH c AS (SELECT year, fye, COUNT(*) AS n FROM gy GROUP BY 1, 2), + r AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY year ORDER BY n DESC) AS rk, + n / SUM(n) OVER (PARTITION BY year) AS share FROM c) + SELECT year, fye, n, share FROM r WHERE rk <= 6 ORDER BY year, rk"), + changes = q( + "SELECT a.year AS from_year, b.year AS to_year, a.type, + COUNT(*) AS in_both_years, SUM((a.fye <> b.fye)::INT) AS changed + FROM gy a JOIN gy b + ON a.canonical_govid = b.canonical_govid AND b.year = a.year + 1 + WHERE a.fye IS NOT NULL AND b.fye IS NOT NULL + GROUP BY 1, 2, 3 ORDER BY 1, 3"), + states = q( + "SELECT fips_state, any_value(gov_name) AS gov_name, year, fye + FROM gy WHERE type = 0 GROUP BY fips_state, year, fye + ORDER BY fips_state, year"), + district_of_columbia = q( + "SELECT year, type, canonical_govid, gov_name, fye + FROM gy WHERE fips_state = 11 ORDER BY year") + ) +} + +if (sys.nframe() == 0L) { + args <- commandArgs(trailingOnly = TRUE) + res <- if (length(args) > 0L) { + measure_fiscal_year_end(args[[1]]) + } else { + measure_fiscal_year_end() + } + for (nm in names(res)) { + cat("\n==", nm, "\n") + print(res[[nm]], row.names = FALSE) + } +} From b6b3e4652a764bfd9b9059c6378d29c758ba5fbc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 18:10:48 +0000 Subject: [PATCH 2/3] feat: fiscal-year periods on the verbs and survey-year helpers (#77) `year` is the Census survey year, and Census files a government's fiscal year under survey year Y when it ended between July 1 of Y-1 and June 30 of Y, so one survey year covers different months for different governments. cog_spending(), cog_revenue() and cog_balances() gain a trailing `fiscal_period = FALSE`. With TRUE, every row gains fiscal_year_end (MMDD integer), period_start and period_end (Dates, inclusive), looked up per government-year from long.fiscal_year_end so a change of fiscal year is respected, plus provenance$transformations$fiscal_period. With the default, columns and provenance are exactly as before. cog_fiscal_periods(govid, years) returns the govid x years grid of periods; cog_survey_year(govid, date) returns the survey year whose period contains a date, using each candidate year's own fiscal_year_end and returning NA rather than borrowing another year's value. State governments, and the District of Columbia, cover the fiscal year ending in calendar year Y. That changes Alabama, Michigan (September 30) and Texas (August 31). Census's 2004 and 2006 technical documentation lists those exceptions, and the corpus agrees for survey year 2020: those states' federal aid rises with the CARES Act in 2020 while New York's and 15 large September-30 local governments' rises in 2021. D.C. rests on the documentation alone. Tests run against the bundled fixture (85 expectations). Refs #77 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qbe6u2h83wgZdTgJgE4Xh5 --- NAMESPACE | 2 + R/balances.R | 17 +- R/fiscal.R | 377 ++++++++++++++++++++++++++++ R/revenue.R | 8 +- R/spending.R | 29 ++- _pkgdown.yml | 7 + man/cog_balances.Rd | 14 +- man/cog_fiscal_periods.Rd | 80 ++++++ man/cog_revenue.Rd | 18 +- man/cog_spending.Rd | 18 +- man/cog_survey_year.Rd | 86 +++++++ tests/testthat/test-fiscal-period.R | 260 +++++++++++++++++++ 12 files changed, 899 insertions(+), 17 deletions(-) create mode 100644 R/fiscal.R create mode 100644 man/cog_fiscal_periods.Rd create mode 100644 man/cog_survey_year.Rd create mode 100644 tests/testthat/test-fiscal-period.R diff --git a/NAMESPACE b/NAMESPACE index 5d956a0..9e71307 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ export(cog_basket_unresolved) export(cog_categories) export(cog_explain) export(cog_find_peers) +export(cog_fiscal_periods) export(cog_geographic_rollup) export(cog_gov_search) export(cog_manifest) @@ -14,3 +15,4 @@ export(cog_peer_compare) export(cog_recipes) export(cog_revenue) export(cog_spending) +export(cog_survey_year) diff --git a/R/balances.R b/R/balances.R index 8ef4e60..651970d 100644 --- a/R/balances.R +++ b/R/balances.R @@ -53,6 +53,11 @@ #' `total_rows`. #' @param offset Rows to skip before `limit` starts counting (0-based). #' Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set. +#' @param fiscal_period If `TRUE`, add `fiscal_year_end`, `period_start` and +#' `period_end` to every row, as [cog_spending()] does. Holdings are a stock +#' measured at the end of the fiscal year, so `period_end` is the date the +#' balance describes and `period_start` is the start of that fiscal year. +#' Defaults to `FALSE`, which leaves the result unchanged. #' #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`, #' `balance_subtype`, `category`, `amt_nominal`, `codes_included`, @@ -60,8 +65,9 @@ #' `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when #' `adjust_to_year` is set), and optional `amt_per_capita_real` (only when #' **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no -#' nominal per-capita column to deflate otherwise). Amounts are full US -#' dollars. +#' nominal per-capita column to deflate otherwise), and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. Amounts are +#' full US dollars. #' #' Carries a `provenance` attribute matching #' `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports @@ -79,7 +85,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, per_capita = FALSE, adjust_to_year = NULL, basis = c("harmonized", "raw"), recipe = NULL, state = NULL, type = NULL, - limit = NULL, offset = NULL) { + limit = NULL, offset = NULL, fiscal_period = FALSE) { call <- match.call() basis <- match.arg(basis, c("harmonized", "raw")) # Coerce FIRST, validate second: .validate_verb_inputs() asserts @@ -102,6 +108,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, # validator's own doc comment for the incident that made that matter. .validate_verb_inputs(govid, years, category, per_capita, adjust_to_year, recipe) + .validate_fiscal_period_arg(fiscal_period) # Same semantics as the money verbs (R/pagination.R). Only the `recipe` # conflict applies here: cog_balances() has no `complete` argument, and a @@ -174,6 +181,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, if (!is.null(adjust_to_year)) { result <- .attach_real_dollars(result, adjust_to_year, per_capita) } + if (fiscal_period) result <- .attach_fiscal_period(result, con) prov <- .build_provenance( verb = "cog_balances", call = call, govid = govid, years = years, @@ -189,6 +197,9 @@ cog_balances <- function(govid = NULL, years, category = NULL, prov$scope$govids_found <- scope$found prov$scope$govids_missing <- scope$missing prov$scope$cohort <- .cohort_provenance(con, cohort) + if (fiscal_period) { + prov$transformations$fiscal_period <- .fiscal_period_provenance(result) + } prov$balance_caveats <- .balance_caveats( con, prov$codes_summed$observed, years diff --git a/R/fiscal.R b/R/fiscal.R new file mode 100644 index 0000000..e8ac14e --- /dev/null +++ b/R/fiscal.R @@ -0,0 +1,377 @@ +# R/fiscal.R +# +# Fiscal-year timing (uscogdata#77). The corpus's `year` is the Census SURVEY +# year, and one survey year covers different calendar months for different +# governments. Census's rule (Local Government Finance FAQ, Q6): survey year Y +# holds each government's fiscal year that ENDED between July 1 of Y-1 and +# June 30 of Y. `long.fiscal_year_end` (an MMDD integer such as 630 or 1231) +# says which months that is for a given government-year. +# +# Two kinds of government follow a different rule: survey year Y holds the +# fiscal year ending in calendar year Y, even when that is after June 30. +# +# * State governments (type 0). Census's 2004 and 2006 technical +# documentation names the Alabama and Michigan (September 30) and Texas +# (August 31) state governments as exceptions "included as though they +# were part of the June 30 group". The published corpus agrees for survey +# year 2020: those three states' federal aid jumps with the CARES Act in +# 2020, as the June-30 states' does, while New York (March 31) and 15 +# large September-30 cities and counties jump in 2021 (measurements in +# uscogdata#77). For the other 47 states both rules give the same months. +# * The District of Columbia (type 2 here), which the same documentation +# lists with the September-30 exceptions. The 2020 data neither confirms +# nor rules this out, so it rests on the documentation alone. + +# canonical_govids that follow the state-government rule without being type 0. +.SURVEY_YEAR_SAME_YEAR_GOVIDS <- c( + "112001124214" # Washington, D.C. +) + +#' Days in a month, vectorized over year and month. +#' @noRd +.days_in_month <- function(year, month) { + dim <- c(31L, 28L, 31L, 30L, 31L, 30L, 31L, 31L, 30L, 31L, 30L, 31L) + leap <- (year %% 4L == 0L & year %% 100L != 0L) | year %% 400L == 0L + out <- dim[month] + out[month == 2L & leap] <- 29L + out +} + +#' Build Dates from year/month/day, clamping the day to the month's length +#' (so a February 29 year end falls on February 28 in a common year). +#' @noRd +.make_date <- function(year, month, day) { + out <- rep(as.Date(NA), length(year)) + ok <- !is.na(year) & !is.na(month) & !is.na(day) + if (any(ok)) { + d <- pmin(day[ok], .days_in_month(year[ok], month[ok])) + out[ok] <- as.Date(sprintf("%04d-%02d-%02d", year[ok], month[ok], d)) + } + out +} + +#' TRUE where `fiscal_year_end` is a usable MMDD value. +#' @noRd +.valid_fiscal_year_end <- function(fye) { + fye <- as.integer(fye) + m <- fye %/% 100L + d <- fye %% 100L + ok <- !is.na(fye) & !is.na(m) & m >= 1L & m <= 12L & d >= 1L + ok[ok] <- d[ok] <= .days_in_month(2000L, m[ok]) # 2000 is a leap year + ok +} + +#' The fiscal period a survey year covers for one government-year. +#' +#' Vectorized over its arguments, which must have equal length. +#' +#' @param year Integer survey year (the corpus's `year`). +#' @param fiscal_year_end Integer MMDD from `long.fiscal_year_end`. +#' @param gov_type Integer government type (0 = state). +#' @param canonical_govid Character canonical id. +#' @return List of Date vectors `period_start` and `period_end`; `NA` where +#' `fiscal_year_end` is missing or not a valid MMDD. +#' @noRd +.survey_period <- function(year, fiscal_year_end, gov_type, canonical_govid) { + year <- as.integer(year) + fye <- as.integer(fiscal_year_end) + ok <- .valid_fiscal_year_end(fye) + month <- fye %/% 100L + day <- fye %% 100L + + same_year <- (!is.na(gov_type) & as.integer(gov_type) == 0L) | + canonical_govid %in% .SURVEY_YEAR_SAME_YEAR_GOVIDS + end_year <- ifelse(same_year | month <= 6L, year, year - 1L) + + month[!ok] <- NA_integer_ + # The period starts the day after the same month-day one year earlier, + # which is when the government's previous fiscal year ended. + list( + period_start = .make_date(end_year - 1L, month, day) + 1L, + period_end = .make_date(end_year, month, day) + ) +} + +#' Look up each government-year's fiscal_year_end and type in `long`. +#' +#' A government-year whose rows carry more than one distinct value gets `NA` +#' rather than an arbitrary pick. None does in the fixture's four years. +#' +#' @return Tibble with `canonical_govid`, `year` (integer), `gov_type` +#' (integer) and `fiscal_year_end` (integer), one row per government-year +#' found in the corpus. +#' @noRd +.fiscal_year_lookup <- function(con, govids, years) { + govids <- unique(govids[!is.na(govids) & nzchar(govids)]) + years <- unique(as.integer(years[!is.na(years)])) + if (length(govids) == 0L || length(years) == 0L) { + return(tibble::tibble(canonical_govid = character(0), year = integer(0), + gov_type = integer(0), + fiscal_year_end = integer(0))) + } + out <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf( + "SELECT canonical_govid, year, + MIN(type) AS gov_type, + CASE WHEN COUNT(DISTINCT fiscal_year_end) = 1 + THEN MAX(fiscal_year_end) END AS fiscal_year_end + FROM long + WHERE canonical_govid IN (%s) AND year IN (%s) + GROUP BY canonical_govid, year", + .sql_lit_chr(govids), paste(years, collapse = ",") + ))) + out$year <- as.integer(out$year) + out$gov_type <- as.integer(out$gov_type) + out$fiscal_year_end <- as.integer(out$fiscal_year_end) + out +} + +#' Fiscal-year columns for every (canonical_govid, year) pair, in input order. +#' @return Tibble with `fiscal_year_end`, `period_start`, `period_end`. +#' @noRd +.fiscal_periods_for <- function(con, govids, years) { + years <- as.integer(years) + fy <- .fiscal_year_lookup(con, govids, years) + idx <- match(paste(govids, years), paste(fy$canonical_govid, fy$year)) + fye <- fy$fiscal_year_end[idx] + per <- .survey_period(years, fye, fy$gov_type[idx], govids) + tibble::tibble( + fiscal_year_end = ifelse(.valid_fiscal_year_end(fye), fye, NA_integer_), + period_start = per$period_start, + period_end = per$period_end + ) +} + +#' Append `fiscal_year_end`, `period_start` and `period_end` to a verb result. +#' +#' Joined by position rather than with `dplyr::left_join()`, so row order and +#' the result's attributes are untouched. Rows with no `canonical_govid` +#' (none today) get `NA`. +#' @noRd +.attach_fiscal_period <- function(result, con) { + if (nrow(result) == 0L) { + result$fiscal_year_end <- integer(0) + result$period_start <- as.Date(character(0)) + result$period_end <- as.Date(character(0)) + return(result) + } + fp <- .fiscal_periods_for(con, as.character(result$canonical_govid), + result$year) + result$fiscal_year_end <- fp$fiscal_year_end + result$period_start <- fp$period_start + result$period_end <- fp$period_end + result +} + +#' Provenance block recorded when a verb is called with `fiscal_period = TRUE`. +#' Absent otherwise, so default output keeps its existing shape. +#' @noRd +.fiscal_period_provenance <- function(result) { + list( + applied = TRUE, + source = "long.fiscal_year_end (MMDD), per government-year", + rule = paste( + "Survey year Y covers the fiscal year that ended between July 1 of Y-1", + "and June 30 of Y; state governments and the District of Columbia", + "cover the fiscal year ending in calendar year Y." + ), + rows_without_period = sum(is.na(result$period_end)) + ) +} + +#' @noRd +.validate_fiscal_period_arg <- function(fiscal_period) { + if (!is.logical(fiscal_period) || length(fiscal_period) != 1L || + is.na(fiscal_period)) { + cli::cli_abort("{.arg fiscal_period} must be {.code TRUE} or {.code FALSE}.") + } + invisible(fiscal_period) +} + +#' Fiscal-year periods behind survey years +#' +#' Returns the calendar months each survey year covers for each government. +#' The corpus's `year` is the Census survey year, and a survey year covers +#' different months for different governments. +#' +#' @section The survey-year rule: +#' Census assigns each government's fiscal year to a survey year by when the +#' fiscal year ended. Survey year Y holds the fiscal year that ended between +#' July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +#' fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +#' `1231`. +#' +#' Survey year 2017 therefore covers, for a county, city or township: +#' +#' * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, +#' 2017, so July 2016 to June 2017. +#' * `930` (September 30): the fiscal year ending September 30, 2016, so +#' October 2015 to September 2016. +#' * `1231` (December 31): the fiscal year ending December 31, 2016, so +#' January to December 2016. +#' +#' A city with a December year end and a city with a June year end are both +#' "2017" in the corpus, but their figures are six months apart. A September +#' year end is nine months behind a June one. +#' +#' State governments (type 0) follow a different rule: survey year Y holds the +#' fiscal year ending in calendar year Y. For Alabama and Michigan (September +#' 30) that means October 2016 to September 2017, and for Texas (August 31) +#' September 2016 to August 2017. Census's 2004 and 2006 technical +#' documentation lists these three states as exceptions to the June 30 window, +#' and the corpus agrees in survey year 2020, the one year checked. The same +#' documentation lists the District of Columbia, so it is treated the same +#' way. That treatment has not been checked against the data. For New York +#' (March 31) and the 46 states with a June 30 year end, the two rules give +#' the same months. +#' +#' A government can change its fiscal year end, so the periods are looked up +#' per government-year. Where the corpus has no `fiscal_year_end` for a +#' government-year, or the government has no rows that year, the period is +#' `NA`. +#' +#' From FY2023 the source files mark some rows as estimates carried forward +#' from an earlier year. The periods here describe the survey year, not the +#' year a carried-forward row's data pertain to. +#' +#' @param govid Character vector of `canonical_govid` values, or a data frame +#' with a `canonical_govid` column. +#' @param years Integer vector of survey years. +#' @return Tibble with one row per requested government and year, in the +#' order `govid` x `years`: `canonical_govid`, `year`, `fiscal_year_end` +#' (integer MMDD), `period_start` and `period_end` (Dates, inclusive). +#' @seealso [cog_survey_year()] for the reverse mapping; the `fiscal_period` +#' argument of [cog_spending()], [cog_revenue()] and [cog_balances()] to get +#' the same columns on every row. +#' @examples +#' \dontrun{ +#' # A June 30 state, a September 30 state and a September 30 county +#' cog_fiscal_periods( +#' c("130000226087", "010000226085", "121011212191"), +#' years = 2020 +#' ) +#' } +#' @export +cog_fiscal_periods <- function(govid, years) { + if (missing(govid)) cli::cli_abort("{.arg govid} is required.") + if (missing(years)) cli::cli_abort("{.arg years} is required.") + govid <- .coerce_govid_input(govid, arg = "govid") + if (length(govid) == 0L || anyNA(govid) || !all(nzchar(govid))) { + cli::cli_abort("{.arg govid} must name at least one government, with no missing or empty ids.") + } + if (!is.numeric(years) || length(years) == 0L || anyNA(years)) { + cli::cli_abort("{.arg years} must be a non-empty integer vector with no missing values.") + } + govid <- unique(govid) + years <- unique(as.integer(years)) + + con <- .ensure_session() + grid <- expand.grid(year = years, canonical_govid = govid, + stringsAsFactors = FALSE) + fp <- .fiscal_periods_for(con, grid$canonical_govid, grid$year) + tibble::tibble( + canonical_govid = grid$canonical_govid, + year = as.integer(grid$year), + fiscal_year_end = fp$fiscal_year_end, + period_start = fp$period_start, + period_end = fp$period_end + ) +} + +#' Survey year that covers a calendar date for a government +#' +#' Maps a date to the survey year whose fiscal period, for that government, +#' contains it. This is the reverse of [cog_fiscal_periods()]: use it to place +#' an event that happened on a known date against the corpus's `year`. +#' +#' The date can fall in survey year Y, Y+1 or Y+2 of its own calendar year Y, +#' depending on the government's fiscal year end. Each candidate is looked up +#' with the `fiscal_year_end` the corpus records for that government-year, so +#' a change of fiscal year is respected. When the government has no rows, or +#' no `fiscal_year_end`, in the survey year that would cover the date, the +#' result is `NA`; the function does not borrow another year's value. If a +#' change of fiscal year makes two survey years' periods overlap, the earlier +#' survey year is returned. +#' +#' @inheritSection cog_fiscal_periods The survey-year rule +#' @param govid Character vector of `canonical_govid` values, or a data frame +#' with a `canonical_govid` column. +#' @param date Dates, or character strings in `"YYYY-MM-DD"` form. `govid` +#' and `date` must have the same length, or one of them must have length 1. +#' @return Tibble with one row per (`govid`, `date`) pair, in input order: +#' `canonical_govid`, `date`, `year` (the survey year, `NA` when it cannot +#' be determined), `fiscal_year_end`, `period_start` and `period_end`. +#' @seealso [cog_fiscal_periods()] +#' @examples +#' \dontrun{ +#' # April 15, 2020 for a September 30 county and a September 30 state +#' cog_survey_year(c("121011212191", "010000226085"), as.Date("2020-04-15")) +#' } +#' @export +cog_survey_year <- function(govid, date) { + if (missing(govid)) cli::cli_abort("{.arg govid} is required.") + if (missing(date)) cli::cli_abort("{.arg date} is required.") + govid <- .coerce_govid_input(govid, arg = "govid") + if (length(govid) == 0L || anyNA(govid) || !all(nzchar(govid))) { + cli::cli_abort("{.arg govid} must name at least one government, with no missing or empty ids.") + } + date <- .coerce_date_input(date) + n <- max(length(govid), length(date)) + if (!length(govid) %in% c(1L, n) || !length(date) %in% c(1L, n)) { + cli::cli_abort(c( + "{.arg govid} and {.arg date} must have the same length, or one must have length 1.", + "x" = "{.arg govid} has length {length(govid)}; {.arg date} has length {length(date)}." + )) + } + govid <- rep_len(govid, n) + date <- rep_len(date, n) + + con <- .ensure_session() + cal_year <- as.integer(format(date, "%Y")) + cand <- data.frame( + i = rep(seq_len(n), each = 3L), + canonical_govid = rep(govid, each = 3L), + year = rep(cal_year, each = 3L) + rep(0:2, times = n), + stringsAsFactors = FALSE + ) + cand <- cand[!is.na(cand$year), , drop = FALSE] + fp <- .fiscal_periods_for(con, cand$canonical_govid, cand$year) + cand <- cbind(cand, fp) + d <- date[cand$i] + hit <- !is.na(cand$period_start) & d >= cand$period_start & + d <= cand$period_end + cand <- cand[hit, , drop = FALSE] + cand <- cand[order(cand$i, cand$year), , drop = FALSE] + first <- cand[!duplicated(cand$i), , drop = FALSE] + idx <- match(seq_len(n), first$i) + + tibble::tibble( + canonical_govid = govid, + date = date, + year = as.integer(first$year[idx]), + fiscal_year_end = first$fiscal_year_end[idx], + period_start = first$period_start[idx], + period_end = first$period_end[idx] + ) +} + +#' Coerce `date` to Date, refusing values that do not parse. +#' @noRd +.coerce_date_input <- function(date) { + if (length(date) == 0L) cli::cli_abort("{.arg date} must not be empty.") + if (inherits(date, "POSIXt")) date <- as.Date(date) + if (!inherits(date, "Date")) { + if (!is.character(date)) { + cli::cli_abort("{.arg date} must be a Date or a character vector of dates.") + } + parsed <- as.Date(date, format = "%Y-%m-%d") + bad <- !is.na(date) & is.na(parsed) + if (any(bad)) { + cli::cli_abort(c( + "{.arg date} has values that are not dates.", + "x" = "Could not parse: {.val {unique(date[bad])}}." + )) + } + date <- parsed + } + date +} diff --git a/R/revenue.R b/R/revenue.R index b6efe03..9dadd89 100644 --- a/R/revenue.R +++ b/R/revenue.R @@ -48,14 +48,15 @@ #' `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`, #' optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, #' optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, -#' and `value_source` when `complete = TRUE`. +#' `value_source` when `complete = TRUE`, and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. #' @export cog_revenue <- function(govid = NULL, years, category = NULL, per_capita = FALSE, adjust_to_year = NULL, basis = c("harmonized", "raw"), recipe = NULL, revenue_concept = c("general", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { # flow_prefixes no longer classifies rows (crosswalk revenue_subtype # membership does -- General Revenue, i.e. everything except # insurance_trust) -- it only scopes the recipe-suggestion machinery to @@ -78,6 +79,7 @@ cog_revenue <- function(govid = NULL, years, category = NULL, limit = limit, offset = offset, state = state, - type = type + type = type, + fiscal_period = fiscal_period ) } diff --git a/R/spending.R b/R/spending.R index 4129a27..b6166e4 100644 --- a/R/spending.R +++ b/R/spending.R @@ -210,11 +210,23 @@ #' and `govids_missing` are empty -- there is no id list to report against -- #' and `provenance$scope$cohort` carries `state`, `type` and #' `n_governments` instead. A `govid`-named cohort reports exactly as before. +#' @param fiscal_period If `TRUE`, add three columns to every row: +#' `fiscal_year_end` (the government's fiscal year end that survey year, an +#' MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +#' inclusive), the calendar months the row covers. `year` is the Census +#' survey year, which covers different months for different governments; +#' see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +#' from each government-year's own `fiscal_year_end`, so a change of fiscal +#' year is respected, and are `NA` where the corpus has no value. The result +#' also gains `provenance$transformations$fiscal_period`. Defaults to +#' `FALSE`, which returns exactly the columns and provenance this verb +#' returned before the argument existed. #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`, #' `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`, #' optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, #' optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, -#' and `value_source` when `complete = TRUE`. +#' `value_source` when `complete = TRUE`, and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. #' Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, #' whose `completion` block reports `applied`, `rows_filled`, and the #' per-year `absence_means` rule that was applied. When `limit` is set, @@ -228,7 +240,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, basis = c("harmonized", "raw"), recipe = NULL, expenditure_concept = c("primary", "direct", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { # flow_prefixes no longer classifies rows (crosswalk subtype membership # does, per expenditure_concept) -- it only scopes the recipe-suggestion # machinery to this verb's recipe families (see R/suggestions.R; the @@ -251,7 +263,8 @@ cog_spending <- function(govid = NULL, years, category = NULL, limit = limit, offset = offset, state = state, - type = type + type = type, + fiscal_period = fiscal_period ) } @@ -280,7 +293,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, expenditure_concept = c("primary", "direct", "total"), revenue_concept = c("general", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { basis_explicit <- length(basis) == 1L basis <- match.arg(basis, c("harmonized", "raw")) # match.arg() itself throws a base `simpleError`, not an rlang-classed @@ -373,6 +386,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, ) } + .validate_fiscal_period_arg(fiscal_period) complete <- isTRUE(complete) if (complete && !is.null(recipe)) { .abort_complete_unsupported( @@ -572,6 +586,10 @@ cog_spending <- function(govid = NULL, years, category = NULL, result$notes <- .notes_column(result, direct_suppressed_info$notes) + # Appended last, after every other column, so the opt-in columns never move + # an existing one; FALSE (the default) leaves the result untouched. + if (fiscal_period) result <- .attach_fiscal_period(result, con) + # Determine expenditure_concept_note: only non-empty for "total", explains # how the IG leg was assembled from legacy-era aggregates. When the Direct # leg is suppressed for at least one requested (year, category), append an @@ -636,6 +654,9 @@ cog_spending <- function(govid = NULL, years, category = NULL, # every fleet-scale response body, which is the cost this path exists to # remove. NULL for a govid-named cohort, so that output is untouched. prov$scope$cohort <- .cohort_provenance(con, cohort) + if (fiscal_period) { + prov$transformations$fiscal_period <- .fiscal_period_provenance(result) + } attr(result, "provenance") <- prov attr(result, ".popyear_range") <- NULL # Attached here, after every downstream transform (per_capita/real-dollar diff --git a/_pkgdown.yml b/_pkgdown.yml index 6dc5c74..35b8ef7 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -22,6 +22,13 @@ reference: - cog_find_peers - cog_peer_compare - cog_geographic_rollup + - title: Fiscal years + desc: > + Which calendar months a survey year covers for each government, and the + survey year that covers a given date. + contents: + - cog_fiscal_periods + - cog_survey_year - title: Corpus metadata desc: > What the corpus contains, where a given result came from, and how to diff --git a/man/cog_balances.Rd b/man/cog_balances.Rd index b4ef178..eb56bed 100644 --- a/man/cog_balances.Rd +++ b/man/cog_balances.Rd @@ -15,7 +15,8 @@ cog_balances( state = NULL, type = NULL, limit = NULL, - offset = NULL + offset = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -86,6 +87,12 @@ returns everything. Cannot be combined with `recipe` -- see `offset` and \item{offset}{Rows to skip before `limit` starts counting (0-based). Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} + +\item{fiscal_period}{If `TRUE`, add `fiscal_year_end`, `period_start` and +`period_end` to every row, as [cog_spending()] does. Holdings are a stock +measured at the end of the fiscal year, so `period_end` is the date the +balance describes and `period_start` is the start of that fiscal year. +Defaults to `FALSE`, which leaves the result unchanged.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, @@ -94,8 +101,9 @@ Tibble with columns `year`, `canonical_govid`, `gov_name`, `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when `adjust_to_year` is set), and optional `amt_per_capita_real` (only when **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no - nominal per-capita column to deflate otherwise). Amounts are full US - dollars. + nominal per-capita column to deflate otherwise), and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. Amounts are + full US dollars. Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports diff --git a/man/cog_fiscal_periods.Rd b/man/cog_fiscal_periods.Rd new file mode 100644 index 0000000..61bc5e1 --- /dev/null +++ b/man/cog_fiscal_periods.Rd @@ -0,0 +1,80 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fiscal.R +\name{cog_fiscal_periods} +\alias{cog_fiscal_periods} +\title{Fiscal-year periods behind survey years} +\usage{ +cog_fiscal_periods(govid, years) +} +\arguments{ +\item{govid}{Character vector of `canonical_govid` values, or a data frame +with a `canonical_govid` column.} + +\item{years}{Integer vector of survey years.} +} +\value{ +Tibble with one row per requested government and year, in the + order `govid` x `years`: `canonical_govid`, `year`, `fiscal_year_end` + (integer MMDD), `period_start` and `period_end` (Dates, inclusive). +} +\description{ +Returns the calendar months each survey year covers for each government. +The corpus's `year` is the Census survey year, and a survey year covers +different months for different governments. +} +\section{The survey-year rule}{ + +Census assigns each government's fiscal year to a survey year by when the +fiscal year ended. Survey year Y holds the fiscal year that ended between +July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +`1231`. + +Survey year 2017 therefore covers, for a county, city or township: + + * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, + 2017, so July 2016 to June 2017. + * `930` (September 30): the fiscal year ending September 30, 2016, so + October 2015 to September 2016. + * `1231` (December 31): the fiscal year ending December 31, 2016, so + January to December 2016. + +A city with a December year end and a city with a June year end are both +"2017" in the corpus, but their figures are six months apart. A September +year end is nine months behind a June one. + +State governments (type 0) follow a different rule: survey year Y holds the +fiscal year ending in calendar year Y. For Alabama and Michigan (September +30) that means October 2016 to September 2017, and for Texas (August 31) +September 2016 to August 2017. Census's 2004 and 2006 technical +documentation lists these three states as exceptions to the June 30 window, +and the corpus agrees in survey year 2020, the one year checked. The same +documentation lists the District of Columbia, so it is treated the same +way. That treatment has not been checked against the data. For New York +(March 31) and the 46 states with a June 30 year end, the two rules give +the same months. + +A government can change its fiscal year end, so the periods are looked up +per government-year. Where the corpus has no `fiscal_year_end` for a +government-year, or the government has no rows that year, the period is +`NA`. + +From FY2023 the source files mark some rows as estimates carried forward +from an earlier year. The periods here describe the survey year, not the +year a carried-forward row's data pertain to. +} + +\examples{ +\dontrun{ +# A June 30 state, a September 30 state and a September 30 county +cog_fiscal_periods( + c("130000226087", "010000226085", "121011212191"), + years = 2020 +) +} +} +\seealso{ +[cog_survey_year()] for the reverse mapping; the `fiscal_period` + argument of [cog_spending()], [cog_revenue()] and [cog_balances()] to get + the same columns on every row. +} diff --git a/man/cog_revenue.Rd b/man/cog_revenue.Rd index e94a3d8..d53ae2d 100644 --- a/man/cog_revenue.Rd +++ b/man/cog_revenue.Rd @@ -17,7 +17,8 @@ cog_revenue( limit = NULL, offset = NULL, state = NULL, - type = NULL + type = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -154,13 +155,26 @@ Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} and `govids_missing` are empty -- there is no id list to report against -- and `provenance$scope$cohort` carries `state`, `type` and `n_governments` instead. A `govid`-named cohort reports exactly as before.} + +\item{fiscal_period}{If `TRUE`, add three columns to every row: +`fiscal_year_end` (the government's fiscal year end that survey year, an +MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +inclusive), the calendar months the row covers. `year` is the Census +survey year, which covers different months for different governments; +see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +from each government-year's own `fiscal_year_end`, so a change of fiscal +year is respected, and are `NA` where the corpus has no value. The result +also gains `provenance$transformations$fiscal_period`. Defaults to +`FALSE`, which returns exactly the columns and provenance this verb +returned before the argument existed.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`, optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, - and `value_source` when `complete = TRUE`. + `value_source` when `complete = TRUE`, and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. } \description{ Mirror of [cog_spending()] for revenue categories. One row per diff --git a/man/cog_spending.Rd b/man/cog_spending.Rd index 9738687..7f8fc3a 100644 --- a/man/cog_spending.Rd +++ b/man/cog_spending.Rd @@ -17,7 +17,8 @@ cog_spending( limit = NULL, offset = NULL, state = NULL, - type = NULL + type = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -174,13 +175,26 @@ Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} and `govids_missing` are empty -- there is no id list to report against -- and `provenance$scope$cohort` carries `state`, `type` and `n_governments` instead. A `govid`-named cohort reports exactly as before.} + +\item{fiscal_period}{If `TRUE`, add three columns to every row: +`fiscal_year_end` (the government's fiscal year end that survey year, an +MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +inclusive), the calendar months the row covers. `year` is the Census +survey year, which covers different months for different governments; +see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +from each government-year's own `fiscal_year_end`, so a change of fiscal +year is respected, and are `NA` where the corpus has no value. The result +also gains `provenance$transformations$fiscal_period`. Defaults to +`FALSE`, which returns exactly the columns and provenance this verb +returned before the argument existed.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`, optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, - and `value_source` when `complete = TRUE`. + `value_source` when `complete = TRUE`, and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, whose `completion` block reports `applied`, `rows_filled`, and the per-year `absence_means` rule that was applied. When `limit` is set, diff --git a/man/cog_survey_year.Rd b/man/cog_survey_year.Rd new file mode 100644 index 0000000..475a7e3 --- /dev/null +++ b/man/cog_survey_year.Rd @@ -0,0 +1,86 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fiscal.R +\name{cog_survey_year} +\alias{cog_survey_year} +\title{Survey year that covers a calendar date for a government} +\usage{ +cog_survey_year(govid, date) +} +\arguments{ +\item{govid}{Character vector of `canonical_govid` values, or a data frame +with a `canonical_govid` column.} + +\item{date}{Dates, or character strings in `"YYYY-MM-DD"` form. `govid` +and `date` must have the same length, or one of them must have length 1.} +} +\value{ +Tibble with one row per (`govid`, `date`) pair, in input order: + `canonical_govid`, `date`, `year` (the survey year, `NA` when it cannot + be determined), `fiscal_year_end`, `period_start` and `period_end`. +} +\description{ +Maps a date to the survey year whose fiscal period, for that government, +contains it. This is the reverse of [cog_fiscal_periods()]: use it to place +an event that happened on a known date against the corpus's `year`. +} +\details{ +The date can fall in survey year Y, Y+1 or Y+2 of its own calendar year Y, +depending on the government's fiscal year end. Each candidate is looked up +with the `fiscal_year_end` the corpus records for that government-year, so +a change of fiscal year is respected. When the government has no rows, or +no `fiscal_year_end`, in the survey year that would cover the date, the +result is `NA`; the function does not borrow another year's value. If a +change of fiscal year makes two survey years' periods overlap, the earlier +survey year is returned. +} +\section{The survey-year rule}{ + +Census assigns each government's fiscal year to a survey year by when the +fiscal year ended. Survey year Y holds the fiscal year that ended between +July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +`1231`. + +Survey year 2017 therefore covers, for a county, city or township: + + * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, + 2017, so July 2016 to June 2017. + * `930` (September 30): the fiscal year ending September 30, 2016, so + October 2015 to September 2016. + * `1231` (December 31): the fiscal year ending December 31, 2016, so + January to December 2016. + +A city with a December year end and a city with a June year end are both +"2017" in the corpus, but their figures are six months apart. A September +year end is nine months behind a June one. + +State governments (type 0) follow a different rule: survey year Y holds the +fiscal year ending in calendar year Y. For Alabama and Michigan (September +30) that means October 2016 to September 2017, and for Texas (August 31) +September 2016 to August 2017. Census's 2004 and 2006 technical +documentation lists these three states as exceptions to the June 30 window, +and the corpus agrees in survey year 2020, the one year checked. The same +documentation lists the District of Columbia, so it is treated the same +way. That treatment has not been checked against the data. For New York +(March 31) and the 46 states with a June 30 year end, the two rules give +the same months. + +A government can change its fiscal year end, so the periods are looked up +per government-year. Where the corpus has no `fiscal_year_end` for a +government-year, or the government has no rows that year, the period is +`NA`. + +From FY2023 the source files mark some rows as estimates carried forward +from an earlier year. The periods here describe the survey year, not the +year a carried-forward row's data pertain to. +} + +\examples{ +\dontrun{ +# April 15, 2020 for a September 30 county and a September 30 state +cog_survey_year(c("121011212191", "010000226085"), as.Date("2020-04-15")) +} +} +\seealso{ +[cog_fiscal_periods()] +} diff --git a/tests/testthat/test-fiscal-period.R b/tests/testthat/test-fiscal-period.R new file mode 100644 index 0000000..d4ec4c5 --- /dev/null +++ b/tests/testthat/test-fiscal-period.R @@ -0,0 +1,260 @@ +# tests/testthat/test-fiscal-period.R +# +# uscogdata#77: fiscal_year_end, period_start/period_end on the verbs, and the +# survey-year helpers. Expected values come from the bundled fixture (FY2011, +# FY2012, FY2019, FY2020), where these governments' fiscal_year_end is: +# Georgia state 630, Alabama state 930, Texas state 831, New York state 331, +# Washington DC (type 2) 930, Broward County FL 930. + +ga_state <- "130000226087" +al_state <- "010000226085" +tx_state <- "480000227541" +ny_state <- "360000227535" +dc <- "112001124214" +broward <- "121011212191" + +fiscal_cols <- c("fiscal_year_end", "period_start", "period_end") + +# --- the rule itself (no corpus) ---------------------------------------------- + +test_that("local governments follow the July 1 - June 30 survey-year rule", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 4), fiscal_year_end = c(630L, 930L, 1231L, 701L), + gov_type = rep(2L, 4), canonical_govid = rep("x", 4)) + expect_equal(p$period_start, + as.Date(c("2016-07-01", "2015-10-01", "2016-01-01", "2015-07-02"))) + expect_equal(p$period_end, + as.Date(c("2017-06-30", "2016-09-30", "2016-12-31", "2016-07-01"))) +}) + +test_that("state governments cover the fiscal year ending in the survey year", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 4), fiscal_year_end = c(930L, 831L, 331L, 630L), + gov_type = rep(0L, 4), canonical_govid = rep("x", 4)) + expect_equal(p$period_start, + as.Date(c("2016-10-01", "2016-09-01", "2016-04-01", "2016-07-01"))) + expect_equal(p$period_end, + as.Date(c("2017-09-30", "2017-08-31", "2017-03-31", "2017-06-30"))) +}) + +test_that("the District of Columbia follows the state-government rule", { + dc_p <- uscogdata:::.survey_period(2017L, 930L, 2L, dc) + city <- uscogdata:::.survey_period(2017L, 930L, 2L, "999999999999") + expect_equal(c(dc_p$period_start, dc_p$period_end), + as.Date(c("2016-10-01", "2017-09-30"))) + expect_equal(c(city$period_start, city$period_end), + as.Date(c("2015-10-01", "2016-09-30"))) +}) + +test_that("February year ends respect leap years", { + p <- uscogdata:::.survey_period( + year = c(2017L, 2017L, 2016L), fiscal_year_end = c(228L, 229L, 229L), + gov_type = rep(1L, 3), canonical_govid = rep("x", 3)) + expect_equal(p$period_end, + as.Date(c("2017-02-28", "2017-02-28", "2016-02-29"))) + expect_equal(p$period_start, + as.Date(c("2016-02-29", "2016-03-01", "2015-03-01"))) +}) + +test_that("missing, zero and impossible fiscal_year_end values give NA periods", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 5), fiscal_year_end = c(NA, 0L, 1331L, 1232L, 231L), + gov_type = rep(1L, 5), canonical_govid = rep("x", 5)) + expect_true(all(is.na(p$period_start))) + expect_true(all(is.na(p$period_end))) +}) + +test_that("a government-year whose rows disagree gets no fiscal_year_end", { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + DBI::dbExecute(con, "CREATE TABLE long AS SELECT * FROM (VALUES + ('a', 2020, 1, 630), ('a', 2020, 1, 630), + ('b', 2020, 2, 630), ('b', 2020, 2, 1231), + ('c', 2020, 3, NULL), ('c', 2020, 3, 930)) + AS t(canonical_govid, year, type, fiscal_year_end)") + fy <- uscogdata:::.fiscal_year_lookup(con, c("a", "b", "c"), 2020L) + fy <- fy[order(fy$canonical_govid), ] + expect_equal(fy$fiscal_year_end, c(630L, NA, 930L)) +}) + +# --- the verbs ----------------------------------------------------------------- + +test_that("fiscal_period defaults to FALSE and changes nothing when unset", { + skip_if_no_corpus() + for (f in list(cog_spending, cog_revenue, cog_balances)) { + expect_false(formals(f)$fiscal_period) + } + s0 <- cog_spending(c(ga_state, broward), 2020, category = "Highways") + s1 <- cog_spending(c(ga_state, broward), 2020, category = "Highways", + fiscal_period = FALSE) + expect_false(any(fiscal_cols %in% names(s0))) + expect_identical(names(s0), names(s1)) + expect_null(attr(s0, "provenance")$transformations$fiscal_period) + + b0 <- suppressMessages(cog_balances(al_state, 2020)) + expect_false(any(fiscal_cols %in% names(b0))) + expect_null(attr(b0, "provenance")$transformations$fiscal_period) +}) + +test_that("cog_spending(fiscal_period = TRUE) adds per-row periods at the end", { + skip_if_no_corpus() + plain <- cog_spending(c(ga_state, al_state, broward), 2019:2020, + category = "Highways") + r <- cog_spending(c(ga_state, al_state, broward), 2019:2020, + category = "Highways", fiscal_period = TRUE) + + expect_identical(names(r), c(names(plain), fiscal_cols)) + expect_equal(r[names(plain)], plain, ignore_attr = TRUE) + expect_type(r$fiscal_year_end, "integer") + expect_s3_class(r$period_start, "Date") + expect_s3_class(r$period_end, "Date") + + one <- function(id, y) { + unique(r[r$canonical_govid == id & r$year == y, fiscal_cols]) + } + expect_equal(one(ga_state, 2020)$period_start, as.Date("2019-07-01")) + expect_equal(one(ga_state, 2020)$period_end, as.Date("2020-06-30")) + expect_equal(one(al_state, 2020)$fiscal_year_end, 930L) + expect_equal(one(al_state, 2020)$period_end, as.Date("2020-09-30")) + expect_equal(one(broward, 2020)$fiscal_year_end, 930L) + expect_equal(one(broward, 2020)$period_start, as.Date("2018-10-01")) + expect_equal(one(broward, 2020)$period_end, as.Date("2019-09-30")) + expect_equal(one(broward, 2019)$period_end, as.Date("2018-09-30")) + + fp <- attr(r, "provenance")$transformations$fiscal_period + expect_true(fp$applied) + expect_equal(fp$rows_without_period, 0L) +}) + +test_that("cog_revenue and cog_balances take fiscal_period", { + skip_if_no_corpus() + tx <- suppressMessages(cog_revenue(tx_state, 2019, category = "IG Federal", + fiscal_period = TRUE)) + expect_gt(nrow(tx), 0L) + expect_true(all(tx$fiscal_year_end == 831L)) + expect_true(all(tx$period_start == as.Date("2018-09-01"))) + expect_true(all(tx$period_end == as.Date("2019-08-31"))) + + ny <- suppressMessages(cog_revenue(ny_state, 2020, category = "IG Federal", + fiscal_period = TRUE)) + expect_true(all(ny$period_start == as.Date("2019-04-01"))) + expect_true(all(ny$period_end == as.Date("2020-03-31"))) + + bal <- suppressMessages(cog_balances(al_state, 2020, fiscal_period = TRUE)) + expect_gt(nrow(bal), 0L) + expect_identical(tail(names(bal), 3L), fiscal_cols) + expect_true(all(bal$period_end == as.Date("2020-09-30"))) + expect_true(attr(bal, "provenance")$transformations$fiscal_period$applied) +}) + +test_that("fiscal_period composes with per_capita, adjust_to_year, limit and complete", { + skip_if_no_corpus() + r <- cog_spending(broward, 2020, per_capita = TRUE, adjust_to_year = 2020, + fiscal_period = TRUE, limit = 3) + expect_equal(nrow(r), 3L) + expect_identical(tail(names(r), 3L), fiscal_cols) + expect_true(all(r$period_end == as.Date("2019-09-30"))) + expect_false(is.null(attr(r, "total_rows"))) + + cpl <- suppressMessages(cog_spending(broward, 2020, category = "Police", + complete = TRUE, fiscal_period = TRUE)) + expect_true("value_source" %in% names(cpl)) + expect_identical(tail(names(cpl), 3L), fiscal_cols) +}) + +test_that("fiscal_period is validated", { + expect_error(cog_spending(ga_state, 2020, fiscal_period = NA), "fiscal_period") + expect_error(cog_spending(ga_state, 2020, fiscal_period = "yes"), "fiscal_period") + expect_error(cog_revenue(ga_state, 2020, fiscal_period = c(TRUE, FALSE)), + "fiscal_period") + expect_error(cog_balances(ga_state, 2020, fiscal_period = 1), "fiscal_period") +}) + +# --- cog_fiscal_periods() --------------------------------------------------------- + +test_that("cog_fiscal_periods() returns the govid x years grid, NA where unknown", { + skip_if_no_corpus() + fp <- cog_fiscal_periods(c(ga_state, al_state, dc, broward), c(2018, 2020)) + expect_s3_class(fp, "tbl_df") + expect_identical(names(fp), c("canonical_govid", "year", fiscal_cols)) + expect_equal(nrow(fp), 8L) + expect_identical(fp$canonical_govid, rep(c(ga_state, al_state, dc, broward), each = 2)) + expect_identical(fp$year, rep(c(2018L, 2020L), 4)) + + y20 <- fp[fp$year == 2020, ] + expect_equal(y20$fiscal_year_end, c(630L, 930L, 930L, 930L)) + expect_equal(y20$period_start, + as.Date(c("2019-07-01", "2019-10-01", "2019-10-01", "2018-10-01"))) + expect_equal(y20$period_end, + as.Date(c("2020-06-30", "2020-09-30", "2020-09-30", "2019-09-30"))) + + # FY2018 is not in the fixture. + expect_true(all(is.na(fp$period_end[fp$year == 2018]))) +}) + +test_that("cog_fiscal_periods() matches the verb columns", { + skip_if_no_corpus() + r <- cog_spending(c(ga_state, broward), 2019:2020, category = "Highways", + fiscal_period = TRUE) + fp <- cog_fiscal_periods(c(ga_state, broward), 2019:2020) + m <- merge(unique(as.data.frame(r[c("canonical_govid", "year", fiscal_cols)])), + as.data.frame(fp), by = c("canonical_govid", "year")) + expect_gt(nrow(m), 0L) + expect_equal(m$period_start.x, m$period_start.y) + expect_equal(m$period_end.x, m$period_end.y) +}) + +test_that("cog_fiscal_periods() validates its inputs", { + expect_error(cog_fiscal_periods(years = 2020), "govid") + expect_error(cog_fiscal_periods(ga_state), "years") + expect_error(cog_fiscal_periods(character(0), 2020), "govid") + expect_error(cog_fiscal_periods(ga_state, "2020"), "years") + expect_error(cog_fiscal_periods(ga_state, c(2020, NA)), "years") +}) + +# --- cog_survey_year() ------------------------------------------------------------- + +test_that("cog_survey_year() maps a date to the survey year that covers it", { + skip_if_no_corpus() + s <- cog_survey_year(c(broward, al_state, ga_state), as.Date("2019-04-15")) + expect_identical(names(s), c("canonical_govid", "date", "year", fiscal_cols)) + # Broward: FY ending 2019-09-30 is survey year 2020 (plain rule). + # Alabama state: FY ending 2019-09-30 is survey year 2019 (state rule). + # Georgia state: FY ending 2019-06-30 is survey year 2019. + expect_identical(s$year, c(2020L, 2019L, 2019L)) + expect_equal(s$period_end, as.Date(c("2019-09-30", "2019-09-30", "2019-06-30"))) +}) + +test_that("cog_survey_year() respects period boundaries and returns NA when it cannot tell", { + skip_if_no_corpus() + s <- cog_survey_year(broward, c("2018-10-01", "2019-09-30", "2018-09-30", + NA, "2019-12-01")) + # 2019-12-01 falls in survey year 2021, which the fixture does not carry. + expect_identical(s$year, c(2020L, 2020L, 2019L, NA, NA)) + expect_s3_class(s$date, "Date") +}) + +test_that("cog_survey_year() and cog_fiscal_periods() agree", { + skip_if_no_corpus() + fp <- cog_fiscal_periods(c(ga_state, al_state, broward), 2020) + s <- cog_survey_year(fp$canonical_govid, fp$period_start) + e <- cog_survey_year(fp$canonical_govid, fp$period_end) + expect_identical(s$year, fp$year) + expect_identical(e$year, fp$year) +}) + +test_that("cog_survey_year() validates its inputs", { + expect_error(cog_survey_year(date = "2020-01-01"), "govid") + expect_error(cog_survey_year(ga_state), "date") + expect_error(cog_survey_year(c(ga_state, broward), c("2020-01-01", "2020-02-01", + "2020-03-01")), + "same length") + expect_error(cog_survey_year(ga_state, "2020-13-01"), "not dates") + expect_error(cog_survey_year(ga_state, 20200101), "Date") + expect_error(cog_survey_year(ga_state, as.Date(character(0))), "empty") +}) + +test_that("new helpers are exported", { + expect_true(all(c("cog_fiscal_periods", "cog_survey_year") %in% + getNamespaceExports("uscogdata"))) +}) From ae21c79de577cd388ec7d5bd46eb4cc6f6988a55 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 18:10:48 +0000 Subject: [PATCH 3/3] docs: explain survey years where vignettes compare governments (#77) NEWS.md gains a development-version entry for fiscal_period, cog_fiscal_periods() and cog_survey_year(). total-spending.Rmd shows that its Florida rollup mixes a June 30 state (July 2019 to June 2020) with two September 30 counties (October 2018 to September 2019) under the same 2020. coverage-counters.Rmd counts the FY2012 fiscal year ends of Georgia cities (273 December 31, 196 June 30, 36 September 30) and what months they cover. population-denominators.Rmd notes that each cohort year is a survey year. The first two chunks run against the bundled fixture. Refs #77 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qbe6u2h83wgZdTgJgE4Xh5 --- NEWS.md | 34 +++++++++++++++++++++++++++ vignettes/coverage-counters.Rmd | 20 ++++++++++++++++ vignettes/population-denominators.Rmd | 6 +++++ vignettes/total-spending.Rmd | 17 ++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/NEWS.md b/NEWS.md index ea1282e..ed1ed00 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,37 @@ +# uscogdata (development version) + +## Fiscal-year periods for each row + +`year` is the Census survey year, and a survey year covers different months for +different governments. Census files a government's fiscal year under survey year Y +when it ended between July 1 of Y-1 and June 30 of Y. For a county, city or township, +"2017" means July 2016 to June 2017 with a June 30 year end, October 2015 to +September 2016 with a September 30 year end, and January to December 2016 with a +December 31 year end. + +`cog_spending()`, `cog_revenue()` and `cog_balances()` gain `fiscal_period = FALSE`. +With `fiscal_period = TRUE`, every row gains three columns at the end: +`fiscal_year_end`, an MMDD integer such as `630`, and `period_start` and `period_end`, +the first and last day the row covers. They come from that government-year's own +fiscal year end, so a change of fiscal year is respected, and they are `NA` where the +corpus has no value. The provenance gains `transformations$fiscal_period`. The default +returns the same columns and provenance as before. + +```r +cog_spending("121011212191", years = 2020, category = "Police", fiscal_period = TRUE) +``` + +`cog_fiscal_periods(govid, years)` returns the months each survey year covers for +each government. `cog_survey_year(govid, date)` returns the survey year whose period +contains a date, which places an event against the corpus's `year`. + +State governments follow a different rule: survey year Y holds the fiscal year ending +in calendar year Y. It changes the months for three states, Alabama and Michigan +(September 30) and Texas (August 31). Census's 2004 and 2006 technical documentation +lists them as exceptions to the June 30 window, and the published corpus agrees for +survey year 2020. The same documentation lists the District of Columbia, so it gets +the state rule too, although the data has not confirmed it. + # uscogdata 0.4.0 ## Cohorts can be named by predicate, not just by id diff --git a/vignettes/coverage-counters.Rmd b/vignettes/coverage-counters.Rmd index a3ef1c2..eea3a29 100644 --- a/vignettes/coverage-counters.Rmd +++ b/vignettes/coverage-counters.Rmd @@ -287,6 +287,26 @@ rows. calendar. Read `n_units_collected` to see how complete a year is: in FY2012, a census year, the fixture has spending rows for 534 of its 566 Georgia cities. +# Survey years and fiscal years + +The counters count governments per survey year, and a survey year covers different +months for different governments. Census files a fiscal year under survey year Y when +it ended between July 1 of Y-1 and June 30 of Y. Count the fiscal year ends of the +Georgia cities in FY2012: + +```{r} +fy2012 <- cog_fiscal_periods(ga_cities$canonical_govid, years = 2012) +table(fy2012$fiscal_year_end, useNA = "ifany") +``` + +In FY2012, 273 of the 566 Georgia cities close their books on December 31, 196 on +June 30 and 36 on September 30. Their FY2012 figures cover calendar 2011, July 2011 to +June 2012, and October 2010 to September 2011. The 31 cities with no value have no rows +in the corpus that year or no recorded year end. A city with a July 31 year end has an +FY2012 that ends on July 31, 2011, 11 months before the June cities' FY2012 ends. +Pass `fiscal_period = TRUE` to `cog_spending()` to see each row's months, and see +`?cog_fiscal_periods` for the rule. + # Summary - `n_units_expected` is the number of governments you named, or the number of peers diff --git a/vignettes/population-denominators.Rmd b/vignettes/population-denominators.Rmd index 1ea7f95..c1271e2 100644 --- a/vignettes/population-denominators.Rmd +++ b/vignettes/population-denominators.Rmd @@ -59,6 +59,12 @@ out <- purrr::map_dfr(years, function(y) { Each row in `out` has `cohort_year == year`, so a faceted plot shows cohort drift directly. +Each `year` is a Census survey year, which covers different months for different +governments: a city with a June 30 fiscal year end and one with a December 31 year end +report figures six months apart under the same `year`. Pass `fiscal_period = TRUE` to +`cog_spending()` to get each row's `period_start` and `period_end`, and see +`?cog_fiscal_periods` for the rule. + # Future direction `pop_source` is a column on the result, not a fixed value, so adding a new denominator (PEP from tidycensus, decennial counts, ACS time-series) is a join change rather than an API change. A future release may add `cog_spending(..., pop_source = "pep")` for users who need a single externally-audited series. diff --git a/vignettes/total-spending.Rmd b/vignettes/total-spending.Rmd index 78a8319..57c32a9 100644 --- a/vignettes/total-spending.Rmd +++ b/vignettes/total-spending.Rmd @@ -132,6 +132,23 @@ ga_state <- cog_spending( ga_state ``` +The rollup also mixes fiscal calendars. `year` is the Census survey year, and Census +files a fiscal year under survey year Y when it ended between July 1 of Y-1 and +June 30 of Y. `cog_fiscal_periods()` shows the months each government's 2020 covers: + +```{r} +cog_fiscal_periods( + c("120000226351", "121011212191", "121099101897"), # Florida, Broward, Palm Beach + years = 2020 +) +``` + +Florida's state government closes its books on June 30, so its 2020 covers July 2019 +to June 2020. Broward and Palm Beach counties close on September 30, so their 2020 +covers October 2018 to September 2019, and their figures end nine months before the +state's. Pass `fiscal_period = TRUE` to `cog_spending()` to get these columns on every +row, and see `?cog_fiscal_periods` for the rule and its exceptions. + Now the same rollup, but asking for `expenditure_concept = "total"`: ```{r, error = TRUE}