diff --git a/NAMESPACE b/NAMESPACE index 5d956a0..9e71307 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ export(cog_basket_unresolved) export(cog_categories) export(cog_explain) export(cog_find_peers) +export(cog_fiscal_periods) export(cog_geographic_rollup) export(cog_gov_search) export(cog_manifest) @@ -14,3 +15,4 @@ export(cog_peer_compare) export(cog_recipes) export(cog_revenue) export(cog_spending) +export(cog_survey_year) diff --git a/NEWS.md b/NEWS.md index ea1282e..ed1ed00 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,37 @@ +# uscogdata (development version) + +## Fiscal-year periods for each row + +`year` is the Census survey year, and a survey year covers different months for +different governments. Census files a government's fiscal year under survey year Y +when it ended between July 1 of Y-1 and June 30 of Y. For a county, city or township, +"2017" means July 2016 to June 2017 with a June 30 year end, October 2015 to +September 2016 with a September 30 year end, and January to December 2016 with a +December 31 year end. + +`cog_spending()`, `cog_revenue()` and `cog_balances()` gain `fiscal_period = FALSE`. +With `fiscal_period = TRUE`, every row gains three columns at the end: +`fiscal_year_end`, an MMDD integer such as `630`, and `period_start` and `period_end`, +the first and last day the row covers. They come from that government-year's own +fiscal year end, so a change of fiscal year is respected, and they are `NA` where the +corpus has no value. The provenance gains `transformations$fiscal_period`. The default +returns the same columns and provenance as before. + +```r +cog_spending("121011212191", years = 2020, category = "Police", fiscal_period = TRUE) +``` + +`cog_fiscal_periods(govid, years)` returns the months each survey year covers for +each government. `cog_survey_year(govid, date)` returns the survey year whose period +contains a date, which places an event against the corpus's `year`. + +State governments follow a different rule: survey year Y holds the fiscal year ending +in calendar year Y. It changes the months for three states, Alabama and Michigan +(September 30) and Texas (August 31). Census's 2004 and 2006 technical documentation +lists them as exceptions to the June 30 window, and the published corpus agrees for +survey year 2020. The same documentation lists the District of Columbia, so it gets +the state rule too, although the data has not confirmed it. + # uscogdata 0.4.0 ## Cohorts can be named by predicate, not just by id diff --git a/R/balances.R b/R/balances.R index 8ef4e60..651970d 100644 --- a/R/balances.R +++ b/R/balances.R @@ -53,6 +53,11 @@ #' `total_rows`. #' @param offset Rows to skip before `limit` starts counting (0-based). #' Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set. +#' @param fiscal_period If `TRUE`, add `fiscal_year_end`, `period_start` and +#' `period_end` to every row, as [cog_spending()] does. Holdings are a stock +#' measured at the end of the fiscal year, so `period_end` is the date the +#' balance describes and `period_start` is the start of that fiscal year. +#' Defaults to `FALSE`, which leaves the result unchanged. #' #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`, #' `balance_subtype`, `category`, `amt_nominal`, `codes_included`, @@ -60,8 +65,9 @@ #' `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when #' `adjust_to_year` is set), and optional `amt_per_capita_real` (only when #' **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no -#' nominal per-capita column to deflate otherwise). Amounts are full US -#' dollars. +#' nominal per-capita column to deflate otherwise), and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. Amounts are +#' full US dollars. #' #' Carries a `provenance` attribute matching #' `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports @@ -79,7 +85,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, per_capita = FALSE, adjust_to_year = NULL, basis = c("harmonized", "raw"), recipe = NULL, state = NULL, type = NULL, - limit = NULL, offset = NULL) { + limit = NULL, offset = NULL, fiscal_period = FALSE) { call <- match.call() basis <- match.arg(basis, c("harmonized", "raw")) # Coerce FIRST, validate second: .validate_verb_inputs() asserts @@ -102,6 +108,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, # validator's own doc comment for the incident that made that matter. .validate_verb_inputs(govid, years, category, per_capita, adjust_to_year, recipe) + .validate_fiscal_period_arg(fiscal_period) # Same semantics as the money verbs (R/pagination.R). Only the `recipe` # conflict applies here: cog_balances() has no `complete` argument, and a @@ -174,6 +181,7 @@ cog_balances <- function(govid = NULL, years, category = NULL, if (!is.null(adjust_to_year)) { result <- .attach_real_dollars(result, adjust_to_year, per_capita) } + if (fiscal_period) result <- .attach_fiscal_period(result, con) prov <- .build_provenance( verb = "cog_balances", call = call, govid = govid, years = years, @@ -189,6 +197,9 @@ cog_balances <- function(govid = NULL, years, category = NULL, prov$scope$govids_found <- scope$found prov$scope$govids_missing <- scope$missing prov$scope$cohort <- .cohort_provenance(con, cohort) + if (fiscal_period) { + prov$transformations$fiscal_period <- .fiscal_period_provenance(result) + } prov$balance_caveats <- .balance_caveats( con, prov$codes_summed$observed, years diff --git a/R/fiscal.R b/R/fiscal.R new file mode 100644 index 0000000..e8ac14e --- /dev/null +++ b/R/fiscal.R @@ -0,0 +1,377 @@ +# R/fiscal.R +# +# Fiscal-year timing (uscogdata#77). The corpus's `year` is the Census SURVEY +# year, and one survey year covers different calendar months for different +# governments. Census's rule (Local Government Finance FAQ, Q6): survey year Y +# holds each government's fiscal year that ENDED between July 1 of Y-1 and +# June 30 of Y. `long.fiscal_year_end` (an MMDD integer such as 630 or 1231) +# says which months that is for a given government-year. +# +# Two kinds of government follow a different rule: survey year Y holds the +# fiscal year ending in calendar year Y, even when that is after June 30. +# +# * State governments (type 0). Census's 2004 and 2006 technical +# documentation names the Alabama and Michigan (September 30) and Texas +# (August 31) state governments as exceptions "included as though they +# were part of the June 30 group". The published corpus agrees for survey +# year 2020: those three states' federal aid jumps with the CARES Act in +# 2020, as the June-30 states' does, while New York (March 31) and 15 +# large September-30 cities and counties jump in 2021 (measurements in +# uscogdata#77). For the other 47 states both rules give the same months. +# * The District of Columbia (type 2 here), which the same documentation +# lists with the September-30 exceptions. The 2020 data neither confirms +# nor rules this out, so it rests on the documentation alone. + +# canonical_govids that follow the state-government rule without being type 0. +.SURVEY_YEAR_SAME_YEAR_GOVIDS <- c( + "112001124214" # Washington, D.C. +) + +#' Days in a month, vectorized over year and month. +#' @noRd +.days_in_month <- function(year, month) { + dim <- c(31L, 28L, 31L, 30L, 31L, 30L, 31L, 31L, 30L, 31L, 30L, 31L) + leap <- (year %% 4L == 0L & year %% 100L != 0L) | year %% 400L == 0L + out <- dim[month] + out[month == 2L & leap] <- 29L + out +} + +#' Build Dates from year/month/day, clamping the day to the month's length +#' (so a February 29 year end falls on February 28 in a common year). +#' @noRd +.make_date <- function(year, month, day) { + out <- rep(as.Date(NA), length(year)) + ok <- !is.na(year) & !is.na(month) & !is.na(day) + if (any(ok)) { + d <- pmin(day[ok], .days_in_month(year[ok], month[ok])) + out[ok] <- as.Date(sprintf("%04d-%02d-%02d", year[ok], month[ok], d)) + } + out +} + +#' TRUE where `fiscal_year_end` is a usable MMDD value. +#' @noRd +.valid_fiscal_year_end <- function(fye) { + fye <- as.integer(fye) + m <- fye %/% 100L + d <- fye %% 100L + ok <- !is.na(fye) & !is.na(m) & m >= 1L & m <= 12L & d >= 1L + ok[ok] <- d[ok] <= .days_in_month(2000L, m[ok]) # 2000 is a leap year + ok +} + +#' The fiscal period a survey year covers for one government-year. +#' +#' Vectorized over its arguments, which must have equal length. +#' +#' @param year Integer survey year (the corpus's `year`). +#' @param fiscal_year_end Integer MMDD from `long.fiscal_year_end`. +#' @param gov_type Integer government type (0 = state). +#' @param canonical_govid Character canonical id. +#' @return List of Date vectors `period_start` and `period_end`; `NA` where +#' `fiscal_year_end` is missing or not a valid MMDD. +#' @noRd +.survey_period <- function(year, fiscal_year_end, gov_type, canonical_govid) { + year <- as.integer(year) + fye <- as.integer(fiscal_year_end) + ok <- .valid_fiscal_year_end(fye) + month <- fye %/% 100L + day <- fye %% 100L + + same_year <- (!is.na(gov_type) & as.integer(gov_type) == 0L) | + canonical_govid %in% .SURVEY_YEAR_SAME_YEAR_GOVIDS + end_year <- ifelse(same_year | month <= 6L, year, year - 1L) + + month[!ok] <- NA_integer_ + # The period starts the day after the same month-day one year earlier, + # which is when the government's previous fiscal year ended. + list( + period_start = .make_date(end_year - 1L, month, day) + 1L, + period_end = .make_date(end_year, month, day) + ) +} + +#' Look up each government-year's fiscal_year_end and type in `long`. +#' +#' A government-year whose rows carry more than one distinct value gets `NA` +#' rather than an arbitrary pick. None does in the fixture's four years. +#' +#' @return Tibble with `canonical_govid`, `year` (integer), `gov_type` +#' (integer) and `fiscal_year_end` (integer), one row per government-year +#' found in the corpus. +#' @noRd +.fiscal_year_lookup <- function(con, govids, years) { + govids <- unique(govids[!is.na(govids) & nzchar(govids)]) + years <- unique(as.integer(years[!is.na(years)])) + if (length(govids) == 0L || length(years) == 0L) { + return(tibble::tibble(canonical_govid = character(0), year = integer(0), + gov_type = integer(0), + fiscal_year_end = integer(0))) + } + out <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf( + "SELECT canonical_govid, year, + MIN(type) AS gov_type, + CASE WHEN COUNT(DISTINCT fiscal_year_end) = 1 + THEN MAX(fiscal_year_end) END AS fiscal_year_end + FROM long + WHERE canonical_govid IN (%s) AND year IN (%s) + GROUP BY canonical_govid, year", + .sql_lit_chr(govids), paste(years, collapse = ",") + ))) + out$year <- as.integer(out$year) + out$gov_type <- as.integer(out$gov_type) + out$fiscal_year_end <- as.integer(out$fiscal_year_end) + out +} + +#' Fiscal-year columns for every (canonical_govid, year) pair, in input order. +#' @return Tibble with `fiscal_year_end`, `period_start`, `period_end`. +#' @noRd +.fiscal_periods_for <- function(con, govids, years) { + years <- as.integer(years) + fy <- .fiscal_year_lookup(con, govids, years) + idx <- match(paste(govids, years), paste(fy$canonical_govid, fy$year)) + fye <- fy$fiscal_year_end[idx] + per <- .survey_period(years, fye, fy$gov_type[idx], govids) + tibble::tibble( + fiscal_year_end = ifelse(.valid_fiscal_year_end(fye), fye, NA_integer_), + period_start = per$period_start, + period_end = per$period_end + ) +} + +#' Append `fiscal_year_end`, `period_start` and `period_end` to a verb result. +#' +#' Joined by position rather than with `dplyr::left_join()`, so row order and +#' the result's attributes are untouched. Rows with no `canonical_govid` +#' (none today) get `NA`. +#' @noRd +.attach_fiscal_period <- function(result, con) { + if (nrow(result) == 0L) { + result$fiscal_year_end <- integer(0) + result$period_start <- as.Date(character(0)) + result$period_end <- as.Date(character(0)) + return(result) + } + fp <- .fiscal_periods_for(con, as.character(result$canonical_govid), + result$year) + result$fiscal_year_end <- fp$fiscal_year_end + result$period_start <- fp$period_start + result$period_end <- fp$period_end + result +} + +#' Provenance block recorded when a verb is called with `fiscal_period = TRUE`. +#' Absent otherwise, so default output keeps its existing shape. +#' @noRd +.fiscal_period_provenance <- function(result) { + list( + applied = TRUE, + source = "long.fiscal_year_end (MMDD), per government-year", + rule = paste( + "Survey year Y covers the fiscal year that ended between July 1 of Y-1", + "and June 30 of Y; state governments and the District of Columbia", + "cover the fiscal year ending in calendar year Y." + ), + rows_without_period = sum(is.na(result$period_end)) + ) +} + +#' @noRd +.validate_fiscal_period_arg <- function(fiscal_period) { + if (!is.logical(fiscal_period) || length(fiscal_period) != 1L || + is.na(fiscal_period)) { + cli::cli_abort("{.arg fiscal_period} must be {.code TRUE} or {.code FALSE}.") + } + invisible(fiscal_period) +} + +#' Fiscal-year periods behind survey years +#' +#' Returns the calendar months each survey year covers for each government. +#' The corpus's `year` is the Census survey year, and a survey year covers +#' different months for different governments. +#' +#' @section The survey-year rule: +#' Census assigns each government's fiscal year to a survey year by when the +#' fiscal year ended. Survey year Y holds the fiscal year that ended between +#' July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +#' fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +#' `1231`. +#' +#' Survey year 2017 therefore covers, for a county, city or township: +#' +#' * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, +#' 2017, so July 2016 to June 2017. +#' * `930` (September 30): the fiscal year ending September 30, 2016, so +#' October 2015 to September 2016. +#' * `1231` (December 31): the fiscal year ending December 31, 2016, so +#' January to December 2016. +#' +#' A city with a December year end and a city with a June year end are both +#' "2017" in the corpus, but their figures are six months apart. A September +#' year end is nine months behind a June one. +#' +#' State governments (type 0) follow a different rule: survey year Y holds the +#' fiscal year ending in calendar year Y. For Alabama and Michigan (September +#' 30) that means October 2016 to September 2017, and for Texas (August 31) +#' September 2016 to August 2017. Census's 2004 and 2006 technical +#' documentation lists these three states as exceptions to the June 30 window, +#' and the corpus agrees in survey year 2020, the one year checked. The same +#' documentation lists the District of Columbia, so it is treated the same +#' way. That treatment has not been checked against the data. For New York +#' (March 31) and the 46 states with a June 30 year end, the two rules give +#' the same months. +#' +#' A government can change its fiscal year end, so the periods are looked up +#' per government-year. Where the corpus has no `fiscal_year_end` for a +#' government-year, or the government has no rows that year, the period is +#' `NA`. +#' +#' From FY2023 the source files mark some rows as estimates carried forward +#' from an earlier year. The periods here describe the survey year, not the +#' year a carried-forward row's data pertain to. +#' +#' @param govid Character vector of `canonical_govid` values, or a data frame +#' with a `canonical_govid` column. +#' @param years Integer vector of survey years. +#' @return Tibble with one row per requested government and year, in the +#' order `govid` x `years`: `canonical_govid`, `year`, `fiscal_year_end` +#' (integer MMDD), `period_start` and `period_end` (Dates, inclusive). +#' @seealso [cog_survey_year()] for the reverse mapping; the `fiscal_period` +#' argument of [cog_spending()], [cog_revenue()] and [cog_balances()] to get +#' the same columns on every row. +#' @examples +#' \dontrun{ +#' # A June 30 state, a September 30 state and a September 30 county +#' cog_fiscal_periods( +#' c("130000226087", "010000226085", "121011212191"), +#' years = 2020 +#' ) +#' } +#' @export +cog_fiscal_periods <- function(govid, years) { + if (missing(govid)) cli::cli_abort("{.arg govid} is required.") + if (missing(years)) cli::cli_abort("{.arg years} is required.") + govid <- .coerce_govid_input(govid, arg = "govid") + if (length(govid) == 0L || anyNA(govid) || !all(nzchar(govid))) { + cli::cli_abort("{.arg govid} must name at least one government, with no missing or empty ids.") + } + if (!is.numeric(years) || length(years) == 0L || anyNA(years)) { + cli::cli_abort("{.arg years} must be a non-empty integer vector with no missing values.") + } + govid <- unique(govid) + years <- unique(as.integer(years)) + + con <- .ensure_session() + grid <- expand.grid(year = years, canonical_govid = govid, + stringsAsFactors = FALSE) + fp <- .fiscal_periods_for(con, grid$canonical_govid, grid$year) + tibble::tibble( + canonical_govid = grid$canonical_govid, + year = as.integer(grid$year), + fiscal_year_end = fp$fiscal_year_end, + period_start = fp$period_start, + period_end = fp$period_end + ) +} + +#' Survey year that covers a calendar date for a government +#' +#' Maps a date to the survey year whose fiscal period, for that government, +#' contains it. This is the reverse of [cog_fiscal_periods()]: use it to place +#' an event that happened on a known date against the corpus's `year`. +#' +#' The date can fall in survey year Y, Y+1 or Y+2 of its own calendar year Y, +#' depending on the government's fiscal year end. Each candidate is looked up +#' with the `fiscal_year_end` the corpus records for that government-year, so +#' a change of fiscal year is respected. When the government has no rows, or +#' no `fiscal_year_end`, in the survey year that would cover the date, the +#' result is `NA`; the function does not borrow another year's value. If a +#' change of fiscal year makes two survey years' periods overlap, the earlier +#' survey year is returned. +#' +#' @inheritSection cog_fiscal_periods The survey-year rule +#' @param govid Character vector of `canonical_govid` values, or a data frame +#' with a `canonical_govid` column. +#' @param date Dates, or character strings in `"YYYY-MM-DD"` form. `govid` +#' and `date` must have the same length, or one of them must have length 1. +#' @return Tibble with one row per (`govid`, `date`) pair, in input order: +#' `canonical_govid`, `date`, `year` (the survey year, `NA` when it cannot +#' be determined), `fiscal_year_end`, `period_start` and `period_end`. +#' @seealso [cog_fiscal_periods()] +#' @examples +#' \dontrun{ +#' # April 15, 2020 for a September 30 county and a September 30 state +#' cog_survey_year(c("121011212191", "010000226085"), as.Date("2020-04-15")) +#' } +#' @export +cog_survey_year <- function(govid, date) { + if (missing(govid)) cli::cli_abort("{.arg govid} is required.") + if (missing(date)) cli::cli_abort("{.arg date} is required.") + govid <- .coerce_govid_input(govid, arg = "govid") + if (length(govid) == 0L || anyNA(govid) || !all(nzchar(govid))) { + cli::cli_abort("{.arg govid} must name at least one government, with no missing or empty ids.") + } + date <- .coerce_date_input(date) + n <- max(length(govid), length(date)) + if (!length(govid) %in% c(1L, n) || !length(date) %in% c(1L, n)) { + cli::cli_abort(c( + "{.arg govid} and {.arg date} must have the same length, or one must have length 1.", + "x" = "{.arg govid} has length {length(govid)}; {.arg date} has length {length(date)}." + )) + } + govid <- rep_len(govid, n) + date <- rep_len(date, n) + + con <- .ensure_session() + cal_year <- as.integer(format(date, "%Y")) + cand <- data.frame( + i = rep(seq_len(n), each = 3L), + canonical_govid = rep(govid, each = 3L), + year = rep(cal_year, each = 3L) + rep(0:2, times = n), + stringsAsFactors = FALSE + ) + cand <- cand[!is.na(cand$year), , drop = FALSE] + fp <- .fiscal_periods_for(con, cand$canonical_govid, cand$year) + cand <- cbind(cand, fp) + d <- date[cand$i] + hit <- !is.na(cand$period_start) & d >= cand$period_start & + d <= cand$period_end + cand <- cand[hit, , drop = FALSE] + cand <- cand[order(cand$i, cand$year), , drop = FALSE] + first <- cand[!duplicated(cand$i), , drop = FALSE] + idx <- match(seq_len(n), first$i) + + tibble::tibble( + canonical_govid = govid, + date = date, + year = as.integer(first$year[idx]), + fiscal_year_end = first$fiscal_year_end[idx], + period_start = first$period_start[idx], + period_end = first$period_end[idx] + ) +} + +#' Coerce `date` to Date, refusing values that do not parse. +#' @noRd +.coerce_date_input <- function(date) { + if (length(date) == 0L) cli::cli_abort("{.arg date} must not be empty.") + if (inherits(date, "POSIXt")) date <- as.Date(date) + if (!inherits(date, "Date")) { + if (!is.character(date)) { + cli::cli_abort("{.arg date} must be a Date or a character vector of dates.") + } + parsed <- as.Date(date, format = "%Y-%m-%d") + bad <- !is.na(date) & is.na(parsed) + if (any(bad)) { + cli::cli_abort(c( + "{.arg date} has values that are not dates.", + "x" = "Could not parse: {.val {unique(date[bad])}}." + )) + } + date <- parsed + } + date +} diff --git a/R/revenue.R b/R/revenue.R index b6efe03..9dadd89 100644 --- a/R/revenue.R +++ b/R/revenue.R @@ -48,14 +48,15 @@ #' `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`, #' optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, #' optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, -#' and `value_source` when `complete = TRUE`. +#' `value_source` when `complete = TRUE`, and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. #' @export cog_revenue <- function(govid = NULL, years, category = NULL, per_capita = FALSE, adjust_to_year = NULL, basis = c("harmonized", "raw"), recipe = NULL, revenue_concept = c("general", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { # flow_prefixes no longer classifies rows (crosswalk revenue_subtype # membership does -- General Revenue, i.e. everything except # insurance_trust) -- it only scopes the recipe-suggestion machinery to @@ -78,6 +79,7 @@ cog_revenue <- function(govid = NULL, years, category = NULL, limit = limit, offset = offset, state = state, - type = type + type = type, + fiscal_period = fiscal_period ) } diff --git a/R/spending.R b/R/spending.R index 4129a27..b6166e4 100644 --- a/R/spending.R +++ b/R/spending.R @@ -210,11 +210,23 @@ #' and `govids_missing` are empty -- there is no id list to report against -- #' and `provenance$scope$cohort` carries `state`, `type` and #' `n_governments` instead. A `govid`-named cohort reports exactly as before. +#' @param fiscal_period If `TRUE`, add three columns to every row: +#' `fiscal_year_end` (the government's fiscal year end that survey year, an +#' MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +#' inclusive), the calendar months the row covers. `year` is the Census +#' survey year, which covers different months for different governments; +#' see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +#' from each government-year's own `fiscal_year_end`, so a change of fiscal +#' year is respected, and are `NA` where the corpus has no value. The result +#' also gains `provenance$transformations$fiscal_period`. Defaults to +#' `FALSE`, which returns exactly the columns and provenance this verb +#' returned before the argument existed. #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`, #' `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`, #' optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, #' optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, -#' and `value_source` when `complete = TRUE`. +#' `value_source` when `complete = TRUE`, and `fiscal_year_end`, +#' `period_start` and `period_end` when `fiscal_period = TRUE`. #' Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, #' whose `completion` block reports `applied`, `rows_filled`, and the #' per-year `absence_means` rule that was applied. When `limit` is set, @@ -228,7 +240,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, basis = c("harmonized", "raw"), recipe = NULL, expenditure_concept = c("primary", "direct", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { # flow_prefixes no longer classifies rows (crosswalk subtype membership # does, per expenditure_concept) -- it only scopes the recipe-suggestion # machinery to this verb's recipe families (see R/suggestions.R; the @@ -251,7 +263,8 @@ cog_spending <- function(govid = NULL, years, category = NULL, limit = limit, offset = offset, state = state, - type = type + type = type, + fiscal_period = fiscal_period ) } @@ -280,7 +293,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, expenditure_concept = c("primary", "direct", "total"), revenue_concept = c("general", "total"), complete = FALSE, limit = NULL, offset = NULL, - state = NULL, type = NULL) { + state = NULL, type = NULL, fiscal_period = FALSE) { basis_explicit <- length(basis) == 1L basis <- match.arg(basis, c("harmonized", "raw")) # match.arg() itself throws a base `simpleError`, not an rlang-classed @@ -373,6 +386,7 @@ cog_spending <- function(govid = NULL, years, category = NULL, ) } + .validate_fiscal_period_arg(fiscal_period) complete <- isTRUE(complete) if (complete && !is.null(recipe)) { .abort_complete_unsupported( @@ -572,6 +586,10 @@ cog_spending <- function(govid = NULL, years, category = NULL, result$notes <- .notes_column(result, direct_suppressed_info$notes) + # Appended last, after every other column, so the opt-in columns never move + # an existing one; FALSE (the default) leaves the result untouched. + if (fiscal_period) result <- .attach_fiscal_period(result, con) + # Determine expenditure_concept_note: only non-empty for "total", explains # how the IG leg was assembled from legacy-era aggregates. When the Direct # leg is suppressed for at least one requested (year, category), append an @@ -636,6 +654,9 @@ cog_spending <- function(govid = NULL, years, category = NULL, # every fleet-scale response body, which is the cost this path exists to # remove. NULL for a govid-named cohort, so that output is untouched. prov$scope$cohort <- .cohort_provenance(con, cohort) + if (fiscal_period) { + prov$transformations$fiscal_period <- .fiscal_period_provenance(result) + } attr(result, "provenance") <- prov attr(result, ".popyear_range") <- NULL # Attached here, after every downstream transform (per_capita/real-dollar diff --git a/_pkgdown.yml b/_pkgdown.yml index 6dc5c74..35b8ef7 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -22,6 +22,13 @@ reference: - cog_find_peers - cog_peer_compare - cog_geographic_rollup + - title: Fiscal years + desc: > + Which calendar months a survey year covers for each government, and the + survey year that covers a given date. + contents: + - cog_fiscal_periods + - cog_survey_year - title: Corpus metadata desc: > What the corpus contains, where a given result came from, and how to diff --git a/data-raw/measure_fiscal_year_end.R b/data-raw/measure_fiscal_year_end.R new file mode 100644 index 0000000..dd60f4c --- /dev/null +++ b/data-raw/measure_fiscal_year_end.R @@ -0,0 +1,101 @@ +# data-raw/measure_fiscal_year_end.R +# +# Measures how usable `long.fiscal_year_end` is before anything is built on it +# (uscogdata#77, step 1). Reads the long partitions of a corpus directly and +# reports, by survey year and government type (0-3): +# +# * government-years with no fiscal_year_end on any row (missing), +# * values of 0, and values that are not a valid MMDD date (implausible), +# * government-years whose rows disagree on the value, +# * the most common values, +# * how often a government's value changes between consecutive years, +# * every state government's value, and the District of Columbia's. +# +# A "government-year" is one (canonical_govid, year). fiscal_year_end is a +# header field, so every row of a government-year should carry the same value. +# +# Usage, from the package root: +# Rscript data-raw/measure_fiscal_year_end.R +# Rscript data-raw/measure_fiscal_year_end.R /path/to/corpus +# +# The default is the bundled fixture, which holds only FY2011, FY2012, FY2019 +# and FY2020. Do not generalize its results to other years. + +measure_fiscal_year_end <- function( + corpus = file.path("inst", "extdata", "fixture_corpus")) { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + run <- function(sql) DBI::dbExecute(con, sql) + q <- function(sql) DBI::dbGetQuery(con, sql) + + run(sprintf( + "CREATE VIEW l AS SELECT * FROM read_parquet('%s/data/long/*/*.parquet', + hive_partitioning = true)", + normalizePath(corpus, mustWork = TRUE) + )) + run("CREATE MACRO valid_mmdd(x) AS x IS NOT NULL + AND x // 100 BETWEEN 1 AND 12 + AND x % 100 BETWEEN 1 AND CASE x // 100 + WHEN 2 THEN 29 WHEN 4 THEN 30 WHEN 6 THEN 30 + WHEN 9 THEN 30 WHEN 11 THEN 30 ELSE 31 END") + run("CREATE TABLE gy AS + SELECT year, type, canonical_govid, + any_value(gov_name) AS gov_name, + any_value(fips_state) AS fips_state, + COUNT(DISTINCT fiscal_year_end) AS n_values, + COUNT(*) FILTER (WHERE fiscal_year_end IS NULL) AS n_null_rows, + COUNT(*) AS n_rows, + any_value(fiscal_year_end) + FILTER (WHERE fiscal_year_end IS NOT NULL) AS fye + FROM l GROUP BY 1, 2, 3") + + list( + partitions = q("SELECT year, COUNT(*) AS rows FROM l GROUP BY 1 ORDER BY 1"), + by_year_type = q( + "SELECT year, type, COUNT(*) AS gov_years, + SUM((fye IS NULL)::INT) AS missing, + SUM((fye = 0)::INT) AS zero, + SUM((fye IS NOT NULL AND fye <> 0 AND NOT valid_mmdd(fye))::INT) + AS implausible, + SUM((n_values > 1)::INT) AS conflicting_within_year, + SUM((n_null_rows > 0 AND n_null_rows < n_rows)::INT) + AS partly_missing + FROM gy GROUP BY 1, 2 ORDER BY 1, 2"), + row_null_share = q( + "SELECT year, COUNT(*) AS rows, + AVG((fiscal_year_end IS NULL)::INT) AS null_share + FROM l GROUP BY 1 ORDER BY 1"), + top_values = q( + "WITH c AS (SELECT year, fye, COUNT(*) AS n FROM gy GROUP BY 1, 2), + r AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY year ORDER BY n DESC) AS rk, + n / SUM(n) OVER (PARTITION BY year) AS share FROM c) + SELECT year, fye, n, share FROM r WHERE rk <= 6 ORDER BY year, rk"), + changes = q( + "SELECT a.year AS from_year, b.year AS to_year, a.type, + COUNT(*) AS in_both_years, SUM((a.fye <> b.fye)::INT) AS changed + FROM gy a JOIN gy b + ON a.canonical_govid = b.canonical_govid AND b.year = a.year + 1 + WHERE a.fye IS NOT NULL AND b.fye IS NOT NULL + GROUP BY 1, 2, 3 ORDER BY 1, 3"), + states = q( + "SELECT fips_state, any_value(gov_name) AS gov_name, year, fye + FROM gy WHERE type = 0 GROUP BY fips_state, year, fye + ORDER BY fips_state, year"), + district_of_columbia = q( + "SELECT year, type, canonical_govid, gov_name, fye + FROM gy WHERE fips_state = 11 ORDER BY year") + ) +} + +if (sys.nframe() == 0L) { + args <- commandArgs(trailingOnly = TRUE) + res <- if (length(args) > 0L) { + measure_fiscal_year_end(args[[1]]) + } else { + measure_fiscal_year_end() + } + for (nm in names(res)) { + cat("\n==", nm, "\n") + print(res[[nm]], row.names = FALSE) + } +} diff --git a/man/cog_balances.Rd b/man/cog_balances.Rd index b4ef178..eb56bed 100644 --- a/man/cog_balances.Rd +++ b/man/cog_balances.Rd @@ -15,7 +15,8 @@ cog_balances( state = NULL, type = NULL, limit = NULL, - offset = NULL + offset = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -86,6 +87,12 @@ returns everything. Cannot be combined with `recipe` -- see `offset` and \item{offset}{Rows to skip before `limit` starts counting (0-based). Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} + +\item{fiscal_period}{If `TRUE`, add `fiscal_year_end`, `period_start` and +`period_end` to every row, as [cog_spending()] does. Holdings are a stock +measured at the end of the fiscal year, so `period_end` is the date the +balance describes and `period_start` is the start of that fiscal year. +Defaults to `FALSE`, which leaves the result unchanged.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, @@ -94,8 +101,9 @@ Tibble with columns `year`, `canonical_govid`, `gov_name`, `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when `adjust_to_year` is set), and optional `amt_per_capita_real` (only when **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no - nominal per-capita column to deflate otherwise). Amounts are full US - dollars. + nominal per-capita column to deflate otherwise), and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. Amounts are + full US dollars. Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports diff --git a/man/cog_fiscal_periods.Rd b/man/cog_fiscal_periods.Rd new file mode 100644 index 0000000..61bc5e1 --- /dev/null +++ b/man/cog_fiscal_periods.Rd @@ -0,0 +1,80 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fiscal.R +\name{cog_fiscal_periods} +\alias{cog_fiscal_periods} +\title{Fiscal-year periods behind survey years} +\usage{ +cog_fiscal_periods(govid, years) +} +\arguments{ +\item{govid}{Character vector of `canonical_govid` values, or a data frame +with a `canonical_govid` column.} + +\item{years}{Integer vector of survey years.} +} +\value{ +Tibble with one row per requested government and year, in the + order `govid` x `years`: `canonical_govid`, `year`, `fiscal_year_end` + (integer MMDD), `period_start` and `period_end` (Dates, inclusive). +} +\description{ +Returns the calendar months each survey year covers for each government. +The corpus's `year` is the Census survey year, and a survey year covers +different months for different governments. +} +\section{The survey-year rule}{ + +Census assigns each government's fiscal year to a survey year by when the +fiscal year ended. Survey year Y holds the fiscal year that ended between +July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +`1231`. + +Survey year 2017 therefore covers, for a county, city or township: + + * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, + 2017, so July 2016 to June 2017. + * `930` (September 30): the fiscal year ending September 30, 2016, so + October 2015 to September 2016. + * `1231` (December 31): the fiscal year ending December 31, 2016, so + January to December 2016. + +A city with a December year end and a city with a June year end are both +"2017" in the corpus, but their figures are six months apart. A September +year end is nine months behind a June one. + +State governments (type 0) follow a different rule: survey year Y holds the +fiscal year ending in calendar year Y. For Alabama and Michigan (September +30) that means October 2016 to September 2017, and for Texas (August 31) +September 2016 to August 2017. Census's 2004 and 2006 technical +documentation lists these three states as exceptions to the June 30 window, +and the corpus agrees in survey year 2020, the one year checked. The same +documentation lists the District of Columbia, so it is treated the same +way. That treatment has not been checked against the data. For New York +(March 31) and the 46 states with a June 30 year end, the two rules give +the same months. + +A government can change its fiscal year end, so the periods are looked up +per government-year. Where the corpus has no `fiscal_year_end` for a +government-year, or the government has no rows that year, the period is +`NA`. + +From FY2023 the source files mark some rows as estimates carried forward +from an earlier year. The periods here describe the survey year, not the +year a carried-forward row's data pertain to. +} + +\examples{ +\dontrun{ +# A June 30 state, a September 30 state and a September 30 county +cog_fiscal_periods( + c("130000226087", "010000226085", "121011212191"), + years = 2020 +) +} +} +\seealso{ +[cog_survey_year()] for the reverse mapping; the `fiscal_period` + argument of [cog_spending()], [cog_revenue()] and [cog_balances()] to get + the same columns on every row. +} diff --git a/man/cog_revenue.Rd b/man/cog_revenue.Rd index e94a3d8..d53ae2d 100644 --- a/man/cog_revenue.Rd +++ b/man/cog_revenue.Rd @@ -17,7 +17,8 @@ cog_revenue( limit = NULL, offset = NULL, state = NULL, - type = NULL + type = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -154,13 +155,26 @@ Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} and `govids_missing` are empty -- there is no id list to report against -- and `provenance$scope$cohort` carries `state`, `type` and `n_governments` instead. A `govid`-named cohort reports exactly as before.} + +\item{fiscal_period}{If `TRUE`, add three columns to every row: +`fiscal_year_end` (the government's fiscal year end that survey year, an +MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +inclusive), the calendar months the row covers. `year` is the Census +survey year, which covers different months for different governments; +see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +from each government-year's own `fiscal_year_end`, so a change of fiscal +year is respected, and are `NA` where the corpus has no value. The result +also gains `provenance$transformations$fiscal_period`. Defaults to +`FALSE`, which returns exactly the columns and provenance this verb +returned before the argument existed.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`, optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, - and `value_source` when `complete = TRUE`. + `value_source` when `complete = TRUE`, and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. } \description{ Mirror of [cog_spending()] for revenue categories. One row per diff --git a/man/cog_spending.Rd b/man/cog_spending.Rd index 9738687..7f8fc3a 100644 --- a/man/cog_spending.Rd +++ b/man/cog_spending.Rd @@ -17,7 +17,8 @@ cog_spending( limit = NULL, offset = NULL, state = NULL, - type = NULL + type = NULL, + fiscal_period = FALSE ) } \arguments{ @@ -174,13 +175,26 @@ Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.} and `govids_missing` are empty -- there is no id list to report against -- and `provenance$scope$cohort` carries `state`, `type` and `n_governments` instead. A `govid`-named cohort reports exactly as before.} + +\item{fiscal_period}{If `TRUE`, add three columns to every row: +`fiscal_year_end` (the government's fiscal year end that survey year, an +MMDD integer such as `630`), and `period_start` and `period_end` (Dates, +inclusive), the calendar months the row covers. `year` is the Census +survey year, which covers different months for different governments; +see "The survey-year rule" in [cog_fiscal_periods()]. The columns come +from each government-year's own `fiscal_year_end`, so a change of fiscal +year is respected, and are `NA` where the corpus has no value. The result +also gains `provenance$transformations$fiscal_period`. Defaults to +`FALSE`, which returns exactly the columns and provenance this verb +returned before the argument existed.} } \value{ Tibble with columns `year`, `canonical_govid`, `gov_name`, `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`, optional `amt_per_capita_nominal`, optional `amt_per_capita_real`, optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`, - and `value_source` when `complete = TRUE`. + `value_source` when `complete = TRUE`, and `fiscal_year_end`, + `period_start` and `period_end` when `fiscal_period = TRUE`. Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`, whose `completion` block reports `applied`, `rows_filled`, and the per-year `absence_means` rule that was applied. When `limit` is set, diff --git a/man/cog_survey_year.Rd b/man/cog_survey_year.Rd new file mode 100644 index 0000000..475a7e3 --- /dev/null +++ b/man/cog_survey_year.Rd @@ -0,0 +1,86 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fiscal.R +\name{cog_survey_year} +\alias{cog_survey_year} +\title{Survey year that covers a calendar date for a government} +\usage{ +cog_survey_year(govid, date) +} +\arguments{ +\item{govid}{Character vector of `canonical_govid` values, or a data frame +with a `canonical_govid` column.} + +\item{date}{Dates, or character strings in `"YYYY-MM-DD"` form. `govid` +and `date` must have the same length, or one of them must have length 1.} +} +\value{ +Tibble with one row per (`govid`, `date`) pair, in input order: + `canonical_govid`, `date`, `year` (the survey year, `NA` when it cannot + be determined), `fiscal_year_end`, `period_start` and `period_end`. +} +\description{ +Maps a date to the survey year whose fiscal period, for that government, +contains it. This is the reverse of [cog_fiscal_periods()]: use it to place +an event that happened on a known date against the corpus's `year`. +} +\details{ +The date can fall in survey year Y, Y+1 or Y+2 of its own calendar year Y, +depending on the government's fiscal year end. Each candidate is looked up +with the `fiscal_year_end` the corpus records for that government-year, so +a change of fiscal year is respected. When the government has no rows, or +no `fiscal_year_end`, in the survey year that would cover the date, the +result is `NA`; the function does not borrow another year's value. If a +change of fiscal year makes two survey years' periods overlap, the earlier +survey year is returned. +} +\section{The survey-year rule}{ + +Census assigns each government's fiscal year to a survey year by when the +fiscal year ended. Survey year Y holds the fiscal year that ended between +July 1 of Y-1 and June 30 of Y. The corpus records each government-year's +fiscal year end as `fiscal_year_end`, an MMDD integer such as `630` or +`1231`. + +Survey year 2017 therefore covers, for a county, city or township: + + * `fiscal_year_end = 630` (June 30): the fiscal year ending June 30, + 2017, so July 2016 to June 2017. + * `930` (September 30): the fiscal year ending September 30, 2016, so + October 2015 to September 2016. + * `1231` (December 31): the fiscal year ending December 31, 2016, so + January to December 2016. + +A city with a December year end and a city with a June year end are both +"2017" in the corpus, but their figures are six months apart. A September +year end is nine months behind a June one. + +State governments (type 0) follow a different rule: survey year Y holds the +fiscal year ending in calendar year Y. For Alabama and Michigan (September +30) that means October 2016 to September 2017, and for Texas (August 31) +September 2016 to August 2017. Census's 2004 and 2006 technical +documentation lists these three states as exceptions to the June 30 window, +and the corpus agrees in survey year 2020, the one year checked. The same +documentation lists the District of Columbia, so it is treated the same +way. That treatment has not been checked against the data. For New York +(March 31) and the 46 states with a June 30 year end, the two rules give +the same months. + +A government can change its fiscal year end, so the periods are looked up +per government-year. Where the corpus has no `fiscal_year_end` for a +government-year, or the government has no rows that year, the period is +`NA`. + +From FY2023 the source files mark some rows as estimates carried forward +from an earlier year. The periods here describe the survey year, not the +year a carried-forward row's data pertain to. +} + +\examples{ +\dontrun{ +# April 15, 2020 for a September 30 county and a September 30 state +cog_survey_year(c("121011212191", "010000226085"), as.Date("2020-04-15")) +} +} +\seealso{ +[cog_fiscal_periods()] +} diff --git a/tests/testthat/test-fiscal-period.R b/tests/testthat/test-fiscal-period.R new file mode 100644 index 0000000..d4ec4c5 --- /dev/null +++ b/tests/testthat/test-fiscal-period.R @@ -0,0 +1,260 @@ +# tests/testthat/test-fiscal-period.R +# +# uscogdata#77: fiscal_year_end, period_start/period_end on the verbs, and the +# survey-year helpers. Expected values come from the bundled fixture (FY2011, +# FY2012, FY2019, FY2020), where these governments' fiscal_year_end is: +# Georgia state 630, Alabama state 930, Texas state 831, New York state 331, +# Washington DC (type 2) 930, Broward County FL 930. + +ga_state <- "130000226087" +al_state <- "010000226085" +tx_state <- "480000227541" +ny_state <- "360000227535" +dc <- "112001124214" +broward <- "121011212191" + +fiscal_cols <- c("fiscal_year_end", "period_start", "period_end") + +# --- the rule itself (no corpus) ---------------------------------------------- + +test_that("local governments follow the July 1 - June 30 survey-year rule", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 4), fiscal_year_end = c(630L, 930L, 1231L, 701L), + gov_type = rep(2L, 4), canonical_govid = rep("x", 4)) + expect_equal(p$period_start, + as.Date(c("2016-07-01", "2015-10-01", "2016-01-01", "2015-07-02"))) + expect_equal(p$period_end, + as.Date(c("2017-06-30", "2016-09-30", "2016-12-31", "2016-07-01"))) +}) + +test_that("state governments cover the fiscal year ending in the survey year", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 4), fiscal_year_end = c(930L, 831L, 331L, 630L), + gov_type = rep(0L, 4), canonical_govid = rep("x", 4)) + expect_equal(p$period_start, + as.Date(c("2016-10-01", "2016-09-01", "2016-04-01", "2016-07-01"))) + expect_equal(p$period_end, + as.Date(c("2017-09-30", "2017-08-31", "2017-03-31", "2017-06-30"))) +}) + +test_that("the District of Columbia follows the state-government rule", { + dc_p <- uscogdata:::.survey_period(2017L, 930L, 2L, dc) + city <- uscogdata:::.survey_period(2017L, 930L, 2L, "999999999999") + expect_equal(c(dc_p$period_start, dc_p$period_end), + as.Date(c("2016-10-01", "2017-09-30"))) + expect_equal(c(city$period_start, city$period_end), + as.Date(c("2015-10-01", "2016-09-30"))) +}) + +test_that("February year ends respect leap years", { + p <- uscogdata:::.survey_period( + year = c(2017L, 2017L, 2016L), fiscal_year_end = c(228L, 229L, 229L), + gov_type = rep(1L, 3), canonical_govid = rep("x", 3)) + expect_equal(p$period_end, + as.Date(c("2017-02-28", "2017-02-28", "2016-02-29"))) + expect_equal(p$period_start, + as.Date(c("2016-02-29", "2016-03-01", "2015-03-01"))) +}) + +test_that("missing, zero and impossible fiscal_year_end values give NA periods", { + p <- uscogdata:::.survey_period( + year = rep(2017L, 5), fiscal_year_end = c(NA, 0L, 1331L, 1232L, 231L), + gov_type = rep(1L, 5), canonical_govid = rep("x", 5)) + expect_true(all(is.na(p$period_start))) + expect_true(all(is.na(p$period_end))) +}) + +test_that("a government-year whose rows disagree gets no fiscal_year_end", { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + DBI::dbExecute(con, "CREATE TABLE long AS SELECT * FROM (VALUES + ('a', 2020, 1, 630), ('a', 2020, 1, 630), + ('b', 2020, 2, 630), ('b', 2020, 2, 1231), + ('c', 2020, 3, NULL), ('c', 2020, 3, 930)) + AS t(canonical_govid, year, type, fiscal_year_end)") + fy <- uscogdata:::.fiscal_year_lookup(con, c("a", "b", "c"), 2020L) + fy <- fy[order(fy$canonical_govid), ] + expect_equal(fy$fiscal_year_end, c(630L, NA, 930L)) +}) + +# --- the verbs ----------------------------------------------------------------- + +test_that("fiscal_period defaults to FALSE and changes nothing when unset", { + skip_if_no_corpus() + for (f in list(cog_spending, cog_revenue, cog_balances)) { + expect_false(formals(f)$fiscal_period) + } + s0 <- cog_spending(c(ga_state, broward), 2020, category = "Highways") + s1 <- cog_spending(c(ga_state, broward), 2020, category = "Highways", + fiscal_period = FALSE) + expect_false(any(fiscal_cols %in% names(s0))) + expect_identical(names(s0), names(s1)) + expect_null(attr(s0, "provenance")$transformations$fiscal_period) + + b0 <- suppressMessages(cog_balances(al_state, 2020)) + expect_false(any(fiscal_cols %in% names(b0))) + expect_null(attr(b0, "provenance")$transformations$fiscal_period) +}) + +test_that("cog_spending(fiscal_period = TRUE) adds per-row periods at the end", { + skip_if_no_corpus() + plain <- cog_spending(c(ga_state, al_state, broward), 2019:2020, + category = "Highways") + r <- cog_spending(c(ga_state, al_state, broward), 2019:2020, + category = "Highways", fiscal_period = TRUE) + + expect_identical(names(r), c(names(plain), fiscal_cols)) + expect_equal(r[names(plain)], plain, ignore_attr = TRUE) + expect_type(r$fiscal_year_end, "integer") + expect_s3_class(r$period_start, "Date") + expect_s3_class(r$period_end, "Date") + + one <- function(id, y) { + unique(r[r$canonical_govid == id & r$year == y, fiscal_cols]) + } + expect_equal(one(ga_state, 2020)$period_start, as.Date("2019-07-01")) + expect_equal(one(ga_state, 2020)$period_end, as.Date("2020-06-30")) + expect_equal(one(al_state, 2020)$fiscal_year_end, 930L) + expect_equal(one(al_state, 2020)$period_end, as.Date("2020-09-30")) + expect_equal(one(broward, 2020)$fiscal_year_end, 930L) + expect_equal(one(broward, 2020)$period_start, as.Date("2018-10-01")) + expect_equal(one(broward, 2020)$period_end, as.Date("2019-09-30")) + expect_equal(one(broward, 2019)$period_end, as.Date("2018-09-30")) + + fp <- attr(r, "provenance")$transformations$fiscal_period + expect_true(fp$applied) + expect_equal(fp$rows_without_period, 0L) +}) + +test_that("cog_revenue and cog_balances take fiscal_period", { + skip_if_no_corpus() + tx <- suppressMessages(cog_revenue(tx_state, 2019, category = "IG Federal", + fiscal_period = TRUE)) + expect_gt(nrow(tx), 0L) + expect_true(all(tx$fiscal_year_end == 831L)) + expect_true(all(tx$period_start == as.Date("2018-09-01"))) + expect_true(all(tx$period_end == as.Date("2019-08-31"))) + + ny <- suppressMessages(cog_revenue(ny_state, 2020, category = "IG Federal", + fiscal_period = TRUE)) + expect_true(all(ny$period_start == as.Date("2019-04-01"))) + expect_true(all(ny$period_end == as.Date("2020-03-31"))) + + bal <- suppressMessages(cog_balances(al_state, 2020, fiscal_period = TRUE)) + expect_gt(nrow(bal), 0L) + expect_identical(tail(names(bal), 3L), fiscal_cols) + expect_true(all(bal$period_end == as.Date("2020-09-30"))) + expect_true(attr(bal, "provenance")$transformations$fiscal_period$applied) +}) + +test_that("fiscal_period composes with per_capita, adjust_to_year, limit and complete", { + skip_if_no_corpus() + r <- cog_spending(broward, 2020, per_capita = TRUE, adjust_to_year = 2020, + fiscal_period = TRUE, limit = 3) + expect_equal(nrow(r), 3L) + expect_identical(tail(names(r), 3L), fiscal_cols) + expect_true(all(r$period_end == as.Date("2019-09-30"))) + expect_false(is.null(attr(r, "total_rows"))) + + cpl <- suppressMessages(cog_spending(broward, 2020, category = "Police", + complete = TRUE, fiscal_period = TRUE)) + expect_true("value_source" %in% names(cpl)) + expect_identical(tail(names(cpl), 3L), fiscal_cols) +}) + +test_that("fiscal_period is validated", { + expect_error(cog_spending(ga_state, 2020, fiscal_period = NA), "fiscal_period") + expect_error(cog_spending(ga_state, 2020, fiscal_period = "yes"), "fiscal_period") + expect_error(cog_revenue(ga_state, 2020, fiscal_period = c(TRUE, FALSE)), + "fiscal_period") + expect_error(cog_balances(ga_state, 2020, fiscal_period = 1), "fiscal_period") +}) + +# --- cog_fiscal_periods() --------------------------------------------------------- + +test_that("cog_fiscal_periods() returns the govid x years grid, NA where unknown", { + skip_if_no_corpus() + fp <- cog_fiscal_periods(c(ga_state, al_state, dc, broward), c(2018, 2020)) + expect_s3_class(fp, "tbl_df") + expect_identical(names(fp), c("canonical_govid", "year", fiscal_cols)) + expect_equal(nrow(fp), 8L) + expect_identical(fp$canonical_govid, rep(c(ga_state, al_state, dc, broward), each = 2)) + expect_identical(fp$year, rep(c(2018L, 2020L), 4)) + + y20 <- fp[fp$year == 2020, ] + expect_equal(y20$fiscal_year_end, c(630L, 930L, 930L, 930L)) + expect_equal(y20$period_start, + as.Date(c("2019-07-01", "2019-10-01", "2019-10-01", "2018-10-01"))) + expect_equal(y20$period_end, + as.Date(c("2020-06-30", "2020-09-30", "2020-09-30", "2019-09-30"))) + + # FY2018 is not in the fixture. + expect_true(all(is.na(fp$period_end[fp$year == 2018]))) +}) + +test_that("cog_fiscal_periods() matches the verb columns", { + skip_if_no_corpus() + r <- cog_spending(c(ga_state, broward), 2019:2020, category = "Highways", + fiscal_period = TRUE) + fp <- cog_fiscal_periods(c(ga_state, broward), 2019:2020) + m <- merge(unique(as.data.frame(r[c("canonical_govid", "year", fiscal_cols)])), + as.data.frame(fp), by = c("canonical_govid", "year")) + expect_gt(nrow(m), 0L) + expect_equal(m$period_start.x, m$period_start.y) + expect_equal(m$period_end.x, m$period_end.y) +}) + +test_that("cog_fiscal_periods() validates its inputs", { + expect_error(cog_fiscal_periods(years = 2020), "govid") + expect_error(cog_fiscal_periods(ga_state), "years") + expect_error(cog_fiscal_periods(character(0), 2020), "govid") + expect_error(cog_fiscal_periods(ga_state, "2020"), "years") + expect_error(cog_fiscal_periods(ga_state, c(2020, NA)), "years") +}) + +# --- cog_survey_year() ------------------------------------------------------------- + +test_that("cog_survey_year() maps a date to the survey year that covers it", { + skip_if_no_corpus() + s <- cog_survey_year(c(broward, al_state, ga_state), as.Date("2019-04-15")) + expect_identical(names(s), c("canonical_govid", "date", "year", fiscal_cols)) + # Broward: FY ending 2019-09-30 is survey year 2020 (plain rule). + # Alabama state: FY ending 2019-09-30 is survey year 2019 (state rule). + # Georgia state: FY ending 2019-06-30 is survey year 2019. + expect_identical(s$year, c(2020L, 2019L, 2019L)) + expect_equal(s$period_end, as.Date(c("2019-09-30", "2019-09-30", "2019-06-30"))) +}) + +test_that("cog_survey_year() respects period boundaries and returns NA when it cannot tell", { + skip_if_no_corpus() + s <- cog_survey_year(broward, c("2018-10-01", "2019-09-30", "2018-09-30", + NA, "2019-12-01")) + # 2019-12-01 falls in survey year 2021, which the fixture does not carry. + expect_identical(s$year, c(2020L, 2020L, 2019L, NA, NA)) + expect_s3_class(s$date, "Date") +}) + +test_that("cog_survey_year() and cog_fiscal_periods() agree", { + skip_if_no_corpus() + fp <- cog_fiscal_periods(c(ga_state, al_state, broward), 2020) + s <- cog_survey_year(fp$canonical_govid, fp$period_start) + e <- cog_survey_year(fp$canonical_govid, fp$period_end) + expect_identical(s$year, fp$year) + expect_identical(e$year, fp$year) +}) + +test_that("cog_survey_year() validates its inputs", { + expect_error(cog_survey_year(date = "2020-01-01"), "govid") + expect_error(cog_survey_year(ga_state), "date") + expect_error(cog_survey_year(c(ga_state, broward), c("2020-01-01", "2020-02-01", + "2020-03-01")), + "same length") + expect_error(cog_survey_year(ga_state, "2020-13-01"), "not dates") + expect_error(cog_survey_year(ga_state, 20200101), "Date") + expect_error(cog_survey_year(ga_state, as.Date(character(0))), "empty") +}) + +test_that("new helpers are exported", { + expect_true(all(c("cog_fiscal_periods", "cog_survey_year") %in% + getNamespaceExports("uscogdata"))) +}) diff --git a/vignettes/coverage-counters.Rmd b/vignettes/coverage-counters.Rmd index a3ef1c2..eea3a29 100644 --- a/vignettes/coverage-counters.Rmd +++ b/vignettes/coverage-counters.Rmd @@ -287,6 +287,26 @@ rows. calendar. Read `n_units_collected` to see how complete a year is: in FY2012, a census year, the fixture has spending rows for 534 of its 566 Georgia cities. +# Survey years and fiscal years + +The counters count governments per survey year, and a survey year covers different +months for different governments. Census files a fiscal year under survey year Y when +it ended between July 1 of Y-1 and June 30 of Y. Count the fiscal year ends of the +Georgia cities in FY2012: + +```{r} +fy2012 <- cog_fiscal_periods(ga_cities$canonical_govid, years = 2012) +table(fy2012$fiscal_year_end, useNA = "ifany") +``` + +In FY2012, 273 of the 566 Georgia cities close their books on December 31, 196 on +June 30 and 36 on September 30. Their FY2012 figures cover calendar 2011, July 2011 to +June 2012, and October 2010 to September 2011. The 31 cities with no value have no rows +in the corpus that year or no recorded year end. A city with a July 31 year end has an +FY2012 that ends on July 31, 2011, 11 months before the June cities' FY2012 ends. +Pass `fiscal_period = TRUE` to `cog_spending()` to see each row's months, and see +`?cog_fiscal_periods` for the rule. + # Summary - `n_units_expected` is the number of governments you named, or the number of peers diff --git a/vignettes/population-denominators.Rmd b/vignettes/population-denominators.Rmd index 1ea7f95..c1271e2 100644 --- a/vignettes/population-denominators.Rmd +++ b/vignettes/population-denominators.Rmd @@ -59,6 +59,12 @@ out <- purrr::map_dfr(years, function(y) { Each row in `out` has `cohort_year == year`, so a faceted plot shows cohort drift directly. +Each `year` is a Census survey year, which covers different months for different +governments: a city with a June 30 fiscal year end and one with a December 31 year end +report figures six months apart under the same `year`. Pass `fiscal_period = TRUE` to +`cog_spending()` to get each row's `period_start` and `period_end`, and see +`?cog_fiscal_periods` for the rule. + # Future direction `pop_source` is a column on the result, not a fixed value, so adding a new denominator (PEP from tidycensus, decennial counts, ACS time-series) is a join change rather than an API change. A future release may add `cog_spending(..., pop_source = "pep")` for users who need a single externally-audited series. diff --git a/vignettes/total-spending.Rmd b/vignettes/total-spending.Rmd index 78a8319..57c32a9 100644 --- a/vignettes/total-spending.Rmd +++ b/vignettes/total-spending.Rmd @@ -132,6 +132,23 @@ ga_state <- cog_spending( ga_state ``` +The rollup also mixes fiscal calendars. `year` is the Census survey year, and Census +files a fiscal year under survey year Y when it ended between July 1 of Y-1 and +June 30 of Y. `cog_fiscal_periods()` shows the months each government's 2020 covers: + +```{r} +cog_fiscal_periods( + c("120000226351", "121011212191", "121099101897"), # Florida, Broward, Palm Beach + years = 2020 +) +``` + +Florida's state government closes its books on June 30, so its 2020 covers July 2019 +to June 2020. Broward and Palm Beach counties close on September 30, so their 2020 +covers October 2018 to September 2019, and their figures end nine months before the +state's. Pass `fiscal_period = TRUE` to `cog_spending()` to get these columns on every +row, and see `?cog_fiscal_periods` for the rule and its exceptions. + Now the same rollup, but asking for `expenditure_concept = "total"`: ```{r, error = TRUE}