From 57b56460ac5ce27595c22257a38fe37c8b7b62dd Mon Sep 17 00:00:00 2001 From: William Curran-groome Date: Sun, 5 Jul 2026 22:20:09 +0300 Subject: [PATCH] code review and documentation updates --- .Rbuildignore | 10 + .github/workflows/R-CMD-check.yaml | 37 ++ .github/workflows/pkgdown.yaml | 1 + .gitignore | 1 + DESCRIPTION | 26 +- NAMESPACE | 1 + NEWS.md | 122 ++++ R/crosswalk-package.R | 24 + R/crosswalk_data.R | 166 ++++- R/get_county_events_crosswalk.R | 613 ++++++++++++++++++ R/get_crosswalk.R | 371 +++++++++-- R/get_crosswalk_chain.R | 3 +- R/get_ctdata_crosswalk.R | 27 +- R/get_geocorr_crosswalk.R | 31 +- R/get_nhgis_crosswalk.R | 110 +++- R/plan_crosswalk_chain.R | 183 +++++- R/sysdata.rda | Bin 0 -> 12881 bytes README.Rmd | 479 ++++++-------- README.md | 386 ++++------- _pkgdown.yml | 32 +- data-raw/README.md | 74 +++ data-raw/build_county_events_sysdata.R | 470 ++++++++++++++ man/crosswalk-package.Rd | 50 ++ man/crosswalk_data.Rd | 89 ++- man/figures/README-unnamed-chunk-6-1.png | Bin 29010 -> 0 bytes man/get_available_crosswalks.Rd | 57 +- man/get_crosswalk.Rd | 136 +++- man/list_nhgis_crosswalks.Rd | 28 +- renv/settings.json | 39 +- .../test-get_county_events_crosswalk-309.R | 14 + tests/testthat/test-crosswalk_data.R | 165 ++++- .../testthat/test-get_available_crosswalks.R | 47 +- .../test-get_county_events_crosswalk.R | 362 +++++++++++ tests/testthat/test-get_crosswalk.R | 136 ++++ tests/testthat/test-plan_crosswalk_chain.R | 146 +++++ vignettes/county-crosswalks.Rmd | 183 ++++++ vignettes/crosswalk.Rmd | 353 ++++++++++ vignettes/how-interpolation-works.Rmd | 293 +++++++++ vignettes/precompile.R | 15 + vignettes/standardizing-longitudinal-data.Rmd | 528 +++++++++------ .../standardizing-longitudinal-data.Rmd.orig | 244 +++++++ 41 files changed, 5057 insertions(+), 995 deletions(-) create mode 100644 .github/workflows/R-CMD-check.yaml create mode 100644 NEWS.md create mode 100644 R/crosswalk-package.R create mode 100644 R/get_county_events_crosswalk.R create mode 100644 R/sysdata.rda create mode 100644 data-raw/README.md create mode 100644 data-raw/build_county_events_sysdata.R create mode 100644 man/crosswalk-package.Rd delete mode 100644 man/figures/README-unnamed-chunk-6-1.png create mode 100644 tests/testthat/_problems/test-get_county_events_crosswalk-309.R create mode 100644 tests/testthat/test-get_county_events_crosswalk.R create mode 100644 vignettes/county-crosswalks.Rmd create mode 100644 vignettes/crosswalk.Rmd create mode 100644 vignettes/how-interpolation-works.Rmd create mode 100644 vignettes/precompile.R create mode 100644 vignettes/standardizing-longitudinal-data.Rmd.orig diff --git a/.Rbuildignore b/.Rbuildignore index f42d0d9..c2e7bc9 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -10,3 +10,13 @@ ^\.github$ ^doc$ ^Meta$ +^\.Rprofile$ +^\.gitattributes$ +^scripts$ +^AGENTS\.md$ +^CLAUDE\.md$ +^\.claude$ +^README\.Rmd$ +^data-raw$ +^vignettes/.*\.Rmd\.orig$ +^vignettes/precompile\.R$ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml new file mode 100644 index 0000000..d089dd1 --- /dev/null +++ b/.github/workflows/R-CMD-check.yaml @@ -0,0 +1,37 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +name: R-CMD-check.yaml + +permissions: read-all + +jobs: + R-CMD-check: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + IPUMS_API_KEY: ${{ secrets.IPUMS_API_KEY }} + CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} + R_KEEP_PKG_SOURCE: yes + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 81597b6..f02de2d 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -21,6 +21,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} IPUMS_API_KEY: ${{ secrets.IPUMS_API_KEY }} + CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} permissions: contents: write steps: diff --git a/.gitignore b/.gitignore index 97e56b2..4bef332 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ config.yml # Cached crosswalks /crosswalks-cache +data-raw/cache/ docs CLAUDE.md settings.local.json diff --git a/DESCRIPTION b/DESCRIPTION index 94baada..d3e2193 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,19 +1,28 @@ Package: crosswalk Type: Package -Title: streamlining inter-temporal and inter-geography crosswalking -Version: 0.0.0.9001 -Description: An R package providing a simple interface to access and apply crosswalks. +Title: Streamlined Inter-Temporal and Inter-Geography Crosswalking of Census Data +Version: 0.0.0.9002 +Description: Provides a standardized interface for accessing and applying + geographic crosswalks between United States Census geographies, both across + geographies (e.g., tracts to ZIP Code Tabulation Areas) and across time + (e.g., 2010 tracts to 2020 tracts). Crosswalks are sourced from the Missouri + Census Data Center's Geocorr applications, IPUMS NHGIS, the CT Data + Collaborative, and a curated registry of county boundary changes drawn from + Census Bureau documentation, and are returned with consistent column names + and rich metadata. Also interpolates count and non-count variables from + source to target geographies, with diagnostics describing join quality. License: MIT + file LICENSE -Authors@R: +Authors@R: person(given = "Will", family = "Curran-Groome", email = "wcurrangroome@urban.org", role = c("aut", "cre")) Encoding: UTF-8 -LazyData: true Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.3 -Depends: +URL: https://ui-research.github.io/crosswalk/, https://github.com/UI-Research/crosswalk +BugReports: https://github.com/UI-Research/crosswalk/issues +Depends: R (>= 4.1.0) Imports: - dplyr, + dplyr (>= 1.1.0), httr, httr2, janitor, @@ -21,6 +30,7 @@ Imports: readr, rlang, rvest, + stats, stringr, tibble, tidyr, @@ -28,10 +38,10 @@ Imports: utils Suggests: ggplot2, + scales, testthat (>= 3.0.0), tidycensus, knitr, rmarkdown Config/testthat/edition: 3 VignetteBuilder: knitr -URL: https://ui-research.github.io/crosswalk/ diff --git a/NAMESPACE b/NAMESPACE index 21319ec..f9d0a2b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,3 +4,4 @@ export(crosswalk_data) export(get_available_crosswalks) export(get_crosswalk) export(list_nhgis_crosswalks) +importFrom(rlang,":=") diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..e239b52 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,122 @@ +# crosswalk 0.0.0.9002 + +## New features + +* **Arbitrary-year county crosswalks.** `get_crosswalk()` now supports + county -> county crosswalks between any pair of years from 2000 through 2025 + (e.g., `2014 -> 2019` or `2014 -> 2023`), accounting for counties redefined + outside decennial censuses: Broomfield County, CO created (2002 vintage); + Clifton Forge, VA merge (2002); the 2007-2008 southeast Alaska + reorganizations (2008-2009); the York/Newport News, VA territory exchange + (2008); the Bedford, VA merge and Petersburg Borough, AK reorganization + (2014); the Shannon -> Oglala Lakota, SD and Wade Hampton -> Kusilvak, AK + FIPS changes (2015); the Valdez-Cordova, AK split (2020 vintage); and the + Connecticut planning regions (2022). Crosswalks compose all events in the + requested interval on top of an identity mapping over the full county + universe, with population-based allocation factors for splits and part + transfers. County crosswalks are forward-only. The event registry, mappings, + and county universes are curated from the Census Bureau's "Substantial + Changes to Counties and County Equivalent Entities" documentation by + `data-raw/build_county_events_sysdata.R` and shipped as internal package + data, so county requests work offline. +* **Sub-county geographies affected by county changes.** Tract, block group, + and block crosswalks are available for same-decade year pairs from 2010 + onward (e.g., tract `2014 -> 2019`), applying county-prefix relabels for + FIPS renames/merges and the CT Data Collaborative crosswalk for + Connecticut's 2022 change. Year pairs spanning a decennial census (e.g., + tract `2014 -> 2023`) are planned automatically as chains through NHGIS + crosswalks, including exact backward relabels to reach an NHGIS-served + vintage (e.g., tract `2017 -> 2023` plans `2017 -> 2014 -> 2020/22 -> 2023`). +* The chain planner's temporal-path search now spans multiple sources: hops + are tagged with the crosswalk source that serves them, and NHGIS is + preferred when both NHGIS and the county-events engine can serve a hop. + `get_available_crosswalks()` lists the new county-events combinations. +* Multi-step plans whose final GeoCorr step targets a year with county codes + differing from the GeoCorr vintage now emit a warning. +* `crosswalk_data()` now always computes and attaches the `join_quality` + attribute; `show_join_quality = FALSE` and `silent = TRUE` suppress only the + printed diagnostics. Previously, disabling the messages also silently + omitted the attribute. +* `get_available_crosswalks()` gains a `crosswalk_source` column identifying + which source serves each combination, and now lists each combination exactly + once: the 2020 <-> 2022 pairs, which both CTData and the county-events + engine can serve, are listed under `ctdata_2020_2022`, mirroring how + `get_crosswalk()` routes direct requests. + +## Documentation + +* Three new vignettes: `vignette("crosswalk")` (getting started; the core + workflow, multi-step chains, metadata, join quality, caching), + `vignette("how-interpolation-works")` (what the interpolation math assumes, + choosing a `weight`, accuracy against published values, diagnosing joins), + and `vignette("county-crosswalks")` (county change events and + arbitrary-year county crosswalks). +* The "Standardizing Longitudinal Data" vignette is now precomputed from + `vignettes/standardizing-longitudinal-data.Rmd.orig` (see + `vignettes/precompile.R`), so building the package requires neither network + access nor an IPUMS API key. It also now displays the join-quality + diagnostics it narrates and closes with a worked example of a split tract's + six-year series. +* The structures of the `join_quality` and `crosswalk_metadata` attributes are + now fully documented (in `?crosswalk_data` and `?get_crosswalk` + respectively), along with caching read-back behavior, the empty-crosswalk + return for nested geographies, and multi-step attribute semantics. +* Added a package-level help topic (`?crosswalk`) and a curated pkgdown + reference index; the README was slimmed to a quick start that links to the + vignettes. + +## Breaking changes + +* `get_crosswalk()` now filters NHGIS crosswalks to a single interpolation + weight matching the `weight` argument (`"population"`, `"housing"`, or + `"land"`). Previously, NHGIS crosswalks were returned in long format with one + row per source-target pair *per weighting factor* (typically eight rows per + pair), and applying them with `crosswalk_data()` silently multiply-counted + values by roughly the number of weighting factors. NHGIS `weighting_factor` + values no longer carry a `weight_` prefix (e.g., `"population"` rather than + `"weight_population"`). +* `crosswalk_data()` now errors when a crosswalk contains multiple rows per + source-target GEOID pair (which would multiply-count values), and when the + input data contains duplicated GEOIDs (panel data must be split by period and + crosswalked separately). +* `crosswalk_data()` no longer emits a `geoid = NA` row aggregating data rows + that failed to match the crosswalk; unmatched rows are dropped and reported + via the `join_quality` attribute. +* `crosswalk_data()` now renames only `target_geoid` -> `geoid` and + `target_geography_name` -> `geography_name` in its output; user columns + containing the string "target_" are no longer mangled. + +## Bug fixes + +* NHGIS 1990 tract GEOIDs are now right-padded with the implied `"00"` tract + suffix, so all 1990 tract GEOIDs have the standard 11 characters (previously + ~66% of rows had 9-character GEOIDs). +* Cached NHGIS and GeoCorr crosswalks are now read back with explicit column + types; previously, type-guessing could silently strip leading zeros from + GEOIDs (e.g., ZCTA `"00601"` became `601`) on cache reads. +* GeoCorr 2018 block-level queries are now chunked into groups of at most 13 + states; previously, chunks of 17 states exceeded the GeoCorr API limit. +* Block-level GeoCorr crosswalks assembled from multiple chunked requests no + longer retain the human-readable label row of each chunk after the first. +* Numeric crosswalk columns (`allocation_factor_target_to_source`, + `population_2020`, `housing_2020`, `land_area_sqmi`, etc.) are no longer + coerced to character by `get_crosswalk()`. +* When only `source_year` is provided, the GeoCorr version is now selected from + the source year; previously `get_crosswalk(..., source_year = 2015)` silently + used GeoCorr 2022 (2020 Census geography). +* The nested-geography check in `get_crosswalk()` now recognizes alternate + spellings (e.g., `"blockgroup"`, `"bg"`). +* NHGIS download failures (e.g., an invalid API key) now raise a clear error + with the HTTP status instead of a warning about an empty zip file. +* The `api_key` argument to the internal NHGIS fetcher is no longer ignored. +* County 2020 -> 2022 crosswalk metadata no longer claims all allocation + factors equal 1 (Connecticut county records are population-weighted). +* `get_ctdata_crosswalk()` now fails informatively when `tidycensus` is not + installed (county crosswalks only). + +# crosswalk 0.0.0.9001 + +* Support for chains of three or more crosswalks. +* Silent mode via `silent` argument and the `crosswalk.silent` option. +* `get_available_crosswalks()` lists all supported combinations. +* GeoCorr 2018 support for 2010s-era geographies. diff --git a/R/crosswalk-package.R b/R/crosswalk-package.R new file mode 100644 index 0000000..0e3ec2f --- /dev/null +++ b/R/crosswalk-package.R @@ -0,0 +1,24 @@ +#' @details +#' The package standardizes access to geographic crosswalks from four sources: +#' GeoCorr (Missouri Census Data Center) for same-year geography changes, IPUMS +#' NHGIS for cross-decade changes, the CT Data Collaborative for Connecticut's +#' 2020-2022 county changes, and a curated registry of county boundary change +#' events for county crosswalks between any pair of years from 2000 onward. +#' +#' Typical usage involves two functions: +#' - [get_crosswalk()] fetches one or more crosswalks (chaining them +#' automatically when both geography and year change) +#' - [crosswalk_data()] applies crosswalk(s) to a dataset, interpolating count +#' and non-count variables and reporting join quality +#' +#' Two helpers support discovery: +#' - [get_available_crosswalks()] lists every supported combination of source +#' and target geography and year +#' - [list_nhgis_crosswalks()] lists the NHGIS subset with source URLs +#' +#' To get oriented, see `vignette("crosswalk")`. For the interpolation +#' methodology and guidance on choosing weights, see +#' `vignette("how-interpolation-works")`. +#' +#' @keywords internal +"_PACKAGE" diff --git a/R/crosswalk_data.R b/R/crosswalk_data.R index beeb6cf..2cc4d32 100644 --- a/R/crosswalk_data.R +++ b/R/crosswalk_data.R @@ -18,10 +18,10 @@ #' Alternatively, a single crosswalk tibble can be provided for backwards #' compatibility. If NULL, the crosswalk will be fetched using `source_geography` #' and `target_geography` parameters. -#' @param source_geography Character or NULL. Source geography name. Required if -#' `crosswalk` is NULL. One of c("block", "block group", "tract", "place", -#' "county", "urban_area", "zcta", "puma", "cd118", "cd119", -#' "core_based_statistical_area"). +#' @param source_geography Character or NULL. Source geography name (e.g., +#' "tract", "zcta"). Required if `crosswalk` is NULL; passed to +#' `get_crosswalk()`. See `get_crosswalk()` for accepted names and +#' `get_available_crosswalks()` for all supported combinations. #' @param target_geography Character or NULL. Target geography name. Required if #' `crosswalk` is NULL. Same options as `source_geography`. #' @param source_year Numeric or NULL. Year of the source geography. If NULL and @@ -48,14 +48,21 @@ #' about join quality, including the number of data rows not matching the crosswalk #' and vice versa. For state-nested geographies (tract, county, block group, etc.), #' also reports state-level concentration of unmatched rows. Set to FALSE to -#' suppress these messages. Automatically suppressed when `silent = TRUE`. +#' suppress these messages; the `join_quality` attribute (see the "Join +#' quality diagnostics" section) is computed and attached to the result +#' either way. Messages are automatically suppressed when `silent = TRUE`. #' @param silent Logical. If `TRUE`, suppresses all informational messages and -#' warnings, including join quality diagnostics regardless of `show_join_quality`. +#' warnings, including join quality diagnostics regardless of `show_join_quality` +#' (the `join_quality` attribute is still attached to the result). #' Defaults to `getOption("crosswalk.silent", FALSE)`. Set #' `options(crosswalk.silent = TRUE)` to silence all calls by default. #' #' @return If `return_intermediate = FALSE` (default), a tibble with data summarized -#' to the final target geography. +#' to the final target geography. The target identifier column is named +#' `geoid` (and, when present in the crosswalk, the geography type is in +#' `geography_name`). Data rows whose GEOIDs do not match the crosswalk +#' cannot be allocated to a target geography and are dropped; use the +#' `join_quality` attribute to inspect them. #' #' If `return_intermediate = TRUE` and there are multiple crosswalk steps, a list with: #' \describe{ @@ -63,8 +70,51 @@ #' \item{intermediate}{A named list of intermediate results (step_1, step_2, etc.)} #' } #' -#' The returned tibble(s) include an attribute `crosswalk_metadata` from the -#' underlying crosswalk (access via `attr(result, "crosswalk_metadata")`). +#' The returned tibble(s) carry two attributes: +#' \describe{ +#' \item{crosswalk_metadata}{Provenance of the crosswalk that produced the +#' result (access via `attr(result, "crosswalk_metadata")`); see the +#' "Crosswalk metadata" section of [get_crosswalk()].} +#' \item{join_quality}{Statistics describing how well the data joined to +#' the crosswalk (access via `attr(result, "join_quality")`); see the +#' "Join quality diagnostics" section below. For multi-step crosswalks, +#' both attributes describe the *final* step; set +#' `return_intermediate = TRUE` to obtain each step's result with its +#' own attributes.} +#' } +#' +#' @section Join quality diagnostics: +#' The `join_quality` attribute is a list with the following elements: +#' \describe{ +#' \item{n_data_total}{Number of unique GEOIDs in the input data.} +#' \item{n_data_unmatched}{Number of data GEOIDs with no match in the +#' crosswalk. These rows cannot be allocated to a target geography and are +#' dropped from the result.} +#' \item{pct_data_unmatched}{`n_data_unmatched` as a percentage (0-100) of +#' `n_data_total`.} +#' \item{data_geoids_unmatched}{Character vector of the unmatched data GEOIDs.} +#' \item{state_analysis_data}{For state-nested geographies with unmatched data +#' rows, a list describing state-level concentration of the unmatched +#' GEOIDs: `state_counts` (a tibble of unmatched counts and percentages by +#' state FIPS), `top_states` (the three most-affected states), and +#' `is_concentrated` (logical; TRUE when any single state accounts for more +#' than 15% of unmatched GEOIDs). NULL when there are no unmatched rows or +#' state analysis is not applicable.} +#' \item{n_crosswalk_total}{Number of unique source GEOIDs in the crosswalk.} +#' \item{n_crosswalk_unmatched}{Number of crosswalk source GEOIDs absent from +#' the data (e.g., geographies with no observations).} +#' \item{pct_crosswalk_unmatched}{`n_crosswalk_unmatched` as a percentage +#' (0-100) of `n_crosswalk_total`.} +#' \item{crosswalk_geoids_unmatched}{Character vector of the crosswalk source +#' GEOIDs absent from the data.} +#' \item{state_analysis_crosswalk}{As `state_analysis_data`, but for crosswalk +#' GEOIDs absent from the data; otherwise NULL.} +#' \item{source_geography}{The source geography of the crosswalk step, when +#' known from its metadata.} +#' \item{state_analysis_applicable}{Logical; whether state-level analysis is +#' meaningful for this geography (FALSE for geographies that cross state +#' lines, such as ZCTAs).} +#' } #' #' @details #' **Two usage patterns**: @@ -81,7 +131,15 @@ #' that overlap with each target geography. #' #' **Non-count variables** (specified in `non_count_columns`) are interpolated using -#' a weighted mean, with the allocation factor serving as the weight. +#' a weighted mean, with the allocation factor serving as the weight. Note this +#' is an approximation: the allocation factor reflects each source geography's +#' share allocated to the target, not the relative size of the source +#' geographies, so weighted means are most accurate when source units are of +#' broadly similar size. +#' +#' **One row per GEOID**: `data` must contain at most one row per GEOID. For +#' panel data with multiple time periods, split the data by period (e.g., with +#' `purrr::map()`) and crosswalk each subset separately. #' #' **Automatic column detection**: If `count_columns` and `non_count_columns` are #' both NULL, the function will automatically detect columns based on naming prefixes: @@ -96,8 +154,13 @@ #' #' **Multi-step crosswalks**: When `get_crosswalk()` returns multiple crosswalks #' (for transformations that change both geography and year), this function -#' automatically applies them in sequence. +#' automatically applies them in sequence. The attributes on the final result +#' describe only the final step; to inspect join quality for earlier steps, set +#' `return_intermediate = TRUE` and examine the attributes of each intermediate +#' tibble. #' +#' @seealso [get_crosswalk()] to fetch and inspect crosswalks before applying +#' them; [get_available_crosswalks()] for all supported combinations. #' @export #' @examples #' \dontrun{ @@ -242,6 +305,17 @@ crosswalk_data <- function( stop("Column '", geoid_column, "' not found in data.") } + # Data must have one row per GEOID; duplicated GEOIDs (e.g., long/panel data + # with multiple time periods) would be collapsed incorrectly during aggregation + duplicated_geoids <- unique(data[[geoid_column]][duplicated(data[[geoid_column]])]) + if (length(duplicated_geoids) > 0) { + stop( + "Column '", geoid_column, "' contains duplicated GEOIDs (e.g., '", + duplicated_geoids[1], "'). crosswalk_data() expects one row per GEOID. ", + "For panel data with multiple time periods, split the data by period ", + "and crosswalk each subset separately.") + } + # Apply crosswalks sequentially n_steps <- length(crosswalk_list) intermediate_results <- list() @@ -553,11 +627,14 @@ format_join_quality_message <- function(join_quality, step_number, total_steps) #' @param total_steps Integer. Total number of steps. #' @param source_geography Character or NULL. The source geography type, used to #' determine if state-level analysis is applicable. -#' @return A list with join quality statistics (also prints messages if issues found). +#' @param print_messages Logical. Whether to print diagnostic messages when +#' issues are found. Statistics are computed and returned regardless. +#' @return A list with join quality statistics. #' @keywords internal #' @noRd report_join_quality <- function(data, crosswalk, geoid_column, step_number = 1, - total_steps = 1, source_geography = NULL) { + total_steps = 1, source_geography = NULL, + print_messages = TRUE) { # Ensure geoid columns are character for consistent comparison data_geoids <- data |> @@ -624,9 +701,11 @@ report_join_quality <- function(data, crosswalk, geoid_column, step_number = 1, ) # Print messages if there are issues - messages <- format_join_quality_message(join_quality, step_number, total_steps) - if (length(messages) > 0) { - purrr::walk(messages, cw_message) + if (print_messages) { + messages <- format_join_quality_message(join_quality, step_number, total_steps) + if (length(messages) > 0) { + purrr::walk(messages, cw_message) + } } return(join_quality) @@ -644,7 +723,8 @@ report_join_quality <- function(data, crosswalk, geoid_column, step_number = 1, #' @param non_count_columns Non-count variable columns #' @param step_number Integer. Current step number for multi-step reporting. #' @param total_steps Integer. Total number of steps for multi-step reporting. -#' @param show_join_quality Logical. Whether to report join quality diagnostics. +#' @param show_join_quality Logical. Whether to print join quality diagnostic +#' messages. The `join_quality` attribute is computed and attached regardless. #' @return Crosswalked data #' @keywords internal #' @noRd @@ -667,6 +747,23 @@ apply_single_crosswalk <- function( return(tibble::tibble()) } + # A valid crosswalk has exactly one row per source-target pair; duplicated + # pairs (e.g., a long-format crosswalk with one row per weighting factor) + # would multiply-count values during interpolation + if (anyDuplicated(crosswalk[c("source_geoid", "target_geoid")]) > 0) { + duplicate_pair_msg <- stringr::str_c( + "Crosswalk contains multiple rows per source-target GEOID pair, which ", + "would multiply-count values during interpolation.") + if ("weighting_factor" %in% names(crosswalk)) { + duplicate_pair_msg <- stringr::str_c( + duplicate_pair_msg, + " Filter the crosswalk to a single weighting_factor (found: ", + paste(unique(crosswalk$weighting_factor), collapse = ", "), + ") before calling crosswalk_data().") + } + stop(duplicate_pair_msg) + } + # Store metadata for later attachment crosswalk_metadata <- attr(crosswalk, "crosswalk_metadata") @@ -677,18 +774,16 @@ apply_single_crosswalk <- function( NULL } - # Report join quality (if enabled) - join_quality <- if (show_join_quality) { - report_join_quality( - data = data, - crosswalk = crosswalk, - geoid_column = geoid_column, - step_number = step_number, - total_steps = total_steps, - source_geography = source_geography) - } else { - NULL - } + # Compute join quality statistics; `show_join_quality` controls only whether + # diagnostic messages print, so the attribute is available even in silent runs + join_quality <- report_join_quality( + data = data, + crosswalk = crosswalk, + geoid_column = geoid_column, + step_number = step_number, + total_steps = total_steps, + source_geography = source_geography, + print_messages = show_join_quality) # Determine grouping columns (target_geography_name may not always be present) group_cols <- "target_geoid" @@ -718,11 +813,13 @@ apply_single_crosswalk <- function( ) - # Join crosswalk to data + # Join crosswalk to data. Inner join: data rows with no crosswalk match + # cannot be allocated to any target geography and are dropped (they are + # reported via the join-quality diagnostics above). result <- data |> dplyr::mutate( dplyr::across(dplyr::all_of(geoid_column), as.character)) |> - dplyr::left_join( + dplyr::inner_join( crosswalk, by = stats::setNames("source_geoid", geoid_column), relationship = "one-to-many") |> @@ -750,9 +847,10 @@ apply_single_crosswalk <- function( .cols = tidytable::all_of(c(current_count_cols, current_non_count_cols)), .fns = ~ tidytable::if_else(get(stringr::str_c(tidytable::cur_column(), "_validx")) > 0, .x, NA))) |> dplyr::select(-dplyr::matches("_validx$")) |> - dplyr::rename_with( - .cols = dplyr::everything(), - .fn = ~ stringr::str_remove_all(.x, "target_")) |> + dplyr::rename( + dplyr::any_of(c( + geoid = "target_geoid", + geography_name = "target_geography_name"))) |> tibble::as_tibble() # Attach metadata diff --git a/R/get_county_events_crosswalk.R b/R/get_county_events_crosswalk.R new file mode 100644 index 0000000..5f257c9 --- /dev/null +++ b/R/get_county_events_crosswalk.R @@ -0,0 +1,613 @@ +#' Get a County-Events Crosswalk Between Arbitrary Years +#' +#' Builds a crosswalk that accounts for substantial changes to counties and +#' county-equivalent entities occurring outside decennial censuses (renames, +#' merges, splits, and part transfers such as the 2013 Bedford, VA merge, the +#' 2015 Shannon -> Oglala Lakota, SD FIPS change, or the 2019 Valdez-Cordova, +#' AK split). Backed by a curated event registry shipped as internal package +#' data (see `data-raw/build_county_events_sysdata.R`). +#' +#' For a request `(geography, source_year, target_year)` the crosswalk is an +#' identity mapping over the full GEOID universe at `source_year`, composed +#' with the mappings of every event whose effective vintage falls in +#' `(source_year, target_year]`. Identity rows are included for all unchanged +#' units because `crosswalk_data()` drops unmatched rows. +#' +#' County crosswalks span any pair of years in range (counties are not +#' redefined at decennial censuses, so no NHGIS chaining is needed). +#' Sub-county crosswalks (tract, block group, block) are limited to year pairs +#' within the same tract-definition era (2010-2019 or 2020+); cross-decade +#' requests are planned as chains with NHGIS crosswalks by +#' `plan_crosswalk_chain()`. +#' +#' @param geography Character. One of "county", "tract", "block_group", or +#' "block" (aliases accepted). +#' @param source_year Numeric. Vintage year of the source GEOIDs. +#' @param target_year Numeric. Vintage year of the target GEOIDs. For county, +#' must be greater than `source_year` (forward-only). For sub-county +#' geographies, backward requests are allowed only when every event in the +#' interval is an exact 1:1 relabel at that geography level. +#' @param cache Directory path or NULL. Where to cache the crosswalk. +#' +#' @return A tibble with the standard crosswalk columns: `source_geoid`, +#' `target_geoid`, `source_geography_name`, `target_geography_name`, +#' `source_year`, `target_year`, `allocation_factor_source_to_target`, +#' `weighting_factor`, `state_fips`, with a `crosswalk_metadata` attribute. +#' @keywords internal +#' @noRd +get_county_events_crosswalk <- function(geography, + source_year, + target_year, + cache = NULL) { + + source_year <- as.numeric(source_year) + target_year <- as.numeric(target_year) + + geography_standardized <- standardize_geography_for_events(geography) + + if (is.na(geography_standardized)) { + stop( +"County-events crosswalks are only available for counties, tracts, block groups, and blocks. +The provided geography '", geography, "' is not supported.")} + + known_through <- county_events_meta$known_through_year + + if (min(source_year, target_year) < 2000 || max(source_year, target_year) > known_through) { + stop( +"County-events crosswalks cover ", 2000, " through ", known_through, ". +The provided years (", source_year, " -> ", target_year, ") are outside this range.")} + + if (source_year == target_year) { + stop( +"source_year and target_year are identical (", source_year, "); no county-events crosswalk is needed.")} + + is_backward <- source_year > target_year + + if (geography_standardized == "county" && is_backward) { + stop( +"County crosswalks from ", source_year, " to ", target_year, " are not supported. +Reversing county changes would require disaggregating merged counties, which is +not implemented. Swap source_year and target_year to crosswalk forward in time.")} + + if (geography_standardized != "county" && min(source_year, target_year) < 2010) { + stop( +"Sub-county county-events crosswalks are only available for 2010 and later. +The provided years (", source_year, " -> ", target_year, ") reach into the 2000s, +for which sub-county mappings are not curated. County-level crosswalks cover +2000 onward.")} + + if (geography_standardized != "county" && + tract_era(source_year) != tract_era(target_year)) { + stop( +"Sub-county county-events crosswalks require source_year and target_year in the +same tract-definition era (2010-2019 or 2020 onward). The provided years (", +source_year, " -> ", target_year, ") span a decennial census; use get_crosswalk(), +which chains county-events relabels with NHGIS inter-decade crosswalks.")} + + # Events in the (chronologically ordered) interval + interval_events <- county_events_in_interval(source_year, target_year) + + if (is_backward) { + reversal_col <- stringr::str_c("reversal_exact_", geography_standardized) + inexact <- interval_events[ + !interval_events[[reversal_col]] & + interval_events$subcounty_handling != "none", ] + if (nrow(inexact) > 0) { + stop( +"Backward crosswalks from ", source_year, " to ", target_year, " for ", +geography_standardized, " are not supported because the interval contains +county changes that are not exact 1:1 relabels:\n", +paste0("- ", inexact$description, collapse = "\n"), +"\nSwap source_year and target_year to crosswalk forward in time.")} + } + + # ---- Cache read ------------------------------------------------------------ + cache_path <- if (is.null(cache)) tempdir() else cache + csv_path <- file.path( + cache_path, + stringr::str_c( + "crosswalk_county_events_", source_year, "_to_", target_year, "_", + geography_standardized, ".csv")) + + if (file.exists(csv_path) && !is.null(cache)) { + cw_message(stringr::str_c( + "Reading county-events ", source_year, " -> ", target_year, " ", + geography_standardized, " crosswalk from cache.")) + result <- readr::read_csv( + csv_path, + col_types = readr::cols( + .default = readr::col_character(), + allocation_factor_source_to_target = readr::col_double()), + show_col_types = FALSE) + attr(result, "crosswalk_metadata") <- county_events_metadata( + geography, geography_standardized, source_year, target_year, + interval_events, csv_path, read_from_cache = TRUE) + return(result) + } + + # ---- Build ----------------------------------------------------------------- + if (nrow(interval_events) == 0) { + cw_message(stringr::str_c( + "No county boundary or code changes between ", source_year, " and ", + target_year, "; returning an identity crosswalk.")) + } else { + cw_message(stringr::str_c( + "Applying ", nrow(interval_events), " county change event(s) between ", + source_year, " and ", target_year, ":\n", + paste0("- ", interval_events$description, collapse = "\n"))) + } + + if (geography_standardized == "county") { + crosswalk <- build_county_events_county_crosswalk( + source_year, target_year, interval_events) + } else { + crosswalk <- build_county_events_subcounty_crosswalk( + geography_standardized, source_year, target_year, interval_events, cache) + } + + result <- crosswalk |> + dplyr::mutate( + source_geography_name = geography_standardized, + target_geography_name = geography_standardized, + source_year = as.character(source_year), + target_year = as.character(target_year), + state_fips = stringr::str_sub(source_geoid, 1, 2)) |> + dplyr::select( + source_geoid, target_geoid, + source_geography_name, target_geography_name, + source_year, target_year, + allocation_factor_source_to_target, + weighting_factor, state_fips) |> + dplyr::arrange(source_geoid, target_geoid) + + # ---- Cache write ----------------------------------------------------------- + if (!is.null(cache)) { + if (!dir.exists(cache_path)) { + dir.create(cache_path, recursive = TRUE) + } + readr::write_csv(result, csv_path) + cw_message(stringr::str_c("Cached to: ", csv_path)) + } + + attr(result, "crosswalk_metadata") <- county_events_metadata( + geography, geography_standardized, source_year, target_year, + interval_events, if (!is.null(cache)) csv_path else NULL, + read_from_cache = FALSE) + + return(result) +} + + +#' Standardize Geography Name for County-Events Crosswalks +#' @noRd +standardize_geography_for_events <- function(geography) { + geography_standardized <- geography |> + stringr::str_to_lower() |> + stringr::str_squish() |> + stringr::str_replace_all("_", " ") + + dplyr::case_when( + geography_standardized %in% c("county", "counties", "co", "cnty") ~ "county", + geography_standardized %in% c("tract", "tracts", "tr", "census tract") ~ "tract", + geography_standardized %in% c("block group", "blockgroup", "bg", "census block group") ~ "block_group", + geography_standardized %in% c("block", "blocks", "blk", "census block") ~ "block", + TRUE ~ NA_character_) +} + + +#' Tract-Definition Era for a Vintage Year +#' @noRd +tract_era <- function(year) { + dplyr::case_when( + year >= 2020 ~ 2020L, + year >= 2010 ~ 2010L, + TRUE ~ 2000L) +} + + +#' County Change Events Within a Year Interval +#' +#' Returns events with effective vintage in `(min(y1, y2), max(y1, y2)]`, +#' ordered chronologically. +#' @noRd +county_events_in_interval <- function(source_year, target_year) { + lower <- min(source_year, target_year) + upper <- max(source_year, target_year) + + county_events |> + dplyr::filter( + effective_vintage_year > lower, + effective_vintage_year <= upper) |> + dplyr::arrange(effective_vintage_year, event_id) +} + + +#' County GEOID Universe at a Vintage Year +#' +#' Rolls the decennial-base county universe forward through every event with +#' effective vintage in `(decade_base, year]`. +#' @noRd +county_universe_at <- function(year) { + base <- tract_era(year) + + universe <- county_universes |> + dplyr::filter(decade_base == base) |> + dplyr::pull(geoid) + + base_events <- county_events_in_interval(base, year) + + for (eid in base_events$event_id) { + mapping <- county_event_mappings |> dplyr::filter(event_id == eid) + universe <- union(setdiff(universe, mapping$source_geoid), mapping$target_geoid) + } + + sort(universe) +} + + +#' Compose an Event Mapping Onto a Working Crosswalk +#' +#' Rows of `crosswalk` whose `target_geoid` is a source of the event mapping +#' fan out to the event targets with multiplied factors; all other rows pass +#' through. The result has one row per source-target pair. +#' @noRd +compose_event_mapping <- function(crosswalk, mapping) { + step <- mapping |> + dplyr::select( + step_source = source_geoid, + step_target = target_geoid, + step_factor = allocation_factor, + step_weighting = weighting_factor) + + affected <- crosswalk |> + dplyr::inner_join( + step, by = c("target_geoid" = "step_source"), + relationship = "many-to-many") |> + dplyr::mutate( + target_geoid = step_target, + allocation_factor_source_to_target = + allocation_factor_source_to_target * step_factor, + weighting_factor = dplyr::if_else( + step_weighting == "identity", weighting_factor, step_weighting)) |> + dplyr::select(-step_target, -step_factor, -step_weighting) + + unaffected <- crosswalk |> + dplyr::anti_join(step, by = c("target_geoid" = "step_source")) + + dplyr::bind_rows(unaffected, affected) |> + dplyr::summarize( + allocation_factor_source_to_target = sum(allocation_factor_source_to_target), + weighting_factor = dplyr::if_else( + all(weighting_factor == "identity"), "identity", "population"), + .by = c(source_geoid, target_geoid)) +} + + +#' Build a County-Level County-Events Crosswalk +#' @noRd +build_county_events_county_crosswalk <- function(source_year, + target_year, + interval_events) { + universe <- county_universe_at(source_year) + + crosswalk <- tibble::tibble( + source_geoid = universe, + target_geoid = universe, + allocation_factor_source_to_target = 1, + weighting_factor = "identity") + + for (eid in interval_events$event_id) { + mapping <- county_event_mappings |> dplyr::filter(event_id == eid) + crosswalk <- compose_event_mapping(crosswalk, mapping) + } + + crosswalk +} + + +#' Sub-County Mapping Rows for One Event +#' +#' Builds the sub-county (tract/block-group/block) source -> target mapping for +#' a single event, according to its `subcounty_handling`. `current_geoids` is +#' the working GEOID universe the mapping will be applied to. Returns NULL for +#' events with no within-era sub-county effect. +#' @noRd +subcounty_event_mapping <- function(event_row, + geography_standardized, + current_geoids, + reverse = FALSE, + cache = NULL) { + handling <- event_row$subcounty_handling + + if (handling == "none") { + return(NULL) + } + + if (handling == "unsupported") { + stop( +"Sub-county crosswalks are not supported across this county change:\n- ", +event_row$description, "\nCounty-level crosswalks are available for this period.")} + + if (handling == "prefix_swap") { + county_mapping <- county_event_mappings |> + dplyr::filter(event_id == event_row$event_id) + stopifnot(all(county_mapping$allocation_factor == 1)) + + prefix_from <- county_mapping$source_geoid + prefix_to <- county_mapping$target_geoid + if (reverse) { + # Reversing a prefix swap is only valid for renames: for merges, only + # some of the target county's GEOIDs originated in the source county + stopifnot(event_row$event_type == "rename") + tmp <- prefix_from + prefix_from <- prefix_to + prefix_to <- tmp + } + + affected <- current_geoids[stringr::str_sub(current_geoids, 1, 5) %in% prefix_from] + if (length(affected) == 0) { + return(NULL) + } + new_prefix <- prefix_to[match(stringr::str_sub(affected, 1, 5), prefix_from)] + + return(tibble::tibble( + source_geoid = affected, + target_geoid = stringr::str_c(new_prefix, stringr::str_sub(affected, 6)), + allocation_factor = 1, + weighting_factor = "identity")) + } + + if (handling == "explicit_rows") { + mapping <- county_event_subcounty_mappings |> + dplyr::filter( + event_id == event_row$event_id, + geography == geography_standardized) |> + dplyr::select(source_geoid, target_geoid, allocation_factor) |> + dplyr::mutate(weighting_factor = dplyr::if_else( + allocation_factor == 1, "identity", "population")) + + if (nrow(mapping) == 0) { + stop( +"Sub-county (", geography_standardized, ") mappings for this county change have +not been curated:\n- ", event_row$description)} + + if (reverse) { + stopifnot(all(mapping$allocation_factor == 1)) + mapping <- mapping |> + dplyr::rename(source_geoid = target_geoid, target_geoid = source_geoid) + } + + return(mapping) + } + + if (handling == "ctdata_runtime") { + ct <- get_ctdata_crosswalk( + geography = geography_standardized, + source_year = if (reverse) 2022 else 2020, + target_year = if (reverse) 2020 else 2022, + cache = cache) + + return(ct |> + dplyr::filter(state_fips == "09") |> + dplyr::select( + source_geoid, target_geoid, + allocation_factor = allocation_factor_source_to_target, + weighting_factor)) + } + + stop("Unknown subcounty_handling: ", handling) +} + + +#' Drop Sub-County GEOIDs That Retain an Abolished County Prefix +#' +#' After an event is applied, no GEOID may keep a county prefix the event +#' abolished. GEOIDs missing from the event's sub-county mapping (e.g., the +#' four Connecticut water tracts absent from the CT Data crosswalk) would +#' otherwise survive as identity rows with stale prefixes; drop them instead, +#' mirroring `get_ctdata_crosswalk()`, which also omits them. +#' @noRd +drop_abolished_prefixes <- function(geoids, event_row, reverse = FALSE) { + county_mapping <- county_event_mappings |> + dplyr::filter(event_id == event_row$event_id) + + from_counties <- county_mapping$source_geoid + to_counties <- county_mapping$target_geoid + if (reverse) { + tmp <- from_counties + from_counties <- to_counties + to_counties <- tmp + } + abolished <- setdiff(from_counties, to_counties) + + stale <- stringr::str_sub(geoids, 1, 5) %in% abolished + if (any(stale)) { + cw_message(stringr::str_c( + "Dropping ", sum(stale), " GEOID(s) not covered by the sub-county ", + "mapping for: ", event_row$description)) + } + geoids[!stale] +} + + +#' Build a Sub-County County-Events Crosswalk +#' +#' Enumerates the era-anchor GEOID universe from the NHGIS decennial crosswalk +#' for the geography, relabels it to `source_year` by applying prior events in +#' the era, then composes the events in `(source_year, target_year]`. +#' @noRd +build_county_events_subcounty_crosswalk <- function(geography_standardized, + source_year, + target_year, + interval_events, + cache = NULL) { + era <- tract_era(source_year) + is_backward <- source_year > target_year + + cw_message(stringr::str_c( + "Enumerating the ", era, "-era ", geography_standardized, + " universe from the NHGIS 2010 -> 2020 crosswalk...")) + + nhgis_crosswalk <- get_nhgis_crosswalk( + source_year = 2010, + source_geography = geography_standardized, + target_year = 2020, + target_geography = geography_standardized, + cache = cache) + + universe_col <- if (era == 2010L) "source_geoid" else "target_geoid" + universe <- nhgis_crosswalk[[universe_col]] |> unique() |> sort() + + # Relabel the era-anchor universe to source_year through prior events + prior_events <- county_events_in_interval(era, source_year) + + for (i in seq_len(nrow(prior_events))) { + mapping <- subcounty_event_mapping( + prior_events[i, ], geography_standardized, universe, reverse = FALSE, + cache = cache) + if (!is.null(mapping)) { + stopifnot(all(mapping$allocation_factor == 1)) + relabeled <- dplyr::coalesce( + mapping$target_geoid[match(universe, mapping$source_geoid)], universe) + universe <- sort(unique( + drop_abolished_prefixes(relabeled, prior_events[i, ], reverse = FALSE))) + } + } + + crosswalk <- tibble::tibble( + source_geoid = universe, + target_geoid = universe, + allocation_factor_source_to_target = 1, + weighting_factor = "identity") + + event_order <- if (is_backward) rev(seq_len(nrow(interval_events))) else seq_len(nrow(interval_events)) + + for (i in event_order) { + mapping <- subcounty_event_mapping( + interval_events[i, ], geography_standardized, + unique(crosswalk$target_geoid), reverse = is_backward, cache = cache) + if (!is.null(mapping)) { + crosswalk <- compose_event_mapping(crosswalk, mapping) |> + dplyr::filter(target_geoid %in% drop_abolished_prefixes( + target_geoid, interval_events[i, ], reverse = is_backward)) + } + } + + crosswalk +} + + +#' List Temporal Edges Provided by the County-Events Engine +#' +#' Enumerates the (source_year, target_year) pairs the county-events engine can +#' serve for a geography, for use by the chain planner and +#' `get_available_crosswalks()`. +#' +#' County: all forward pairs in `[2000, known_through_year]`. Sub-county: +#' same-era forward pairs in `[2010, known_through_year]`, plus backward pairs +#' whose interval contains only exact 1:1 relabels (or no events), excluding +#' any interval containing an event whose sub-county mappings are unsupported +#' or not yet curated. +#' +#' @param geography_std Character. Standardized geography name ("county", +#' "tract", "block_group", or "block"). +#' @return A tibble with columns `source_year`, `target_year` (character). +#' @keywords internal +#' @noRd +list_county_events_edges <- function(geography_std) { + known_through <- county_events_meta$known_through_year + + if (geography_std == "county") { + return( + tidyr::crossing( + source_year = 2000:known_through, + target_year = 2000:known_through) |> + dplyr::filter(source_year < target_year) |> + dplyr::mutate( + source_year = as.character(source_year), + target_year = as.character(target_year))) + } + + if (!geography_std %in% c("tract", "block_group", "block")) { + return(tibble::tibble(source_year = character(), target_year = character())) + } + + eras <- list(2010:2019, 2020:known_through) + + event_supported <- function(event_row) { + handling <- event_row$subcounty_handling + if (handling %in% c("none", "prefix_swap", "ctdata_runtime")) return(TRUE) + if (handling == "unsupported") return(FALSE) + # explicit_rows: supported only once curated + nrow(county_event_subcounty_mappings |> + dplyr::filter( + event_id == event_row$event_id, + geography == geography_std)) > 0 + } + + event_reversal_exact <- function(event_row) { + if (event_row$subcounty_handling == "none") return(TRUE) + event_row[[stringr::str_c("reversal_exact_", geography_std)]] + } + + purrr::map_dfr(eras, function(era_years) { + tidyr::crossing(y1 = era_years, y2 = era_years) |> + dplyr::filter(y1 != y2) |> + dplyr::rowwise() |> + dplyr::filter({ + evs <- county_events_in_interval(y1, y2) + supported <- nrow(evs) == 0 || + all(purrr::map_lgl(seq_len(nrow(evs)), ~ event_supported(evs[.x, ]))) + exact_ok <- y1 < y2 || nrow(evs) == 0 || + all(purrr::map_lgl(seq_len(nrow(evs)), ~ event_reversal_exact(evs[.x, ]))) + supported && exact_ok + }) |> + dplyr::ungroup() |> + dplyr::transmute( + source_year = as.character(y1), + target_year = as.character(y2)) + }) +} + + +#' Build the Metadata Attribute for a County-Events Crosswalk +#' @noRd +county_events_metadata <- function(geography, + geography_standardized, + source_year, + target_year, + interval_events, + cache_file, + read_from_cache) { + event_notes <- if (nrow(interval_events) > 0) { + stringr::str_c("Event applied: ", interval_events$description) + } else { + "No county boundary or code changes in this interval; identity crosswalk." + } + + list( + data_source = "county_events", + data_source_full_name = "U.S. Census Bureau county boundary change records (curated in the crosswalk package)", + documentation_url = county_events_meta$source_documentation_url, + source_year = as.character(source_year), + target_year = as.character(target_year), + source_geography = geography, + source_geography_standardized = geography_standardized, + target_geography = geography, + target_geography_standardized = geography_standardized, + state_coverage = "National (all 50 states, DC, and Puerto Rico)", + notes = c( + event_notes, + "Allocation factors for split/transferred counties are population-based (see factor derivations in data-raw/build_county_events_sysdata.R).", + "Minor county boundary corrections and non-county-related ACS tract revisions are not modeled."), + retrieved_at = Sys.time(), + cached = !is.null(cache_file), + cache_path = cache_file, + read_from_cache = read_from_cache) +} + + +utils::globalVariables(c( + "effective_vintage_year", "event_id", "decade_base", "geoid", "geography", + "subcounty_handling", "allocation_factor", "step_source", "step_target", + "step_factor", "step_weighting", "y1", "y2")) diff --git a/R/get_crosswalk.R b/R/get_crosswalk.R index 144d1af..737740f 100644 --- a/R/get_crosswalk.R +++ b/R/get_crosswalk.R @@ -5,10 +5,10 @@ #' structure containing one or more crosswalk tibbles. #' #' @details This function sources crosswalks from Geocorr 2022, Geocorr 2018, -#' IPUMS NHGIS, and CT Data Collaborative. Crosswalk weights are from the -#' original sources and have not been modified; this function merely standardizes -#' the format of the returned crosswalks and enables easy programmatic access -#' and caching. +#' IPUMS NHGIS, CT Data Collaborative, and a curated registry of county +#' change events. Crosswalk weights are from the original sources and have +#' not been modified; this function merely standardizes the format of the +#' returned crosswalks and enables easy programmatic access and caching. #' #' **GeoCorr version selection**: For same-year geography crosswalks, the #' appropriate GeoCorr version is selected automatically based on the year: @@ -35,30 +35,75 @@ #' For this case, the function combines CT Data Collaborative crosswalks for #' Connecticut with identity mappings for other states. #' +#' **Arbitrary-year county crosswalks (county change events)**: Counties and +#' county equivalents are occasionally redefined outside decennial censuses +#' (e.g., the 2013 Bedford, VA merge; the 2015 Shannon -> Oglala Lakota, SD +#' FIPS change; the Valdez-Cordova, AK split, first reflected in 2020-vintage +#' products; the 2022 Connecticut planning regions). County -> county +#' crosswalks are therefore available between ANY pair of years from 2000 +#' through the curated horizon (e.g., 2014 -> 2019 or 2014 -> 2023), built +#' from an identity mapping plus the documented change events in the +#' interval, with population-based allocation factors for splits and +#' part transfers. County crosswalks are forward-only (source_year < +#' target_year). For sub-county geographies (tract, block group, block) +#' affected by these county changes, same-decade crosswalks are available +#' from 2010 onward (e.g., tract 2014 -> 2019); year pairs spanning a +#' decennial census are planned automatically as chains through NHGIS +#' crosswalks (e.g., tract 2014 -> 2023). +#' +#' **Nested geographies**: When the source geography nests exactly within +#' the target geography and the years match (e.g., tract -> county, block +#' group -> tract), no crosswalk is needed: every source unit belongs to +#' exactly one target unit. In this case the function warns and returns the +#' standard list structure with an *empty* crosswalk tibble -- aggregate +#' your data directly instead (e.g., sum tract values by the first five +#' GEOID characters to obtain county values). +#' +#' **Caching**: When `cache` is a directory path, each component crosswalk +#' is written there as a CSV, and subsequent calls with the same parameters +#' read the file from disk instead of re-downloading. Cached files never +#' expire; delete a file to force a fresh download. The metadata attribute +#' records `cache_path` and whether the crosswalk was `read_from_cache`. +#' #' Note that an IPUMS NHGIS API key is required to access crosswalks from that #' source. Use `usethis::edit_r_environ(scope = "user")` to save your API key #' to your .Renviron; the name of the key should be "IPUMS_API_KEY". You can #' obtain a key from: https://account.ipums.org/api_keys. #' #' @param source_year Character or numeric. Year of the source geography, one of -#' c(1990, 2000, 2010, 2020). -#' @param source_geography Character. Source geography name. One of c("block", -#' "block group", "tract", "place", "county", "urban_area", "zcta", "puma", -#' "puma12", "puma22", "cd115", "cd116", "cd118", "cd119", "urban_area", -#' "core_based_statistical_area"). +#' c(1990, 2000, 2010, 2020) for decennial years, or c(2011, 2012, 2014, 2015, +#' 2022) for non-census years (limited to block groups, tracts, and counties). +#' For county -> county crosswalks, any year from 2000 onward is supported; +#' for same-decade tract/block group/block crosswalks, any year from 2010 +#' onward (see Details). +#' @param source_geography Character. Source geography name. For same-year +#' (GeoCorr) crosswalks, one of c("block", "block group", "tract", "county", +#' "place", "zcta", "puma", "puma12", "puma22", "cd115", "cd116", "cd118", +#' "cd119"). For cross-decade (NHGIS) crosswalks, sources are limited to +#' c("block", "block group", "tract"). Use `get_available_crosswalks()` for +#' the full listing of supported combinations. #' @param target_year Character or numeric. Year of the target geography, one of #' c(1990, 2000, 2010, 2020) for decennial crosswalks, or c(2011, 2012, 2014, #' 2015, 2022) for non-census year crosswalks (limited to block groups, tracts, -#' and counties). -#' @param target_geography Character. Target geography name. One of c("block", -#' "block group", "tract", "place", "county", "urban_area", "zcta", "puma", -#' "puma12", "puma22", "cd115", "cd116", "cd118", "cd119", "urban_area", -#' "core_based_statistical_area"). -#' @param weight Character. Weighting variable for Geocorr crosswalks. One of -#' c("population", "housing", "land"). -#' @param cache Directory path. Where to download the crosswalk to. If NULL (default), -#' crosswalk is returned but not saved to disk. Individual component crosswalks -#' are cached separately when provided. +#' and counties). For county -> county crosswalks, any year from 2000 onward +#' is supported; for same-decade tract/block group/block crosswalks, any year +#' from 2010 onward (see Details). +#' @param target_geography Character. Target geography name. For same-year +#' (GeoCorr) crosswalks, one of c("block", "block group", "tract", "county", +#' "place", "zcta", "puma", "puma12", "puma22", "cd115", "cd116", "cd118", +#' "cd119"). For cross-decade (NHGIS) crosswalks, targets additionally +#' include c("urban_area", "core_based_statistical_area"). Use +#' `get_available_crosswalks()` for the full listing. +#' @param weight Character. Weighting variable used to construct or select +#' allocation factors. One of c("population", "housing", "land"). Applies to +#' GeoCorr crosswalks and to the selection of the NHGIS interpolation weight +#' (block-based NHGIS crosswalks provide a single combined weight, which is +#' used regardless of this argument). +#' @param cache Directory path. Where to save the crosswalk. If NULL (default), +#' the crosswalk is returned but not saved to disk. When provided, individual +#' component crosswalks are cached separately, and subsequent calls with the +#' same parameters read from the cached files instead of re-downloading +#' (see Details). #' @param silent Logical. If `TRUE`, suppresses all informational messages and #' warnings. Defaults to `getOption("crosswalk.silent", FALSE)`. Set #' `options(crosswalk.silent = TRUE)` to silence all calls by default. @@ -71,9 +116,13 @@ #' \item{message}{A formatted message describing the crosswalk chain} #' } #' +#' For nested same-year requests (e.g., tract -> county), `crosswalks$step_1` +#' is an empty tibble and a warning explains that no crosswalk is needed +#' (see Details). +#' #' Each crosswalk tibble includes an attribute `crosswalk_metadata` (access via -#' `attr(result$crosswalks$step_1, "crosswalk_metadata")`) containing comprehensive -#' information about how the crosswalk was produced. +#' `attr(result$crosswalks$step_1, "crosswalk_metadata")`) documenting how the +#' crosswalk was produced; see the "Crosswalk metadata" section. #' #' Columns in returned crosswalk dataframes (some may not be present depending on source): #' \describe{ @@ -94,6 +143,39 @@ #' \item{state_fips}{Two-digit state FIPS code, if applicable} #' } #' +#' @section Crosswalk metadata: +#' The `crosswalk_metadata` attribute attached to each crosswalk tibble is a +#' list recording the crosswalk's provenance. Fields with no value for a given +#' source are NULL. Elements include: +#' \describe{ +#' \item{call_parameters}{The arguments the crosswalk was requested with +#' (`source_geography`, `target_geography`, `source_year`, `target_year`, +#' `weight`, `cache`).} +#' \item{data_source, data_source_full_name}{Short code and full name of the +#' source that produced the crosswalk (e.g., "geocorr", "Geocorr 2022 +#' (Missouri Census Data Center)").} +#' \item{download_url, api_endpoint, documentation_url, citation_url, +#' github_repository}{Links to the underlying data, its documentation, and +#' citation guidance, where applicable.} +#' \item{source_geography, target_geography, source_year, target_year}{The +#' requested geographies and years, plus `source_geography_standardized` / +#' `target_geography_standardized` (the source-specific codes they resolved +#' to) and `reference_year` (the source's reference vintage, e.g., the +#' GeoCorr version).} +#' \item{weighting_variable}{The weighting variable used to construct or +#' select allocation factors.} +#' \item{state_coverage, notes}{Geographic coverage notes and source-specific +#' caveats.} +#' \item{retrieved_at, cached, cache_path, read_from_cache}{When the +#' crosswalk was retrieved, and whether it was written to or read from a +#' local cache.} +#' \item{is_multi_step, crosswalk_package_version}{Whether the crosswalk is +#' part of a multi-step chain, and the package version that produced it.} +#' } +#' +#' @seealso [crosswalk_data()] to apply the returned crosswalk(s) to data; +#' [get_available_crosswalks()] for all supported geography/year +#' combinations; [list_nhgis_crosswalks()] for the NHGIS-specific listing. #' @export #' @examples #' \dontrun{ @@ -132,6 +214,14 @@ #' crosswalk = result, #' count_columns = "count_population", #' return_intermediate = TRUE) +#' +#' # Arbitrary-year county crosswalk handling non-decennial county changes +#' # (e.g., 2015 Shannon -> Oglala Lakota, SD; Valdez-Cordova, AK split) +#' result <- get_crosswalk( +#' source_geography = "county", +#' target_geography = "county", +#' source_year = 2014, +#' target_year = 2023) #' } get_crosswalk <- function( @@ -151,10 +241,14 @@ get_crosswalk <- function( years_match <- (is.null(source_year) && is.null(target_year)) || (!is.null(source_year) && !is.null(target_year) && isTRUE(source_year == target_year)) - is_nested <- (source_geography == "block" && target_geography %in% c("block group", "tract", "county", "core_based_statistical_area")) || - (source_geography == "block group" && target_geography %in% c("tract", "county", "core_based_statistical_area")) || - (source_geography == "tract" && target_geography %in% c("county", "core_based_statistical_area")) || - (source_geography == "county" && target_geography == "core_based_statistical_area") + # Standardize names so aliases (e.g. "blockgroup", "bg") are detected too + source_geog_nested <- standardize_geography_for_chain(source_geography) + target_geog_nested <- standardize_geography_for_chain(target_geography) + + is_nested <- (source_geog_nested == "block" && target_geog_nested %in% c("block_group", "tract", "county", "cbsa")) || + (source_geog_nested == "block_group" && target_geog_nested %in% c("tract", "county", "cbsa")) || + (source_geog_nested == "tract" && target_geog_nested %in% c("county", "cbsa")) || + (source_geog_nested == "county" && target_geog_nested == "cbsa") if (is_nested && years_match) { cw_warning( @@ -199,9 +293,14 @@ simply aggregate your data to the desired geography.", call. = FALSE) #' Get a Single-Step Crosswalk (Internal) #' #' Internal function that retrieves a single crosswalk from the appropriate source. -#' This handles routing to Geocorr, NHGIS, or CTData based on the parameters. +#' This handles routing to Geocorr, NHGIS, CTData, or the county-events engine +#' based on the parameters. #' #' @inheritParams get_crosswalk +#' @param crosswalk_source Character or NULL. The crosswalk source determined +#' by the chain planner ("geocorr", "nhgis", "ctdata_2020_2022", or +#' "county_events"). When NULL, the source is derived from the other +#' parameters. #' @return A tibble containing the crosswalk. #' @keywords internal #' @noRd @@ -211,29 +310,47 @@ get_crosswalk_single <- function( source_year = NULL, target_year = NULL, weight = "population", - cache = NULL) { + cache = NULL, + crosswalk_source = NULL) { # Convert years to character for consistent processing source_year_chr <- if (!is.null(source_year)) as.character(source_year) else NULL target_year_chr <- if (!is.null(target_year)) as.character(target_year) else NULL - # Determine which source to use + # Year context for GeoCorr version selection: prefer the target year, but + # fall back to the source year when only the source year is provided + geocorr_year_context <- if (!is.null(target_year)) target_year else source_year - # Use Geocorr for: no years specified, or same year - use_geocorr <- is.null(source_year) || is.null(target_year) || - (!is.null(source_year) && !is.null(target_year) && isTRUE(source_year == target_year)) + # Determine which source to use when the planner did not specify one + if (is.null(crosswalk_source)) { - # Use CTData for 2020 <-> 2022 (Connecticut planning region changes) - use_ctdata <- !is.null(source_year_chr) && !is.null(target_year_chr) && - ((source_year_chr == "2020" && target_year_chr == "2022") || - (source_year_chr == "2022" && target_year_chr == "2020")) + # Use Geocorr for: no years specified, or same year + use_geocorr <- is.null(source_year) || is.null(target_year) || + (!is.null(source_year) && !is.null(target_year) && isTRUE(source_year == target_year)) - if (use_geocorr) { - crosswalk_source <- "geocorr" - } else if (use_ctdata) { - crosswalk_source <- "ctdata_2020_2022" - } else { - crosswalk_source <- "nhgis" + # Use CTData for 2020 <-> 2022 (Connecticut planning region changes) + use_ctdata <- !is.null(source_year_chr) && !is.null(target_year_chr) && + ((source_year_chr == "2020" && target_year_chr == "2022") || + (source_year_chr == "2022" && target_year_chr == "2020")) + + if (use_geocorr) { + crosswalk_source <- "geocorr" + } else if (use_ctdata) { + crosswalk_source <- "ctdata_2020_2022" + } else { + # Same-geography temporal requests that NHGIS cannot serve fall through + # to the county-events engine when it covers the geography and years + events_geog <- standardize_geography_for_events(source_geography) + use_county_events <- !is.na(events_geog) && + isTRUE(events_geog == standardize_geography_for_events(target_geography)) && + !is_nhgis_crosswalk_available( + source_geography, target_geography, source_year, target_year) && + min(as.numeric(source_year), as.numeric(target_year)) >= 2000 && + max(as.numeric(source_year), as.numeric(target_year)) <= + county_events_meta$known_through_year + + crosswalk_source <- if (use_county_events) "county_events" else "nhgis" + } } # Fetch the crosswalk from the appropriate source @@ -244,6 +361,20 @@ get_crosswalk_single <- function( target_year = target_year, cache = cache) + } else if (crosswalk_source == "county_events") { + result <- get_county_events_crosswalk( + geography = source_geography, + source_year = source_year, + target_year = target_year, + cache = cache) + + if (weight != "population") { + cw_message(stringr::str_c( + "County-events crosswalks use population-derived allocation factors ", + "for changed counties; the requested '", weight, + "' weight does not apply to this step.")) + } + } else if (crosswalk_source == "nhgis") { result <- get_nhgis_crosswalk( source_year = source_year, @@ -252,8 +383,53 @@ get_crosswalk_single <- function( target_geography = target_geography, cache = cache) + # NHGIS crosswalks are long format, with one row per source-target pair + # per interpolation weight. Keep only the weight matching the `weight` + # argument so each pair appears exactly once; otherwise downstream + # interpolation would multiply-count values. Candidates are ordered by + # preference: block-based crosswalks provide a single combined + # housing/population weight ("housing_population") rather than separate + # population and housing weights. + if (nrow(result) > 0 && "weighting_factor" %in% names(result)) { + nhgis_weight_map <- list( + "population" = c("population", "housing_population"), + "housing" = c("housing_all", "housing_population"), + "land" = c("landarea")) + + if (!weight %in% names(nhgis_weight_map)) { + stop( + "Invalid weight: '", weight, "'. Must be one of: ", + paste(names(nhgis_weight_map), collapse = ", "), ".") + } + + available_weights <- unique(result$weighting_factor) + candidate_weights <- intersect(nhgis_weight_map[[weight]], available_weights) + + if (length(candidate_weights) > 0) { + selected_weight <- candidate_weights[1] + if (selected_weight != nhgis_weight_map[[weight]][1]) { + cw_message(stringr::str_c( + "This NHGIS crosswalk does not provide a separate '", weight, + "' weight; using the combined '", selected_weight, + "' interpolation weight instead.")) + } + result <- result |> + dplyr::filter(weighting_factor == selected_weight) + } else if (length(available_weights) == 1) { + cw_message(stringr::str_c( + "This NHGIS crosswalk provides a single interpolation weight ('", + available_weights, + "'), which is used regardless of the requested weight.")) + } else { + stop( + "The requested weight '", weight, "' is not available for this ", + "NHGIS crosswalk. Available weighting factors: ", + paste(available_weights, collapse = ", "), ".") + } + } + } else { - geocorr_version <- determine_geocorr_version(target_year) + geocorr_version <- determine_geocorr_version(geocorr_year_context) result <- get_geocorr_crosswalk( source_geography = source_geography, target_geography = target_geography, @@ -286,11 +462,12 @@ get_crosswalk_single <- function( switch(crosswalk_source, "nhgis" = "IPUMS NHGIS (National Historical Geographic Information System)", "geocorr" = { - geocorr_ver <- determine_geocorr_version(target_year) + geocorr_ver <- determine_geocorr_version(geocorr_year_context) stringr::str_c("Geocorr ", get_geocorr_config(geocorr_ver)$reference_year, " (Missouri Census Data Center)") }, "ctdata_2020_2022" = "CT Data Collaborative", + "county_events" = "U.S. Census Bureau county boundary change records (curated in the crosswalk package)", crosswalk_source) }, @@ -333,14 +510,21 @@ get_crosswalk_single <- function( attr(result, "crosswalk_metadata") <- metadata + # Standardize types: GEOIDs, names, and years as character; allocation + # factors and overlap measures as numeric + crosswalk_numeric_columns <- c( + "allocation_factor_source_to_target", "allocation_factor_target_to_source", + "population_2020", "housing_2020", "population_2010", "housing_2010", + "land_area_sqmi") + result <- result |> dplyr::mutate( dplyr::across( - .cols = -allocation_factor_source_to_target, + .cols = -dplyr::any_of(crosswalk_numeric_columns), .fns = as.character), dplyr::across( - .cols = allocation_factor_source_to_target, - as.numeric)) + .cols = dplyr::any_of(crosswalk_numeric_columns), + .fns = as.numeric)) return(result) } @@ -428,11 +612,57 @@ and counties. The provided geography '", geography, "' is not supported.")} #' List All Available Crosswalk Combinations #' #' Returns a tibble of all source/target geography and year combinations -#' supported by `get_crosswalk()`. +#' supported by `get_crosswalk()`, along with the crosswalk source that serves +#' each combination. +#' +#' @details +#' **How to read the year columns.** Years identify the geography *vintages* a +#' crosswalk translates between, not the only request years `get_crosswalk()` +#' accepts: +#' - **GeoCorr** (same-year geography changes) rows are listed under their +#' reference years: 2022 for 2020-Census geography and 2018 for 2010-Census +#' geography. Requests with other year contexts resolve to the matching +#' version -- years 2020 and later (or no years at all) use GeoCorr 2022, +#' and years 2010-2019 use GeoCorr 2018. For example, a request for 2021 +#' tracts -> 2021 ZCTAs is served by the tract -> zcta 2022 row. +#' - **County-events** rows enumerate every supported year pair explicitly +#' (county -> county: any forward pair from 2000 on; tract, block group, and +#' block: same-decade pairs from 2010 on, including exactly reversible +#' relabels). +#' +#' **Nested geographies.** Same-year pairs in which the source geography nests +#' within the target (e.g., tract -> county) are listed because GeoCorr serves +#' them, but `get_crosswalk()` returns an empty crosswalk with a warning for +#' these: no crosswalk is needed -- aggregate your data directly instead (e.g., +#' tracts nest within counties, so tract values can be summed by the first five +#' GEOID characters). +#' +#' **API keys by source.** The `crosswalk_source` column indicates which source +#' `get_crosswalk()` uses for a direct request of that combination (multi-step +#' chains may combine several): +#' - `geocorr`: no API key required +#' - `nhgis`: requires `IPUMS_API_KEY` (see [get_crosswalk()]) +#' - `ctdata_2020_2022`: county requests require a Census API key +#' (`CENSUS_API_KEY`) and the `tidycensus` package; sub-county requests +#' require neither +#' - `county_events`: no API key; the data ships with the package and works +#' offline #' #' @return A tibble with columns: `source_geography`, `target_geography`, -#' `source_year`, `target_year`. +#' `source_year`, `target_year`, `crosswalk_source`. +#' @seealso [get_crosswalk()] to fetch a crosswalk; [list_nhgis_crosswalks()] +#' for the underlying NHGIS listing. #' @export +#' @examples +#' available <- get_available_crosswalks() +#' +#' # All combinations that translate data into 2020 tracts +#' available |> +#' dplyr::filter(target_geography == "tract", target_year == 2020) +#' +#' # County -> county crosswalks support arbitrary year pairs from 2000 on +#' available |> +#' dplyr::filter(source_geography == "county", target_geography == "county") get_available_crosswalks <- function() { # 1. NHGIS: reuse list_nhgis_crosswalks(), select and coerce years to integer @@ -440,7 +670,8 @@ get_available_crosswalks <- function() { dplyr::select(source_geography, target_geography, source_year, target_year) |> dplyr::mutate( source_year = as.integer(source_year), - target_year = as.integer(target_year)) + target_year = as.integer(target_year), + crosswalk_source = "nhgis") # 2. Geocorr 2022: all pairwise combinations of 9 canonical geographies geocorr_2022_geographies <- c( @@ -453,7 +684,8 @@ get_available_crosswalks <- function() { dplyr::filter(source_geography != target_geography) |> dplyr::mutate( source_year = 2022L, - target_year = 2022L) + target_year = 2022L, + crosswalk_source = "geocorr") # 3. Geocorr 2018: all pairwise combinations of 9 canonical geographies geocorr_2018_geographies <- c( @@ -466,7 +698,8 @@ get_available_crosswalks <- function() { dplyr::filter(source_geography != target_geography) |> dplyr::mutate( source_year = 2018L, - target_year = 2018L) + target_year = 2018L, + crosswalk_source = "geocorr") # 4. CTData: 7 manually specified combinations (2020<->2022) ctdata <- tibble::tibble( @@ -475,14 +708,40 @@ get_available_crosswalks <- function() { target_geography = c("block", "block_group", "tract", "county", "block", "block_group", "tract"), source_year = c(rep(2020L, 4), rep(2022L, 3)), - target_year = c(rep(2022L, 4), rep(2020L, 3))) - - # 5. Combine, deduplicate, and sort - dplyr::bind_rows(nhgis, geocorr_2022, geocorr_2018, ctdata) |> + target_year = c(rep(2022L, 4), rep(2020L, 3)), + crosswalk_source = "ctdata_2020_2022") + + # 5. County events: same-geography temporal combinations served by the + # curated county-change engine (shares list_county_events_edges() with the + # chain planner so the registry cannot drift from routing) + county_events_combos <- purrr::map_dfr( + c("county", "tract", "block_group", "block"), + \(geog) list_county_events_edges(geog) |> + dplyr::mutate( + source_geography = geog, + target_geography = geog, + source_year = as.integer(source_year), + target_year = as.integer(target_year), + crosswalk_source = "county_events") |> + dplyr::select(source_geography, target_geography, source_year, + target_year, crosswalk_source)) + + # 6. Combine, deduplicate, and sort. The 2020 <-> 2022 pairs can be served by + # both CTData and the county-events engine; keep the source the planner + # routes direct requests to (CTData), mirroring get_crosswalk()'s behavior + source_preference <- c( + "geocorr" = 1L, "nhgis" = 2L, "ctdata_2020_2022" = 3L, "county_events" = 4L) + + dplyr::bind_rows(nhgis, geocorr_2022, geocorr_2018, ctdata, county_events_combos) |> dplyr::distinct() |> + dplyr::slice_min( + source_preference[crosswalk_source], + by = c(source_geography, target_geography, source_year, target_year), + with_ties = FALSE) |> dplyr::arrange(source_geography, target_geography, source_year, target_year) } utils::globalVariables(c( "allocation_factor_source_to_target", "geoid", "label", - "n_unmatched", "pct_of_unmatched", "state_abbr")) \ No newline at end of file + "n_unmatched", "pct_of_unmatched", "state_abbr", + "source_geography", "target_geography", "crosswalk_source")) \ No newline at end of file diff --git a/R/get_crosswalk_chain.R b/R/get_crosswalk_chain.R index a59fca0..4b4a0d0 100644 --- a/R/get_crosswalk_chain.R +++ b/R/get_crosswalk_chain.R @@ -69,7 +69,8 @@ get_crosswalk_chain <- function( source_year = if (!is.na(step$source_year)) as.numeric(step$source_year) else NULL, target_year = if (!is.na(step$target_year)) as.numeric(step$target_year) else NULL, weight = weight, - cache = cache) + cache = cache, + crosswalk_source = step$crosswalk_source) result$crosswalks[[step_name]] <- crosswalk_i } diff --git a/R/get_ctdata_crosswalk.R b/R/get_ctdata_crosswalk.R index b4ef83c..8774877 100644 --- a/R/get_ctdata_crosswalk.R +++ b/R/get_ctdata_crosswalk.R @@ -38,8 +38,11 @@ #' \item{target_geography_name}{The geography type} #' \item{source_year}{The source year (2020 or 2022)} #' \item{target_year}{The target year (2020 or 2022)} -#' \item{allocation_factor_source_to_target}{1 for all records (identity or CT FIPS change)} -#' \item{weighting_factor}{"identity" for non-CT, varies for CT county} +#' \item{allocation_factor_source_to_target}{1 for identity records (all +#' geographies outside Connecticut and sub-county geographies within it); +#' population-weighted shares for Connecticut county records} +#' \item{weighting_factor}{"identity" for identity records, "population" +#' for Connecticut county records} #' \item{state_fips}{Two-digit state FIPS code} #' } #' @keywords internal @@ -90,6 +93,21 @@ Connecticut's county boundaries changed (9 planning regions -> 8 counties), requiring population-weighted disaggregation which is not implemented. Only block, block_group, and tract geographies support the 2022 -> 2020 direction.")} + # County crosswalks require tidycensus (in Suggests) for county GEOIDs and + # tract populations + if (geography_standardized == "county" && !requireNamespace("tidycensus", quietly = TRUE)) { + stop( +"The 'tidycensus' package is required for county 2020 <-> 2022 crosswalks. +Install it with install.packages('tidycensus').")} + + # Weighting note for metadata: county crosswalks use population weights for + # Connecticut; all other geographies/states are identity mappings + weighting_note <- if (geography_standardized == "county") { + "Connecticut county records use population-weighted allocation factors; all other records are identity mappings with allocation_factor = 1." + } else { + "All records have allocation_factor = 1 (identity mapping or CT FIPS code change)." + } + if (is.null(cache)) { cache_path <- tempdir() } else { @@ -120,9 +138,6 @@ Only block, block_group, and tract geographies support the 2022 -> 2020 directio allocation_factor_source_to_target = readr::col_double()), show_col_types = FALSE) - # Weighting note for metadata - weighting_note <- "All records have allocation_factor = 1 (identity mapping or CT FIPS code change)." - attr(result, "crosswalk_metadata") <- list( data_source = "ctdata_nhgis_combined", data_source_full_name = "CT Data Collaborative (CT) + NHGIS-derived identity mapping (other states)", @@ -372,8 +387,6 @@ Only block, block_group, and tract geographies support the 2022 -> 2020 directio - Other states: Identity mapping derived from NHGIS 2010-2020 crosswalk")) # Attach metadata to result - weighting_note <- "All records have allocation_factor = 1 (identity mapping or CT FIPS code change)." - attr(result, "crosswalk_metadata") <- list( data_source = "ctdata_nhgis_combined", data_source_full_name = "CT Data Collaborative (CT) + NHGIS-derived identity mapping (other states)", diff --git a/R/get_geocorr_crosswalk.R b/R/get_geocorr_crosswalk.R index 4db677c..484730c 100644 --- a/R/get_geocorr_crosswalk.R +++ b/R/get_geocorr_crosswalk.R @@ -135,7 +135,8 @@ resolve_geocorr_geography <- function(geography, geocorr_version = "2022") { #' Census geography. #' #' @return A dataframe representing the requested GeoCorr crosswalk for all 51 -#' states and Puerto Rico. +#' states and Puerto Rico (GeoCorr 2018 excludes Puerto Rico). +#' @importFrom rlang := #' @keywords internal #' @noRd get_geocorr_crosswalk <- function( @@ -160,9 +161,20 @@ get_geocorr_crosswalk <- function( config$reference_year, "_", source_geography, "_to_", target_geography, "_weightedby_", weight, ".csv")) } + geocorr_numeric_columns <- c( + "allocation_factor_source_to_target", "allocation_factor_target_to_source", + "population_2020", "housing_2020", "population_2010", "housing_2010", + "land_area_sqmi") + ## if the file exists and the user does not wish to overwrite it if (file.exists(outpath) & !is.null(cache)) { - result = readr::read_csv(outpath, show_col_types = FALSE) + ## read GEOIDs as character (type-guessing can strip leading zeros) + result = readr::read_csv( + outpath, + col_types = readr::cols(.default = readr::col_character()), + show_col_types = FALSE) |> + dplyr::mutate( + dplyr::across(dplyr::any_of(geocorr_numeric_columns), as.numeric)) cw_message("Reading file from cache.") @@ -304,15 +316,17 @@ get_geocorr_crosswalk <- function( if (is.na(csv_path)) { stop("Unable to acquire the specified crosswalk; please file an issue.") } readr::read_csv(file.path("https://mcdc.missouri.edu", "temp", csv_path), show_col_types = FALSE) |> - janitor::clean_names() + janitor::clean_names() |> + ## the first row of every GeoCorr CSV is a human-readable label row; + ## drop it here so chunked (multi-request) queries don't retain the + ## label rows of chunks after the first + dplyr::slice(-1) } ## for block-level crosswalks, the maximum number of states per query is 13 if ("block" %in% c(source_api_code, target_api_code)) { - n = length(states_data) / 13 - groups = cut(seq_along(states_data), n, labels = FALSE) - states_chunked = split(states_data, groups) + states_chunked = split(states_data, ceiling(seq_along(states_data) / 13)) df1 = purrr::map_dfr( states_chunked, @@ -324,7 +338,6 @@ get_geocorr_crosswalk <- function( weight_rename_pattern <- paste(names(config$weight_rename), collapse = "|") df2 = df1 |> - dplyr::slice(2:nrow(df1)) |> ## naming conventions for some geographies are inconsistent; we standardize dplyr::rename_with( .cols = dplyr::matches("zip_name"), @@ -480,7 +493,7 @@ get_geocorr_crosswalk <- function( target_api_code == "tract" & nchar(target_geoid) == 13 ~ stringr::str_sub(target_geoid, 3, 13), TRUE ~ target_geoid), weighting_factor = weight, - dplyr::across(.cols = dplyr::matches("allocation"), .fns = as.numeric)) + dplyr::across(.cols = dplyr::any_of(geocorr_numeric_columns), .fns = as.numeric)) if (!is.null(cache)) { ## if the file does not already exist and cache is TRUE @@ -509,4 +522,4 @@ get_geocorr_crosswalk <- function( return(df2) } -utils::globalVariables(c("afact", "afact2", "county", "stab", "state")) +utils::globalVariables(c("afact", "afact2", "block", "county", "stab", "state", "tract")) diff --git a/R/get_nhgis_crosswalk.R b/R/get_nhgis_crosswalk.R index 979e660..013a93b 100644 --- a/R/get_nhgis_crosswalk.R +++ b/R/get_nhgis_crosswalk.R @@ -111,18 +111,27 @@ available crosswalks.") #' List supported NHGIS crosswalks #' -#' Returns a tibble of all available NHGIS geographic crosswalks with their -#' corresponding parameters that can be used with get_nhgis_crosswalk(). +#' Returns a tibble of all IPUMS NHGIS crosswalks available through +#' `get_crosswalk()`. NHGIS provides the package's inter-temporal +#' (cross-decade) crosswalks; fetching them requires an `IPUMS_API_KEY` +#' (see [get_crosswalk()]). This listing itself is static and works offline. #' -#' @return A tibble with columns: -#' \itemize{ -#' \item source_year: Year of the source geography -#' \item source_geography: Source geography name -#' \item target_year: Year of the target geography -#' \item target_geography: Target geography name +#' @return A tibble with one row per NHGIS crosswalk and columns: +#' \describe{ +#' \item{source_geography}{Source geography name (e.g., "block", "tract")} +#' \item{source_year}{Year of the source geography (character)} +#' \item{target_geography}{Target geography name} +#' \item{target_year}{Year of the target geography (character)} +#' \item{crosswalk_path}{URL of the underlying NHGIS crosswalk file} #' } #' +#' @seealso [get_available_crosswalks()] for supported combinations across all +#' sources; [get_crosswalk()] to fetch a crosswalk. #' @export +#' @examples +#' # All NHGIS crosswalks from 2010 tracts +#' list_nhgis_crosswalks() |> +#' dplyr::filter(source_geography == "tract", source_year == "2010") list_nhgis_crosswalks <- function() { nhgis_crosswalks_vector = c( ## ========================================================================= @@ -506,8 +515,9 @@ standardize_geography_for_nhgis_check <- function(geography) { #' @param cache Directory path. Where to download the crosswalk to. If NULL (default), #' crosswalk is returned but not saved to disk. #' -#' @return A dataframe representing the requested Geocorr22 crosswalk for all -#' 51 states and Puerto Rico. Depending on the desired geographies, some +#' @return A dataframe representing the requested NHGIS crosswalk in long +#' format, with one row per source-target GEOID pair per interpolation +#' weight (`weighting_factor`). Depending on the desired geographies, some #' fields may not be included. #' \describe{ #' \item{source_geoid}{A unique identifier for the source geography} @@ -552,7 +562,24 @@ get_nhgis_crosswalk <- function( ## if the file exists and cache == TRUE if (file.exists(csv_path) & !is.null(cache)) { - result = readr::read_csv(csv_path, show_col_types = FALSE) + ## read GEOIDs as character (type-guessing can strip leading zeros) and + ## normalize values written by earlier package versions + result = readr::read_csv( + csv_path, + col_types = readr::cols(.default = readr::col_character()), + show_col_types = FALSE) |> + dplyr::mutate( + allocation_factor_source_to_target = as.numeric(allocation_factor_source_to_target), + weighting_factor = stringr::str_remove(weighting_factor, "^weight_")) + + if (source_year == "1990" && source_geography_standardized == "tr") { + result = result |> + dplyr::mutate( + source_geoid = dplyr::if_else( + nchar(source_geoid) == 9, + stringr::str_c(source_geoid, "00"), + source_geoid)) + } cw_message( "Use of NHGIS crosswalks is subject to the same conditions as for all NHGIS data. @@ -586,8 +613,6 @@ See https://www.nhgis.org/citation-and-use-nhgis-data.") valid_decennial_years <- c("1990", "2000", "2010", "2020") valid_noncensus_years <- c("2011", "2012", "2014", "2015", "2022") valid_years <- c(valid_decennial_years, valid_noncensus_years) - valid_source_geogs <- c("blk", "bg", "tr") - valid_target_geogs <- c("blk", "bg", "tr", "co", "ua", "zcta", "puma", "cbsa") noncensus_geogs <- c("bg", "tr", "co") # Helper to determine decade for a year @@ -666,21 +691,13 @@ County targets are only available from source years: 1990, 2000, 2010, 2014, years 2010, 2014, and 2015 (not ", target_year, ").") } - if (is.null(source_geography_standardized)) { - stop( -"source_geography '", source_geography, "' is not valid. Must be one of: blocks, -block group parts, or tracts (various spellings accepted).")} - - if (is.null(target_geography_standardized)) { - stop( -"target_geography '", target_geography, "' is not valid. Must be one of: blocks, -block groups, tracts, or counties (various spellings accepted)")} - if (!(crosswalk_path %in% list_nhgis_crosswalks()$crosswalk_path)) { stop(stringr::str_c( "There is no available crosswalk between the specified geographies and years.")) } - api_key = Sys.getenv("IPUMS_API_KEY") + if (is.null(api_key) || api_key == "") { + api_key = Sys.getenv("IPUMS_API_KEY") + } if (api_key == "") { stop( "API key required. Save your API key to the IPUMS_API_KEY environment @@ -705,20 +722,27 @@ variable. Get your key at https://account.ipums.org/api_keys") } ) } - crosswalk_df1 = tryCatch({ + # Use a unique temporary directory for downloading and extracting + temp_dir = file.path(tempdir(), stringr::str_c("nhgis_", crosswalk_sub_path, "_", format(Sys.time(), "%Y%m%d%H%M%S"))) + dir.create(temp_dir, recursive = TRUE) + on.exit(unlink(temp_dir, recursive = TRUE), add = TRUE) - # Use a unique temporary directory for downloading and extracting - temp_dir = file.path(tempdir(), stringr::str_c("nhgis_", crosswalk_sub_path, "_", format(Sys.time(), "%Y%m%d%H%M%S"))) - dir.create(temp_dir, recursive = TRUE) - on.exit(unlink(temp_dir, recursive = TRUE), add = TRUE) + zip_path = file.path(temp_dir, stringr::str_c(crosswalk_sub_path, ".zip")) - zip_path = file.path(temp_dir, stringr::str_c(crosswalk_sub_path, ".zip")) + # Download the crosswalk file + response = httr::GET( + crosswalk_path, + httr::add_headers(Authorization = api_key), + httr::write_disk(zip_path, overwrite = TRUE)) - # Download the crosswalk file - response = httr::GET( - crosswalk_path, - httr::add_headers(Authorization = api_key), - httr::write_disk(zip_path, overwrite = TRUE), overwrite = TRUE) + if (httr::http_error(response)) { + stop( + "Failed to download NHGIS crosswalk ", crosswalk_sub_path, " (HTTP ", + httr::status_code(response), "). If the status is 401 or 403, check that ", + "your IPUMS API key is valid; see https://account.ipums.org/api_keys.") + } + + crosswalk_df1 = tryCatch({ # Check what's in the zip before extracting zip_contents = safe_unzip_list(zip_path) @@ -849,7 +873,21 @@ variable. Get your key at https://account.ipums.org/api_keys") } tidyr::pivot_longer( cols = dplyr::matches("weight_"), names_to = "weighting_factor", - values_to = "allocation_factor_source_to_target") + values_to = "allocation_factor_source_to_target") |> + dplyr::mutate( + weighting_factor = stringr::str_remove(weighting_factor, "^weight_")) + + ## 1990 tract GEOIDs from NHGIS omit the implied "00" suffix for tracts + ## without a suffix, yielding 9-character GEOIDs; right-pad to the + ## standard 11 characters + if (source_year == "1990" && source_geography_standardized == "tr") { + crosswalk_df = crosswalk_df |> + dplyr::mutate( + source_geoid = dplyr::if_else( + nchar(source_geoid) == 9, + stringr::str_c(source_geoid, "00"), + source_geoid)) + } ## if the file does not already exist and cache is not NULL if (!file.exists(csv_path) & !is.null(cache)) { diff --git a/R/plan_crosswalk_chain.R b/R/plan_crosswalk_chain.R index 5c4535a..0fd876c 100644 --- a/R/plan_crosswalk_chain.R +++ b/R/plan_crosswalk_chain.R @@ -103,7 +103,16 @@ plan_crosswalk_chain <- function( return(result) } - # Find temporal path via NHGIS same-geography crosswalks + # Targeted validation for county temporal requests (served by the + # county-events engine, which is forward-only and bounded) + county_events_error <- validate_county_events_years( + source_geog_std, source_year_chr, target_year_chr) + if (!is.null(county_events_error)) { + result$error <- county_events_error + return(result) + } + + # Find temporal path via NHGIS and county-events same-geography crosswalks temporal_path <- find_temporal_path( source_geog_std, source_year_chr, target_year_chr, available_crosswalks) @@ -112,7 +121,8 @@ plan_crosswalk_chain <- function( "No temporal crosswalk path found from ", source_year_chr, " to ", target_year_chr, " for geography '", source_geography, "'. NHGIS may not provide same-geography crosswalks for this ", - "geography or year combination.") + "geography or year combination.", + temporal_path_hint(source_geog_std)) return(result) } @@ -126,10 +136,11 @@ plan_crosswalk_chain <- function( source_year = source_year_chr, target_geography = target_geography, target_year = target_year_chr, - crosswalk_source = "nhgis", + crosswalk_source = temporal_path[[1]]$crosswalk_source, description = stringr::str_c( source_year_chr, " ", source_geog_std, " -> ", - target_year_chr, " ", target_geog_std, " (inter-temporal)")) + target_year_chr, " ", target_geog_std, + " (inter-temporal via ", temporal_path[[1]]$crosswalk_source, ")")) result$composition_note <- "Single crosswalk; use allocation_factor_source_to_target directly." } else { # Multi-hop temporal chain @@ -144,11 +155,11 @@ plan_crosswalk_chain <- function( source_year = purrr::map_chr(temporal_path, ~ .x$source_year), target_geography = rep(target_geography[1], n_hops), target_year = purrr::map_chr(temporal_path, ~ .x$target_year), - crosswalk_source = rep("nhgis", n_hops), + crosswalk_source = purrr::map_chr(temporal_path, ~ .x$crosswalk_source), description = purrr::map_chr(seq_len(n_hops), ~ stringr::str_c( temporal_path[[.x]]$source_year, " ", source_geog_std, " -> ", temporal_path[[.x]]$target_year, " ", source_geog_std, - " (inter-temporal via nhgis)"))) + " (inter-temporal via ", temporal_path[[.x]]$crosswalk_source, ")"))) result$composition_note <- stringr::str_c( "Compose crosswalks by joining on intermediate geography (", @@ -164,10 +175,12 @@ plan_crosswalk_chain <- function( # Case 3: Different geography, same year (or no years) - single Geocorr crosswalk if (geography_changes && !year_changes) { - # Determine which GeoCorr version will be used based on year context - geocorr_ref <- if (!is.na(target_year_chr)) { - target_year_num <- as.numeric(target_year_chr) - if (target_year_num >= 2020) "Geocorr 2022" else if (target_year_num >= 2010) "Geocorr 2018" else "Geocorr" + # Determine which GeoCorr version will be used based on year context: + # prefer the target year, fall back to the source year if only it is given + year_context_chr <- if (!is.na(target_year_chr)) target_year_chr else source_year_chr + geocorr_ref <- if (!is.na(year_context_chr)) { + year_context_num <- as.numeric(year_context_chr) + if (year_context_num >= 2020) "Geocorr 2022" else if (year_context_num >= 2010) "Geocorr 2018" else "Geocorr" } else { "Geocorr 2022" } @@ -236,6 +249,13 @@ plan_crosswalk_chain <- function( src_std <- source_geog_std[1] tgt_std <- target_geog_std[1] + # Targeted validation for county temporal requests + county_events_error <- validate_county_events_years(src_std, src_year, tgt_year) + if (!is.null(county_events_error)) { + result$error <- county_events_error + return(result) + } + # Find temporal path at source geography from source_year to target_year temporal_path <- find_temporal_path( src_std, src_year, tgt_year, available_crosswalks) @@ -245,7 +265,8 @@ plan_crosswalk_chain <- function( "Multi-step crosswalk not possible: no temporal crosswalk path found from ", src_year, " to ", tgt_year, " for geography '", source_geography, "'. NHGIS may not provide same-geography crosswalks for this ", - "geography or year combination.") + "geography or year combination.", + temporal_path_hint(src_std)) return(result) } @@ -268,11 +289,11 @@ plan_crosswalk_chain <- function( source_year = purrr::map_chr(temporal_path, ~ .x$source_year), target_geography = rep(src_geog, n_temporal), target_year = purrr::map_chr(temporal_path, ~ .x$target_year), - crosswalk_source = rep("nhgis", n_temporal), + crosswalk_source = purrr::map_chr(temporal_path, ~ .x$crosswalk_source), description = purrr::map_chr(seq_len(n_temporal), ~ stringr::str_c( temporal_path[[.x]]$source_year, " ", src_std, " -> ", temporal_path[[.x]]$target_year, " ", src_std, - " (inter-temporal via nhgis)"))) + " (inter-temporal via ", temporal_path[[.x]]$crosswalk_source, ")"))) # Geocorr geography step (final step) geocorr_step <- tibble::tibble( @@ -286,6 +307,23 @@ plan_crosswalk_chain <- function( tgt_year, " ", src_std, " -> ", tgt_year, " ", tgt_std, " (inter-geography via Geocorr)")) + # GeoCorr snapshots geography at its reference year (2018 or 2022); warn when + # county change events make the requested year's county codes differ from the + # GeoCorr vintage's codes (county codes prefix block/bg/tract GEOIDs, so this + # affects the join between the temporal steps and the GeoCorr step) + tgt_year_num <- as.numeric(tgt_year) + geocorr_ref_year <- if (tgt_year_num >= 2020) 2022 else 2018 + if (tgt_year_num >= 2000 && + tgt_year_num <= county_events_meta$known_through_year && + !tgt_year_num %in% c(2010, 2020, geocorr_ref_year) && + !identical(county_universe_at(tgt_year_num), county_universe_at(geocorr_ref_year))) { + cw_warning(stringr::str_c( + "The final geography step uses GeoCorr ", geocorr_ref_year, + " for year ", tgt_year, ". County codes changed between ", + geocorr_ref_year, " and ", tgt_year, + " (county change events), so some GEOIDs may not align with the GeoCorr vintage.")) + } + result$steps <- dplyr::bind_rows(temporal_steps, geocorr_step) result$composition_note <- stringr::str_c( @@ -355,6 +393,63 @@ determine_temporal_source <- function(source_year, target_year) { } +#' Validate County Temporal Requests Against County-Events Constraints +#' +#' County temporal crosswalks are served by the county-events engine, which is +#' forward-only and covers 2000 through the curated horizon. Returns an error +#' string for requests that violate those constraints, or NULL. +#' +#' @param geography_std Character. Standardized geography name. +#' @param source_year Character. Source year. +#' @param target_year Character. Target year. +#' @return Character error message, or NULL if the request is valid. +#' @keywords internal +#' @noRd +validate_county_events_years <- function(geography_std, source_year, target_year) { + if (!isTRUE(geography_std == "county")) return(NULL) + + source_year_num <- as.numeric(source_year) + target_year_num <- as.numeric(target_year) + known_through <- county_events_meta$known_through_year + + if (source_year_num > target_year_num) { + return(stringr::str_c( + "County crosswalks from ", source_year, " to ", target_year, + " are not supported: county temporal crosswalks are forward-only ", + "(reversing county changes would require disaggregating merged counties). ", + "Swap source_year and target_year to crosswalk forward in time.")) + } + + if (source_year_num < 2000 || target_year_num > known_through) { + return(stringr::str_c( + "County temporal crosswalks cover 2000 through ", known_through, + ". The provided years (", source_year, " -> ", target_year, + ") are outside this range.")) + } + + NULL +} + + +#' Hint Appended to No-Temporal-Path Errors +#' +#' For geographies covered by the county-events engine, explains which year +#' combinations it can serve. +#' +#' @param geography_std Character. Standardized geography name. +#' @return Character. Hint text (possibly empty). +#' @keywords internal +#' @noRd +temporal_path_hint <- function(geography_std) { + if (!geography_std %in% c("tract", "block_group", "block")) return("") + + stringr::str_c( + " County-change-aware ", geography_std, " crosswalks are available for ", + "same-decade year pairs from 2010 onward (e.g., 2014 -> 2019); year pairs ", + "spanning a decennial census chain through NHGIS crosswalks where those exist.") +} + + #' Format Crosswalk Chain Plan as Message #' #' Formats a crosswalk chain plan as a human-readable message. @@ -396,21 +491,24 @@ format_chain_plan_message <- function(plan) { } -#' Find Temporal Path Between Years via NHGIS Same-Geography Crosswalks +#' Find Temporal Path Between Years via Same-Geography Crosswalks #' -#' Uses BFS to find the shortest sequence of same-geography NHGIS crosswalks -#' bridging a temporal span. The graph is built from `list_nhgis_crosswalks()` -#' so future NHGIS additions are picked up automatically. +#' Uses BFS to find the shortest sequence of same-geography temporal crosswalks +#' bridging a temporal span. The graph combines NHGIS same-geography edges +#' (from `list_nhgis_crosswalks()`) with county-events edges (from +#' `list_county_events_edges()`); NHGIS edges are preferred when both provide +#' the same hop. Each hop is tagged with the `crosswalk_source` that serves it. #' #' @param geography_std Character. Standardized geography name (e.g. "tract", -#' "block", "block_group") matching `list_nhgis_crosswalks()` output. +#' "block", "block_group", "county") matching `list_nhgis_crosswalks()` +#' output. #' @param source_year Character or numeric. Source year. #' @param target_year Character or numeric. Target year. #' @param available_crosswalks Tibble or NULL. Pre-fetched output of #' `list_nhgis_crosswalks()` to avoid redundant calls. #' -#' @return A list of `list(source_year, target_year)` hops representing the -#' shortest path, or `NULL` if no path exists. +#' @return A list of `list(source_year, target_year, crosswalk_source)` hops +#' representing the shortest path, or `NULL` if no path exists. #' @keywords internal #' @noRd find_temporal_path <- function(geography_std, @@ -424,20 +522,31 @@ find_temporal_path <- function(geography_std, source_year_chr <- as.character(source_year) target_year_chr <- as.character(target_year) - # Filter to same-geography temporal edges - same_geog <- available_crosswalks |> - dplyr::filter( - source_geography == geography_std, - target_geography == geography_std) - - if (nrow(same_geog) == 0) return(NULL) - - # Build adjacency list: year -> list of reachable years + # NHGIS same-geography temporal edges, then county-events edges. Insertion + # order matters: BFS prefers earlier edges on equal path length, so NHGIS + # remains the source for hops both can serve. + edges <- dplyr::bind_rows( + available_crosswalks |> + dplyr::filter( + source_geography == geography_std, + target_geography == geography_std) |> + dplyr::transmute( + source_year, target_year, crosswalk_source = "nhgis"), + list_county_events_edges(geography_std) |> + dplyr::mutate(crosswalk_source = "county_events")) + + if (nrow(edges) == 0) return(NULL) + + # Build adjacency list: year -> tibble of (target_year, crosswalk_source), + # keeping the first (preferred) source per target year adj <- list() - for (i in seq_len(nrow(same_geog))) { - from <- same_geog$source_year[i] - to <- same_geog$target_year[i] - adj[[from]] <- unique(c(adj[[from]], to)) + for (i in seq_len(nrow(edges))) { + from <- edges$source_year[i] + to <- edges$target_year[i] + if (to %in% adj[[from]]$target_year) next + adj[[from]] <- dplyr::bind_rows( + adj[[from]], + tibble::tibble(target_year = to, crosswalk_source = edges$crosswalk_source[i])) } # BFS from source_year to target_year @@ -458,11 +567,15 @@ find_temporal_path <- function(geography_std, neighbors <- adj[[current$year]] if (is.null(neighbors)) next - for (neighbor in neighbors) { + for (j in seq_len(nrow(neighbors))) { + neighbor <- neighbors$target_year[j] if (!neighbor %in% visited) { new_path <- c( current$path, - list(list(source_year = current$year, target_year = neighbor))) + list(list( + source_year = current$year, + target_year = neighbor, + crosswalk_source = neighbors$crosswalk_source[j]))) queue <- c(queue, list(list(year = neighbor, path = new_path))) } } diff --git a/R/sysdata.rda b/R/sysdata.rda new file mode 100644 index 0000000000000000000000000000000000000000..f7dcde50147e7a3b95dc41b51a0193d7191009bc GIT binary patch literal 12881 zcmcJ03p|wD_xHGjI+a_>ofy>QRPGWJp^Px7DcvZXLJdtQX_QJujgc7@3Mt2BTqbf$ zD8g`#ZbqhzB;zCv)r1nAde<|9I-PTV|KI=fzVGvSKF{oDU)ElG?X|vZt^KfiOTY>v zyUqIRH~Tfv&BZ?4J^Jdj^b= zw!5=h4}}VbVr35srGT2VBw9TBr*0{f1V>uRN#c%}?N4Ib+A?Ab#rV-E)IJp%jS=XJ zTvE}2s~3w)Dr(CF`PUhvM_S@&N6rw9lY;{Lk}tU`y6B&3-{oqMlclJio=OQwonvbH zklkh~klCTAPiZ#|%*npi9(nrcI3wrO0|EhOrlRU!AF$K(p{kLQ#F}`O<@R#QGRlhb z*X+er1AJ8j*!pX|If)I%#B@iJ)IymGJJSBJThGoPlni9WCLiib9E9K|RX6B9;jx zP!emRg=>Z7nxdqRU9;`ZI*xV*k--%pd_s4kt*C@N#kQ-i5+^+QDR5WEXn~_EN;@}3 zPlI-%*fF<_W@kIkK*nu9>caw>?cR>w-)&`095mi0DqaaE<^NeV@#fe^g+X$BVI=?P zqvs#~oNdZjSs+d{kUZ)A@%AG_>j%+2KetF9M9Jiy{t2aeTpXVvExTly{kel0@kZ2+ zuZ792a%Z#`k-su9;d)7RuiIh_Ase(vdM*yx&+!I-}l6PR{+&tU=X- z(RMV4MW;i=^fjN&`$PLDI}b~-4V?w)Ua~enb!wmZ&C#)DYaQy1yf?XDmiiU4DV2-Ey(DVyJgcn?^j3daPV!P!REX)tquJ z5xF~YXxevo#tNK496Cu|{r7bLm#)XEesV&qB+DF6OgnbfCFb`B2lw8(bff&K+0fmHCCpLQ+z`O*6@oh@rs`lk5A> z9DYCkn?Upi{=TaJOlOJ!VM%EuW zy`yhvcm>!LQT4x)xZMA*O`_@PPY3@qA0H0?4-{Z{`ddwt%OW)~sN=B`H2+~p%|F!9 z@L{xw@2Bry0liF2uyaFM2@2EuJwksbB8pEIKmlo3*EZeym%@L!cmKqTTj=+k{8M?w z{lCoh1CeQs7S{h8gJ>FlcgcSuGes3|O@45qXAC=c_%WYL^QVQFNe zPo#(XA>KF87nz)~pgNyB&M`Uy>M+O6+To9qZ!kQ#W~^3$)uo%Hj{C2Jc4e?4te z6M5LC#wPNk%b4|F`2H_!`A0pQR2ZnK(=Yd_sw?-=*OQ!l|nE26EfO1ZXP&RN%Ht#Vb#5|aE1 zN4-TUGageOzO;B*Ilx-Ha9?PW(LEJKmjCVGiCxcc$BGpx*^1dSB^_tT5Kt5EWHSSr z#gD7KcXxEKb`pCUJ^B!ZnWdhq9R14HXU4nZ(Tfg$&DXZ|m@znRe8?R$v1q-u?$8dS zgcp20bGf-Jm%F2j4xS??V;f{!2Kr{c&h*Y!eD!wxxmMbSw>AbhCq|Amkmo+Z&Wn&s z3riU-8IC%A>vdJtf{*VG4w`t+kh!>XpQ1Kj!S3mSlY2w#n;u~tm)d@4DQ`aNJu~Oj z?xml+%v?H_&i`r8;2_x@-{J5uWH7*-sqfMg$9;Gp)v`i7!1Oo0MCIRm#1Gu4Z~JX1 zMYYFlP1D)knK8|e%bfidXs8=sHVblF(=ESdY@a(T_QiaKlET7)Ir{qwf8RDzSzX`r zT#qbRGEr7~U3~mp=H6fzRI*A^n$9i9Zpwr8gyObLDMj~pGG*unyJESHnNCRy1dBZ{ zEYs?!5LgQ?b)ldVsylZ^lLGj}~=!syGn}rd}4-+xQ^HMJvm1s`Td+)EO|JR#O0_r znTa%2zpQZ2_Rm8+9+R2Fd5}gP?!TF_@;SN!U9#o5Qd3`WHemX-_m46aq-1ps>Rf=tFQrzpE8g_9#q^m6936)A*rFs$Sm(0vs>Cb%526XM`6I_ z{uTE_(@ji-znA+B*sQ{k2gxkvdMdgDmIcINm>E*YF3T10CcCZRPZnUpu~9asa8JMT z=xb6#>hxk@a8TqZzmHfhVuj*=s;1`c(yZAElfyuKv3%2MH#D0VV{H$bW6E|9%uYSwYTB+En0?xd$6>#=iR2FPIZs@C zCnbTn9_xs)u~9zn)h6|zIYx;T0^tnMW)u}ut3GKdCTxv5&Ggx66ekh{76LCAqot); zLq6=^iFAD7g*02_evm#VcKN6Ha$cD%FWY;8P@Jd8NzlZ0kpstwV@DK~ zfz4Fmh9O(8uxC9qnz3-5t}Y-Qp+2}EPyI6TCq64db(9s@i=#l3E3(6dNK>3y9-G51 zuO@Xy(7hUj8~{N+hl#MxVS#XK?emog1b@ONFS@Fq5&>awGgJ^LzKk=7+=>Z$$lg`; zx)~A(XO;P!1dW`yLO(@h247^J0zYMzf}iRrx7?b<^)@q$6Dp}txq#_7qxi@nuBPP> zQ#BRz6)KSCVv3(Sj>4xN{ngc@#>IXRSuD-}Bgad)I zI7^<^t}*fi+JH^PTCv#p94^ZTFWd#DxAi35DB2IqH1MdFym({+x-zHe`xyX)oPTpJ)Gk$+;{I<>&OY*!jZuXzk;MnF!^q!bY}*m5EeP# z5hUoR%(#k3uz-Lg5WLt$mGrYp1mD{^t!YQ3|G0`X<-cDww>I_2n&*il9m4sx0XHNN z4@-5zUb8lq9*+fW#K+fS)2qXgX^W6vhz#SSPFe6+u6`_yGTwSG;qs87NR#SizwKrs z(fJ_^n?XGbL`Wg!#YYSjj0sr8i1xouH=4(x9&yqk2Y!^HbA0gXmlSWnnQZys=qS<3N5>y6I>IdqC6y=8YVM{fy*1A!T4A2B;t_O>~D9GEO8g9*b&2 z4(}sUNIdEjfqpTa@XD=m1wbpD1Fh=epv~%$V}OdFa|BGj zdZ7M!0$~emAqE7@Xfh%=61*7sN}d2MraUUTw~^qL|H+CS5+A_{o6AR zV1mLPgggoK-@6iKigz>OWm^hZd%-vmDJFOWUrm)fN)RbQ!-z#)>DQ2ounSA%bBaKe ze(Nbw)=|?fARvSs2iD4BX&f4(;T8zAE?nI@zeQoIwt&OtabDzbL+*YPm8k0rmd|Q@ z6)(iwY9dfj0eB}AHA1Na=8mFsG^$AahDmNU!VwOauQ#Zg8CD&!FA(Yy*bn~3Bg{Yqpp!rezk7zh{o7rsH#%mixnnXeydrir?H zgPmf=<qmh-#3a35{4D0@jz23L(usTL#$KK-3eT(*}qv1WMOE6(Kq73L@e(;`bod zVcC&I&jF4Q8pyp)l4tBFMV zux|8m)ZL!%O64J|78-<2kf+yX(nz(2l@bYKj{6xT8|9q`L1?$5jmRW@31xlW{4dI;l z+6>i*Q64jAdw3Hpbt}T*7^ZIHV%5N#rel13LV^Z)nhJjFFPf(8@|pu3#ek&=+pECh zEqUHmVFOI>vH}FKIKf~jR7*}#nh_s#LI9qE>q6ocR*hD)$%MZyJi{nP5Q{7nJAPEhG^bI52i*6c)`}9E`G%7Nai*Y#Sc7%fSXg4ZoOyAC||j$CKBdoGUo#FlEAdK zk(xUFD8veIGl-#0QS+hVR*9F7H$yax$xq0>hB&jr$xu1*41~tu#*_$z_*HYVxO4^I z#$AUv>~}U%wcLoY&!9#KN1S%&QP?ORcrRFi8gVd!8w}nDc`G!1Z%wSwv=Xq?iP=kB ztr12im(m!RllbJb6U}*${n~*>2D|mHMF>KDF^^a&;0R^3C6G4&3^ToLfLbFdClP7t z==4)AEPcf%R`Lr=n~b&wa@w769T$*@o*da82>L?EHu8t(Mq?w9R~FjEi_%LQ&ij^s z>W!*L3r5CCd=VW%pN=zxd{zPw7iA*^rG@3ZqT;Q-Le#Ly2Mg(6eR}gQ>RVT);RSoax3U4yg zAUnlb_MQhOsw4iLJ$9gZ9;We#3Ga=(fsd;>AD?aIL@xwuv0^pzgR6L|p@qc=Ia9t{ zBR{y5?^dSvaW72~r(jLPX)0jRPTb@6d%FZ3rOzHf(>X5WLz|08{aYCfw)!WtpiV2#sul#7FiWj`$kAEB-GWY|Z+3{Q} zBu9H33H06`fApv1=l2N4GNo?}`;P}8#UeY{WtMcU;h#!jtIPz*)&H_N4F&{Kj+|`j zA~kV|%kas9VSH-nvo{|P1FEL{^|I-<=^lJ=@Yb`GOKcP5GREn>#w|u*7dhjDdKh*? z*fRC7$OgzdTpa=RHr;OzOyZaSxGdBA-1){c5L#^eQ3*g75c{2sDnJSH-_Ish#Mi-r zbmaJ65~oA#L4vw^PIiB&eSWUg3UdFF3LFZdFaosily> z%?B@Hp$o2KKkhvaO-!$y?F>g<-8P(@9ATJh15T5M<8$)NxxR%RJg7N-|P3abph6(L4&xiS3GEFe=Z*u?RqfBx1s!0_*Y9{f@?y z&PHONj+?bV&8Z0kdH0db&mUt36_1Q;*o?B6bSYj z5!?p)B|OBSvl29{!bC^aNc6*tgFd3L1o|QsRAdl+3&$T!)1{HCs{Xv$nsf5n`gSU*H!t5?@~fBsr-blV zIK8PGy@Gc-L9s!2P6R%G+D}K&Kz{)BK{vU9zrv63+L$UV$CtR=728vCEO*I;WjRLv~kvT#8a>7ULLq3e})&PWj-9koKS8yqMT5!d!gIh z)0Rcc;!&W>7Ehxcj1|&ej66(!0ir_$KLsHR9$>GH8_pA`)xiibMqtkBHpUqyh+WxH z@6<)~-$ePo^JY-{f-?*TD?u&3)=1xQQ&!@u589W4=!6^g($G z2#G+x848i3l|oa|!Oxf1OYA{6J0#~BQfO#8sVvdk7=kT0cfbMVd?kNa&o4DO6m$qMz&N}~ONJ%LSP&;VSN2npU5Y98A(Wu;`>M&;3cB>zn! zfrF3<1Ya{{{{<#Yg9DPO*NBM1p(GSUc5Z;2*%1id zH4v;rFat}_tO>%6jDMS6n7{##LO6$C#OF>Sj9aZ$*yQnNnL#^U2oA_d5ypL6n2>Wa z4h_!N?Gl%25C@p#iL>ZuXItafH_(x+%uiK>QEty}9Y_apA^97=exmw8#`e!|Yi~`g zicM~wh^qR0f{J|}Qc;wM9OG=$(b3!H;Cv@GHa5;jdxy5X{F23gopk?U6YIYxE|BxM zASr|BLQZ%gH<&!*t(ndgg!&9>%gl11#950xAJ`J$20;%FLNWQ8R$aJqY**lWgFlKn zO*s8r1S{$htN_5EA;0jVtK3ZAA`I?8jo|VRc5NE`Sw2W{*vz19;cgvgfzS~l(dx7^ zV>4V4EsG6T&`%9i(9enqLbF9G4D$+4YtBICfPml=9HDK*G@K{Imve(C63@Q=x(K5uhANp=F7J6fy03vA(wcF>I-F_x_80E3 zO7>&cRtOLxb^foeWeC6v)8lCc{?~%b)TW3|_49s?FaE1|Hli5?#M1Qcnx&z%k*UtD z_xK$UQ0<=FI%J*BjABJ%^LdI8ZADV^Eg-~U^3{mtkCpV|mh7Eufyko(QXvp0{JBCi zBVS?TQ=!!Zc0&r`TYw{YL1tZINWl3zbA`?Wc3(J1M6AQ{=tbYvZaO#vw2qrLjRe6N zhv}o00VrSu-w%Y&AH20DoO1}xHe+oYVj&1FOiLldM+@E%2U$3EyaJ8-h{ZKbccLL4 zb@B}{5-7sgjPWQmM8Ob&L?BJVK-UurZiw9@CNy0mj2xiPBzEW%3eT2RO z9$dZolcy#yaPpqQ522Sf0C+%-X~G3az%Xg#(~-^Ja0{UUe%^qIWULTKP7)QGyYIIE z;J^?U4k7@En%G1pJU^nDkoAhv|0x&q|M+)`JYL7OT zB$g8^D2aKw%`}lsFo0u59CVKd^^9zA5k<7$)#hQo&}~z6(3fM>B`?R$yvf_dfDuR~cp6dU^{=`F3ZY_306?Cn z;O8EPsYf?0Dl#?#u}?7o;77sbDg7F^ghD~E0n;p$Be6UWps`%kQwKQpjaQXuKK5s> zU5U)ea~_7`V!t>bOmmnyaI7y7glA()$B+xO7YrOMkPPQV?~Y)NsQF}@=>c2 z$r<8t;PqkLkTBywFpG9@h@&1pT$(l^7#S^$h@g4HUjvPz>Fr3t20{dM679b>uqnj< z!NR6rTZJce!N|8%$PGL(ElTm`+#p&xR~T_9v%Z}RPCJi~%~QIEXyT;$eakeuZHe`;W_0i(iCrCKj_8pyBKxB^tkq{`i$QW1?qboy+Ybl`R@AgUAjN#s<$E9Y)E?pJuSZ96;-LGBM+L@|g zd-1q?u~8fCRoZ%wONT?|B+(bEw)WrhQ(Wx-e4&kDqPeAaGsDC46HlXkKAz zJe8J78Q<|BwUvQOd?I0MKHHA<$5Ug?Q)T+fIGR&zY>X~XV%g!IG0jzrzFd~8eD+vx zX09E2v$(^egQ_W65@L7u>K3gi!JJ#3u&yeEmQ=5Nd|s^7K3z+~#ky1l8=LiIb5Sn$ zU8Jj5^bM-eBsA|N=5N9MOuln{gXynSZIfKp)x$|EN%!N3JDrZlpj9;W=4KTYUBNX_ zvN9FhG8flv(XiJUSZEyoWW%2-7rL|)55;&FJ&JAH(O#(@`T2^<+30{falsZUCF{hN zT++B;CV5F$*Sub>tkq*Q^lqM`#JMfY}H-uq+z=uLS0aa$<1bpO2@a`G*6 z$YFchN#YKY_PB2FtJjv&`%*RS#zulIf1)Nomr*$8 zeyDV&tlSlY8T}GySG9m^J7QNao&8eHY@ozYjr)XF*kjzH=7-XkTq|SX?Q+49r*hW9 zXYRi4h$pk6R-QiSUwv7nu1@2A^7SChI(f@G56`2l6B_#iP%3g8#TQDSvppdvwa;pA zN0?so>J9p|m=s_CmOV+7`^S>!oIL2|zxPBG)8TxGjM$}Kx_HhXPV*Rc3o7(&7wgEM z8-I^8I+mKZdav4|q3a~~9*Hvv+)K?=38$3~$L?pgiSKJ&?|w1RAE9=>1UPacwyU4W)zJSOc+SEtSfrrVNug` zA}iqZ67}^?dkwYiQRjQyxzb_QALoRnymT4fb}+H=K*XJKec8dD-qX^~$!B#AT+#o{ zc+kN~$)f)Emgi8^%U>>w3+r{5(S_N!SBjEkA%@0gVvj9PnPnU0(|?BEmR&D@eRAj?t;_;|?$2+4ax&KKXtfJ6lTLK1}i&RU&5J-6bwJl9HCpo2zo; zXQlF+iw{|KnBfncY)(EoScfr5dg|G;?uDn!eV1ACxz(vFoZd+!&o`F5(|;lboi&oX z=JLmd+ft6%#h=}+_Qu1aXJ+Yj`fD%zg|IG{M8m;^T|?zuNsIl-n0di@wR4T_`m$1H z5Yq=67pq7l=eorDipwZ3lsP3Md7Bq*V!p|Cw!wy-m8UXFH!CHIXWj2I)y>mSH0gNt znwD7*Ep4^hGSf3iR&npS%&yScf7VK5i+#4qGd!-Iva(+4v%!p=2a^q2{ePRYXX)8xR zzvCB~C#p2u^+Ut{eRr*+O~_xRuT5O)Pfa-W$=l%5p~$1R-+WmZGWWvp&qIE8Z_ZHC zwxs+2eur)UMeDR}oYT&AcPMgKY?cLB*S*as&a+PsYr?9(Co`9Q`NFI^a&f`6;i9*- zOHW*UJdyHcnc;g|ce}Lb_)m-6-`C*jQ8LusaShd9pLp0wYu;$>`}o-Av*wzRlqbKP ziPf&l;m=w;>@;g3K}UODWjt9|T)Y>hZMp6fe=JWWOlI7R93!^)=ijfmXm3}LTmE31 X=Gk$xlUHTGF4*sA>3$CR7n1)2oK=xB literal 0 HcmV?d00001 diff --git a/README.Rmd b/README.Rmd index b663855..43171b6 100644 --- a/README.Rmd +++ b/README.Rmd @@ -1,290 +1,189 @@ ---- -output: github_document ---- - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>", - fig.path = "man/figures/README-", - out.width = "100%", - eval = TRUE, - echo = TRUE, - message = FALSE, - warning = FALSE) - -devtools::load_all() -``` - -# crosswalk - - An R package for translating data across space and time. - -## Overview - -This package provides a consistent API and standardized versions of crosswalks to enable consistent approaches -that work across different geography and year combinations. The package also facilitates -interpolation--that is, adjusting source geography/year values by their crosswalk weights and translating -these values to the desired target geography/year--including diagnostics of the joins between source data -and crosswalks. - -The package sources crosswalks from: - -- **Geocorr** (Missouri Census Data Center) - for inter-geography crosswalks (same-decade) -- **IPUMS NHGIS** - for inter-temporal crosswalks (across decades) -- **CT Data Collaborative** - for Connecticut 2020→2022 crosswalks (planning region changes) - -## Why Use `crosswalk`? - -- **Programmatic access**: No more manual downloads from web interfaces; data is cached for speed -- **Standardized output**: Consistent column names across all crosswalk sources -- **Metadata tracking**: Full provenance of crosswalks stored as attributes -- **Crosswalk chaining**: Automatic chaining when multiple crosswalks are required - -## Installation - -```{r, eval = FALSE} -# Install from GitHub -renv::install("UI-Research/crosswalk") -``` - -## Quick Start - -First we obtain a crosswalk and apply it to our data: -```{r} -library(crosswalk) -library(dplyr) -library(ggplot2) -library(stringr) -library(sf) -library(tidycensus) -library(tigris) -library(scales) - -source_data = get_acs( - year = 2023, - geography = "zcta", - output = "wide", - variables = c(below_poverty_level = "B17001_002")) %>% - select( - source_geoid = GEOID, - count_below_poverty_level = below_poverty_levelE) - -# Get a crosswalk from ZCTAs to PUMAs (same year, uses Geocorr (2022)) -zcta_puma_crosswalk <- get_crosswalk( - source_geography = "zcta", - target_geography = "puma22", - weight = "population") - -# Apply the crosswalk to your data -crosswalked_data <- crosswalk_data( - data = source_data, - crosswalk = zcta_puma_crosswalk) - -## Or in a single step -crosswalked_data = crosswalk_data( - data = source_data, - source_geography = "zcta", - target_geography = "puma22", - weight = "population") -``` - -What does the crosswalk(s) reflect and how was it sourced? -```{r} -## and there's more (not shown) -names(attr(crosswalked_data, "crosswalk_metadata")) %>% head() -``` - -How well did the crosswalk join to our source data? -```{r} -## look at all the characteristics of the join(s) between the source data -## and the crosswalks -join_quality = attr(crosswalked_data, "join_quality") - -## what share of records in the source data do not join to a crosswalk and -## thus are dropped during the crosswalking process? -join_quality$pct_data_unmatched - -## zctas aren't nested within states, otherwise join_quality$state_analysis_data -## would help us to ID whether non-joining source data were clustered within one -## or a few states. instead we can join to spatial data to diagnose further: -zctas_sf = zctas(year = 2023, progress_bar = FALSE) -states_sf = states(year = 2023, cb = TRUE, progress_bar = FALSE) - -## apart from DC, which has a disproportionate number of non-joining ZCTAs-- -## seemingly corresponding to federal areas and buildings--the distribution of -## non-joining ZCTAs appears proportionate to state-level populations and is -## distributed across many states: -zctas_sf %>% - filter(GEOID20 %in% join_quality$data_geoids_unmatched) %>% - st_intersection(states_sf %>% select(NAME)) %>% - st_drop_geometry() %>% - count(NAME, sort = TRUE) %>% - head() -``` - -And how accurate was the crosswalking process? -```{r} -comparison_data = get_acs( - year = 2023, - geography = "puma", - output = "wide", - variables = c( - below_poverty_level = "B17001_002")) %>% - select( - source_geoid = GEOID, - count_below_poverty_level_acs = below_poverty_levelE) - -combined_data = left_join( - comparison_data, - crosswalked_data, - by = c("source_geoid" = "geoid")) - -combined_data %>% - select(source_geoid, matches("count")) %>% - mutate(difference_percent = (count_below_poverty_level_acs - count_below_poverty_level) / count_below_poverty_level_acs) %>% - ggplot() + - geom_histogram(aes(x = difference_percent)) + - theme_minimal() + - theme(panel.grid = element_blank()) + - scale_x_continuous(labels = percent) + - labs( - title = "Crosswalked data approximates observed values", - subtitle = "Block group-level source data would produce more accurate crosswalked values", - y = "", - x = "Percent difference between observed and crosswalked values") -``` - -## Core Functions - -The package has two main functions, though you can also specify the needed crosswalk(s) -directly from `crosswalk_data()` and omit the intermediate `get_crosswalk()` call. - -| Function | Purpose | -|--------------------------------------|----------------------------------| -| `get_crosswalk()` | Fetch crosswalk(s) | -| `crosswalk_data()` | Apply crosswalk(s) to interpolate data to the target geography-year | - -## Output Structure - -`get_crosswalk()` **always returns a list** structured as follows: - -The list contains three elements: - -| Element | Description | -|------------------------------|------------------------------------------| -| `crosswalks` | A named list of crosswalks (`step_1`, `step_2`, etc.) | -| `plan` | Details about what crosswalks are being fetched | -| `message` | A description of the crosswalk chain | - -### Multi-Step Crosswalks - -For some source year/geography -> target year/geography combinations, there is not a single direct crosswalk. -In such cases, we need two crosswalks. The package automatically plans and fetches the required crosswalks: - -1. **Step 1 (NHGIS)**: Change year, keep geography constant -2. **Step 2 (Geocorr)**: Change geography at target year - -```{r, eval = FALSE} -result <- get_crosswalk( - source_geography = "tract", - target_geography = "zcta", - source_year = 2010, - target_year = 2020, - weight = "population", - silent = TRUE) - -# Two crosswalks are returned -# Step 1: 2010 tracts -> 2020 tracts (NHGIS) -# Step 2: 2020 tracts -> 2020 ZCTAs (Geocorr) -``` - -### Crosswalk Structure - -Each crosswalk contains standardized columns: - -| Column | Description | -|----------------------------|--------------------------------------------| -| `source_geoid` | Identifier for source geography | -| `target_geoid` | Identifier for target geography | -| `allocation_factor_source_to_target` | Weight for interpolating values | -| `weighting_factor` | What attribute was used (population, housing, land) | - -Additional columns may include `source_year`, `target_year`, `population_2020`, `housing_2020`, -and `land_area_sqmi` depending on the source of the crosswalk. - -### Accessing Metadata - -Each crosswalk tibble has a `crosswalk_metadata` attribute that documents what the crosswalk -represents and how it was created: - -```{r, eval = FALSE} -metadata <- attr(result$crosswalks$step_1, "crosswalk_metadata") -names(metadata) -``` - -## Interpolation - -`crosswalk_data()` applies crosswalk weights to transform your data. If you're in a hurry, -you can omit a call to `get_crosswalk()` and specify the needed crosswalk parameters -to `crosswalk_data()`, which will pass these to `get_crosswalk()` behind the scenes. Or you -can call `get_crosswalk()` explicitly and then pass the result to `crosswalk_data()`. - -### Column Naming Convention - -The function auto-detects columns based on prefixes: - -| Prefix | Treatment | -|-------------------------------|-----------------------------------------| -| `count_` | Summed after weighting (for counts like population, housing units) | -| `mean_`, `median_`, `percent_`, `ratio_` | Weighted mean (for rates, percentages, averages) | - -You can also specify columns explicitly via `count_columns` and `non_count_columns`. -All non-count variables are interpolated using weighted means, weighting by the allocation factor from the crosswalk. - -## Supported Geography and Year Combinations - -`get_available_crosswalks()` returns a listing of all supported year-geography combinations. - -```{r} -get_available_crosswalks() %>% - head() -``` - -## API Keys - -NHGIS crosswalks require an IPUMS API key. Get one at https://account.ipums.org/api_keys and add to your `.Renviron`: - -```{r, eval = FALSE} -usethis::edit_r_environ() -# Add: IPUMS_API_KEY=your_key_here -``` - -## Caching - -Use the `cache` parameter to save crosswalks locally for ease: - -```{r, eval = FALSE} -result <- get_crosswalk( - source_geography = "tract", - target_geography = "zcta", - weight = "population", - cache = here::here("crosswalks-cache")) -``` - - -## Citations - -Cite the organizations that produce the crosswalks returned by this package: - -**For NHGIS**, see requirements at: https://www.nhgis.org/citation-and-use-nhgis-data - -**For Geocorr**, a suggested citation (update the year): - -> Missouri Census Data Center, University of Missouri. (2022/2018). Geocorr 2022/2018: Geographic Correspondence Engine. Retrieved from: https://mcdc.missouri.edu/applications/geocorr2022/2018.html - -**For CTData**, a suggested citation (adjust for alternate source geography): - -> CT Data Collaborative. (2023). 2022 Census Tract Crosswalk. Retrieved from: https://github.com/CT-Data-Collaborative/2022-tract-crosswalk. - -**For this package**, refer here: https://ui-research.github.io/crosswalk/authors.html#citation \ No newline at end of file +--- +output: github_document +--- + + + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + fig.path = "man/figures/README-", + out.width = "100%", + eval = TRUE, + echo = TRUE, + message = FALSE, + warning = FALSE) + +devtools::load_all() +``` + +# crosswalk + + +[![R-CMD-check](https://github.com/UI-Research/crosswalk/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/UI-Research/crosswalk/actions/workflows/R-CMD-check.yaml) + + +An R package for translating data across space and time. + +## Overview + +Census geographies change across time (tracts are redrawn every decade; +counties are occasionally renamed, merged, or split), and many analyses need +to move data between geographies that don't nest within one another (ZCTAs to +PUMAs, tracts to places). `crosswalk` provides a consistent interface for +fetching the crosswalks that relate these geographies and for applying them to +your data -- adjusting source values by crosswalk weights to produce estimates +for the target geography and year, with diagnostics describing the quality of +the join between your data and the crosswalk. + +Crosswalks are sourced from: + +- **Geocorr** (Missouri Census Data Center) -- same-year crosswalks between + geographies (Geocorr 2022 for 2020s geography; Geocorr 2018 for 2010s + geography) +- **IPUMS NHGIS** -- inter-temporal (cross-decade) crosswalks +- **CT Data Collaborative** -- Connecticut's 2020-2022 change from counties + to planning regions +- **A curated registry of county change events** (ships with the package) -- + county -> county crosswalks between any pair of years from 2000 onward + +## Why use `crosswalk`? + +- **Programmatic access**: no manual downloads from web interfaces, with + optional local caching +- **Standardized output**: consistent column names and structure across all + crosswalk sources +- **Provenance and diagnostics**: full crosswalk metadata and join-quality + statistics attached to every result +- **Crosswalk chaining**: transformations that change both geography and + year are planned and applied automatically + +## Installation + +```{r, eval = FALSE} +# install.packages("pak") +pak::pak("UI-Research/crosswalk") + +## or, in renv-managed projects +renv::install("UI-Research/crosswalk") +``` + +## Quick start + +Translate ZCTA-level poverty counts to PUMAs. `crosswalk_data()` fetches the +needed crosswalk (here from Geocorr, population-weighted) and applies it; the +`count_` prefix tells it to treat the variable as a count (multiply by the +allocation factor, then sum by target geography): + +```{r quick-start} +library(crosswalk) +library(dplyr) + +zcta_poverty <- tidycensus::get_acs( + year = 2023, + geography = "zcta", + output = "wide", + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> + select( + source_geoid = GEOID, + count_below_poverty = below_povertyE) + +puma_poverty <- crosswalk_data( + data = zcta_poverty, + source_geography = "zcta", + target_geography = "puma", + weight = "population") + +head(puma_poverty) +``` + +To inspect or reuse a crosswalk, fetch it explicitly with `get_crosswalk()` +and pass it to `crosswalk_data()`: + +```{r two-step, eval = FALSE} +zcta_to_puma <- get_crosswalk( + source_geography = "zcta", + target_geography = "puma", + weight = "population") + +puma_poverty <- crosswalk_data( + data = zcta_poverty, + crosswalk = zcta_to_puma) +``` + +Every result carries its provenance and join diagnostics as attributes: + +```{r attributes} +## where did the crosswalk come from? +attr(puma_poverty, "crosswalk_metadata")$data_source_full_name + +## what share of data GEOIDs failed to match the crosswalk (and were dropped)? +attr(puma_poverty, "join_quality")$pct_data_unmatched +``` + +## Core functions + +| Function | Purpose | +|---|---| +| `get_crosswalk()` | Fetch crosswalk(s), chaining steps automatically when geography and year both change | +| `crosswalk_data()` | Apply crosswalk(s) to a dataset, interpolating count and non-count variables | +| `get_available_crosswalks()` | List every supported geography/year combination and its source | +| `list_nhgis_crosswalks()` | List the NHGIS (cross-decade) crosswalks specifically | + +```{r available} +get_available_crosswalks() |> + head() +``` + +## API keys + +- **IPUMS_API_KEY** -- required for NHGIS (cross-decade) crosswalks. Get one + at . +- **CENSUS_API_KEY** -- required for county-level 2020 <-> 2022 requests + (used by `tidycensus`, which also powers the examples above). Get one at + . + +Store keys in your `.Renviron` (e.g., via `usethis::edit_r_environ()`). +Geocorr and county-events crosswalks require no keys. + +## Caching + +Pass a directory as the `cache` argument to `get_crosswalk()` or +`crosswalk_data()` and each fetched crosswalk is saved there as a CSV; later +calls with the same parameters read from disk instead of re-downloading. +Cached files never expire -- delete a file to force a re-download. + +## Learn more + +- [Getting started](https://ui-research.github.io/crosswalk/articles/crosswalk.html) + -- the core workflow: fetching, applying, multi-step chains, metadata, and + join quality +- [How interpolation works](https://ui-research.github.io/crosswalk/articles/how-interpolation-works.html) + -- allocation factors, choosing a `weight`, accuracy, and diagnosing joins +- [County crosswalks between any years](https://ui-research.github.io/crosswalk/articles/county-crosswalks.html) + -- county redefinitions since 2000 and how the package handles them +- [Standardizing longitudinal data](https://ui-research.github.io/crosswalk/articles/standardizing-longitudinal-data.html) + -- a worked example building a six-year tract panel from mixed-vintage + data + +## Citations + +Cite the organizations that produce the crosswalks returned by this package: + +**For NHGIS**, see requirements at: https://www.nhgis.org/citation-and-use-nhgis-data + +**For Geocorr**, a suggested citation (update the year): + +> Missouri Census Data Center, University of Missouri. (2022/2018). Geocorr 2022/2018: Geographic Correspondence Engine. Retrieved from: https://mcdc.missouri.edu/applications/geocorr2022/2018.html + +**For CTData**, a suggested citation (adjust for alternate source geography): + +> CT Data Collaborative. (2023). 2022 Census Tract Crosswalk. Retrieved from: https://github.com/CT-Data-Collaborative/2022-tract-crosswalk. + +**For county change events**, the underlying documentation is: + +> U.S. Census Bureau. Substantial Changes to Counties and County Equivalent Entities: 1970-Present. Retrieved from: https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html + +**For this package**, refer here: https://ui-research.github.io/crosswalk/authors.html#citation diff --git a/README.md b/README.md index 6505ad4..0202f43 100644 --- a/README.md +++ b/README.md @@ -1,308 +1,184 @@ + + # crosswalk + + +[![R-CMD-check](https://github.com/UI-Research/crosswalk/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/UI-Research/crosswalk/actions/workflows/R-CMD-check.yaml) + + An R package for translating data across space and time. ## Overview -This package provides a consistent API and standardized versions of -crosswalks to enable consistent approaches that work across different -geography and year combinations. The package also facilitates -interpolation–that is, adjusting source geography/year values by their -crosswalk weights and translating these values to the desired target -geography/year–including diagnostics of the joins between source data -and crosswalks. - -The package sources crosswalks from: - -- **Geocorr** (Missouri Census Data Center) - for inter-geography - crosswalks (same-decade) -- **IPUMS NHGIS** - for inter-temporal crosswalks (across decades) -- **CT Data Collaborative** - for Connecticut 2020→2022 crosswalks - (planning region changes) - -## Why Use `crosswalk`? - -- **Programmatic access**: No more manual downloads from web interfaces; - data is cached for speed -- **Standardized output**: Consistent column names across all crosswalk - sources -- **Metadata tracking**: Full provenance of crosswalks stored as - attributes -- **Crosswalk chaining**: Automatic chaining when multiple crosswalks - are required +Census geographies change across time (tracts are redrawn every decade; +counties are occasionally renamed, merged, or split), and many analyses +need to move data between geographies that don’t nest within one another +(ZCTAs to PUMAs, tracts to places). `crosswalk` provides a consistent +interface for fetching the crosswalks that relate these geographies and +for applying them to your data – adjusting source values by crosswalk +weights to produce estimates for the target geography and year, with +diagnostics describing the quality of the join between your data and the +crosswalk. + +Crosswalks are sourced from: + +- **Geocorr** (Missouri Census Data Center) – same-year crosswalks + between geographies (Geocorr 2022 for 2020s geography; Geocorr 2018 + for 2010s geography) +- **IPUMS NHGIS** – inter-temporal (cross-decade) crosswalks +- **CT Data Collaborative** – Connecticut’s 2020-2022 change from + counties to planning regions +- **A curated registry of county change events** (ships with the + package) – county -\> county crosswalks between any pair of years from + 2000 onward + +## Why use `crosswalk`? + +- **Programmatic access**: no manual downloads from web interfaces, with + optional local caching +- **Standardized output**: consistent column names and structure across + all crosswalk sources +- **Provenance and diagnostics**: full crosswalk metadata and + join-quality statistics attached to every result +- **Crosswalk chaining**: transformations that change both geography and + year are planned and applied automatically ## Installation ``` r -# Install from GitHub +# install.packages("pak") +pak::pak("UI-Research/crosswalk") + +## or, in renv-managed projects renv::install("UI-Research/crosswalk") ``` -## Quick Start +## Quick start -First we obtain a crosswalk and apply it to our data: +Translate ZCTA-level poverty counts to PUMAs. `crosswalk_data()` fetches +the needed crosswalk (here from Geocorr, population-weighted) and +applies it; the `count_` prefix tells it to treat the variable as a +count (multiply by the allocation factor, then sum by target geography): ``` r library(crosswalk) library(dplyr) -library(ggplot2) -library(stringr) -library(sf) -library(tidycensus) -library(tigris) -library(scales) - -source_data = get_acs( + +zcta_poverty <- tidycensus::get_acs( year = 2023, geography = "zcta", output = "wide", - variables = c(below_poverty_level = "B17001_002")) %>% + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> select( source_geoid = GEOID, - count_below_poverty_level = below_poverty_levelE) + count_below_poverty = below_povertyE) -# Get a crosswalk from ZCTAs to PUMAs (same year, uses Geocorr (2022)) -zcta_puma_crosswalk <- get_crosswalk( +puma_poverty <- crosswalk_data( + data = zcta_poverty, source_geography = "zcta", - target_geography = "puma22", + target_geography = "puma", weight = "population") -# Apply the crosswalk to your data -crosswalked_data <- crosswalk_data( - data = source_data, - crosswalk = zcta_puma_crosswalk) - -## Or in a single step -crosswalked_data = crosswalk_data( - data = source_data, - source_geography = "zcta", - target_geography = "puma22", - weight = "population") +head(puma_poverty) +#> # A tibble: 6 × 3 +#> geoid geography_name count_below_poverty +#> +#> 1 0100100 Lauderdale, Colbert & Franklin Counties 27907. +#> 2 0100200 Limestone County 10927. +#> 3 0100300 Morgan & Lawrence Counties--Decatur City 21102. +#> 4 0100401 Madison County (North & East)--Huntsville City (E… 8940. +#> 5 0100402 Huntsville (North & Far West), Madison (East) & T… 17422. +#> 6 0100403 Huntsville City (Central & South) 12836. ``` -What does the crosswalk(s) reflect and how was it sourced? +To inspect or reuse a crosswalk, fetch it explicitly with +`get_crosswalk()` and pass it to `crosswalk_data()`: ``` r -## and there's more (not shown) -names(attr(crosswalked_data, "crosswalk_metadata")) %>% head() -#> [1] "call_parameters" "data_source" "data_source_full_name" -#> [4] "download_url" "api_endpoint" "documentation_url" +zcta_to_puma <- get_crosswalk( + source_geography = "zcta", + target_geography = "puma", + weight = "population") + +puma_poverty <- crosswalk_data( + data = zcta_poverty, + crosswalk = zcta_to_puma) ``` -How well did the crosswalk join to our source data? +Every result carries its provenance and join diagnostics as attributes: ``` r -## look at all the characteristics of the join(s) between the source data -## and the crosswalks -join_quality = attr(crosswalked_data, "join_quality") +## where did the crosswalk come from? +attr(puma_poverty, "crosswalk_metadata")$data_source_full_name +#> [1] "Geocorr 2022 (Missouri Census Data Center)" -## what share of records in the source data do not join to a crosswalk and -## thus are dropped during the crosswalking process? -join_quality$pct_data_unmatched +## what share of data GEOIDs failed to match the crosswalk (and were dropped)? +attr(puma_poverty, "join_quality")$pct_data_unmatched #> [1] 0.4234277 - -## zctas aren't nested within states, otherwise join_quality$state_analysis_data -## would help us to ID whether non-joining source data were clustered within one -## or a few states. instead we can join to spatial data to diagnose further: -zctas_sf = zctas(year = 2023, progress_bar = FALSE) -states_sf = states(year = 2023, cb = TRUE, progress_bar = FALSE) - -## apart from DC, which has a disproportionate number of non-joining ZCTAs-- -## seemingly corresponding to federal areas and buildings--the distribution of -## non-joining ZCTAs appears proportionate to state-level populations and is -## distributed across many states: -zctas_sf %>% - filter(GEOID20 %in% join_quality$data_geoids_unmatched) %>% - st_intersection(states_sf %>% select(NAME)) %>% - st_drop_geometry() %>% - count(NAME, sort = TRUE) %>% - head() -#> NAME n -#> 1 District of Columbia 19 -#> 2 New York 15 -#> 3 Texas 9 -#> 4 California 8 -#> 5 Colorado 6 -#> 6 Utah 6 -``` - -And how accurate was the crosswalking process? - -``` r -comparison_data = get_acs( - year = 2023, - geography = "puma", - output = "wide", - variables = c( - below_poverty_level = "B17001_002")) %>% - select( - source_geoid = GEOID, - count_below_poverty_level_acs = below_poverty_levelE) - -combined_data = left_join( - comparison_data, - crosswalked_data, - by = c("source_geoid" = "geoid")) - -combined_data %>% - select(source_geoid, matches("count")) %>% - mutate(difference_percent = (count_below_poverty_level_acs - count_below_poverty_level) / count_below_poverty_level_acs) %>% - ggplot() + - geom_histogram(aes(x = difference_percent)) + - theme_minimal() + - theme(panel.grid = element_blank()) + - scale_x_continuous(labels = percent) + - labs( - title = "Crosswalked data approximates observed values", - subtitle = "Block group-level source data would produce more accurate crosswalked values", - y = "", - x = "Percent difference between observed and crosswalked values") ``` - - -## Core Functions - -The package has two main functions, though you can also specify the -needed crosswalk(s) directly from `crosswalk_data()` and omit the -intermediate `get_crosswalk()` call. +## Core functions | Function | Purpose | |----|----| -| `get_crosswalk()` | Fetch crosswalk(s) | -| `crosswalk_data()` | Apply crosswalk(s) to interpolate data to the target geography-year | - -## Output Structure - -`get_crosswalk()` **always returns a list** structured as follows: - -The list contains three elements: - -| Element | Description | -|--------------|-------------------------------------------------------| -| `crosswalks` | A named list of crosswalks (`step_1`, `step_2`, etc.) | -| `plan` | Details about what crosswalks are being fetched | -| `message` | A description of the crosswalk chain | - -### Multi-Step Crosswalks - -For some source year/geography -\> target year/geography combinations, -there is not a single direct crosswalk. The package automatically plans -and fetches the required chain of crosswalks, using a year-first -strategy: - -1. **NHGIS step(s)**: Change year while keeping geography constant - (multiple hops if the temporal span requires it, e.g. 1990→2010→2020) -2. **Geocorr step**: Change geography at the target year +| `get_crosswalk()` | Fetch crosswalk(s), chaining steps automatically when geography and year both change | +| `crosswalk_data()` | Apply crosswalk(s) to a dataset, interpolating count and non-count variables | +| `get_available_crosswalks()` | List every supported geography/year combination and its source | +| `list_nhgis_crosswalks()` | List the NHGIS (cross-decade) crosswalks specifically | ``` r -result <- get_crosswalk( - source_geography = "tract", - target_geography = "zcta", - source_year = 2010, - target_year = 2020, - weight = "population", - silent = TRUE) - -# Two crosswalks are returned -# Step 1: 2010 tracts -> 2020 tracts (NHGIS) -# Step 2: 2020 tracts -> 2020 ZCTAs (Geocorr) - -# Longer chains are produced when needed, e.g. -# 2000 tracts -> 2020 ZCTAs produces three steps: -# Step 1: 2000 tracts -> 2010 tracts (NHGIS) -# Step 2: 2010 tracts -> 2020 tracts (NHGIS) -# Step 3: 2020 tracts -> 2020 ZCTAs (Geocorr) -``` - -### Crosswalk Structure - -Each crosswalk contains standardized columns: - -| Column | Description | -|----|----| -| `source_geoid` | Identifier for source geography | -| `target_geoid` | Identifier for target geography | -| `allocation_factor_source_to_target` | Weight for interpolating values | -| `weighting_factor` | What attribute was used (population, housing, land) | - -Additional columns may include `source_year`, `target_year`, -`population_2020`, `housing_2020`, and `land_area_sqmi` depending on the -source of the crosswalk. - -### Accessing Metadata - -Each crosswalk tibble has a `crosswalk_metadata` attribute that -documents what the crosswalk represents and how it was created: - -``` r -metadata <- attr(result$crosswalks$step_1, "crosswalk_metadata") -names(metadata) -``` - -## Interpolation - -`crosswalk_data()` applies crosswalk weights to transform your data. If -you’re in a hurry, you can omit a call to `get_crosswalk()` and specify -the needed crosswalk parameters to `crosswalk_data()`, which will pass -these to `get_crosswalk()` behind the scenes. Or you can call -`get_crosswalk()` explicitly and then pass the result to -`crosswalk_data()`. - -### Column Naming Convention - -The function auto-detects columns based on prefixes: - -| Prefix | Treatment | -|----|----| -| `count_` | Summed after weighting (for counts like population, housing units) | -| `mean_`, `median_`, `percent_`, `ratio_` | Weighted mean (for rates, percentages, averages) | - -You can also specify columns explicitly via `count_columns` and -`non_count_columns`. All non-count variables are interpolated using -weighted means, weighting by the allocation factor from the crosswalk. - -## Supported Geography and Year Combinations - -`get_available_crosswalks()` returns a listing of all supported -year-geography combinations. - -``` r -get_available_crosswalks() %>% +get_available_crosswalks() |> head() -#> # A tibble: 6 × 4 -#> source_geography target_geography source_year target_year -#> -#> 1 block block 1990 2010 -#> 2 block block 2000 2010 -#> 3 block block 2010 2020 -#> 4 block block 2020 2010 -#> 5 block block 2020 2022 -#> 6 block block 2022 2020 +#> # A tibble: 6 × 5 +#> source_geography target_geography source_year target_year crosswalk_source +#> +#> 1 block block 1990 2010 nhgis +#> 2 block block 2000 2010 nhgis +#> 3 block block 2010 2011 county_events +#> 4 block block 2010 2012 county_events +#> 5 block block 2010 2013 county_events +#> 6 block block 2010 2014 county_events ``` -## API Keys +## API keys -NHGIS crosswalks require an IPUMS API key. Get one at - and add to your `.Renviron`: +- **IPUMS_API_KEY** – required for NHGIS (cross-decade) crosswalks. Get + one at . +- **CENSUS_API_KEY** – required for county-level 2020 \<-\> 2022 + requests (used by `tidycensus`, which also powers the examples above). + Get one at . -``` r -usethis::edit_r_environ() -# Add: IPUMS_API_KEY=your_key_here -``` +Store keys in your `.Renviron` (e.g., via `usethis::edit_r_environ()`). +Geocorr and county-events crosswalks require no keys. ## Caching -Use the `cache` parameter to save crosswalks locally for ease: - -``` r -result <- get_crosswalk( - source_geography = "tract", - target_geography = "zcta", - weight = "population", - cache = here::here("crosswalks-cache")) -``` +Pass a directory as the `cache` argument to `get_crosswalk()` or +`crosswalk_data()` and each fetched crosswalk is saved there as a CSV; +later calls with the same parameters read from disk instead of +re-downloading. Cached files never expire – delete a file to force a +re-download. + +## Learn more + +- [Getting + started](https://ui-research.github.io/crosswalk/articles/crosswalk.html) + – the core workflow: fetching, applying, multi-step chains, metadata, + and join quality +- [How interpolation + works](https://ui-research.github.io/crosswalk/articles/how-interpolation-works.html) + – allocation factors, choosing a `weight`, accuracy, and diagnosing + joins +- [County crosswalks between any + years](https://ui-research.github.io/crosswalk/articles/county-crosswalks.html) + – county redefinitions since 2000 and how the package handles them +- [Standardizing longitudinal + data](https://ui-research.github.io/crosswalk/articles/standardizing-longitudinal-data.html) + – a worked example building a six-year tract panel from mixed-vintage + data ## Citations @@ -324,5 +200,11 @@ geography): > CT Data Collaborative. (2023). 2022 Census Tract Crosswalk. Retrieved > from: . +**For county change events**, the underlying documentation is: + +> U.S. Census Bureau. Substantial Changes to Counties and County +> Equivalent Entities: 1970-Present. Retrieved from: +> + **For this package**, refer here: diff --git a/_pkgdown.yml b/_pkgdown.yml index b49f5d3..305744f 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,6 +1,26 @@ -url: https://ui-research.github.io/crosswalk/ -template: - bootstrap: 5 -home: - sidebar: - structure: [toc, links, license, community, citation, authors, dev] +url: https://ui-research.github.io/crosswalk/ +template: + bootstrap: 5 +home: + sidebar: + structure: [toc, links, license, community, citation, authors, dev] +reference: +- title: Fetch crosswalks + desc: Retrieve standardized crosswalks, chaining sources when needed + contents: + - get_crosswalk +- title: Apply crosswalks to data + desc: Interpolate count and non-count variables to the target geography + contents: + - crosswalk_data +- title: Discover supported combinations + contents: + - get_available_crosswalks + - list_nhgis_crosswalks +articles: +- title: Guides + navbar: ~ + contents: + - how-interpolation-works + - county-crosswalks + - standardizing-longitudinal-data diff --git a/data-raw/README.md b/data-raw/README.md new file mode 100644 index 0000000..0af6969 --- /dev/null +++ b/data-raw/README.md @@ -0,0 +1,74 @@ +# data-raw + +Scripts that build the package's internal data (`R/sysdata.rda`). This +directory is excluded from the built package via `.Rbuildignore`. + +## build_county_events_sysdata.R + +Builds the curated county-change ("county events") data backing +`get_county_events_crosswalk()`: + +- `county_events` -- one row per substantial change to counties or county + equivalents since 2000 (renames, merges, splits, part transfers) +- `county_event_mappings` -- county-level source-target mappings with + allocation factors (factors sum to 1 per source county within an event) +- `county_event_subcounty_mappings` -- explicit tract/block-group/block + mappings for events flagged `explicit_rows` (currently empty: every curated + event is a county-prefix relabel, has no sub-county code changes, delegates + to CT Data at runtime, or predates sub-county support; the table is the + escape hatch for future events that genuinely renumber sub-county GEOIDs) +- `county_universes` -- full county GEOID inventories at the 2000/2010/2020 + decennial bases (50 states + DC + Puerto Rico) +- `county_events_meta` -- `known_through_year`, build timestamp, source URL + +### Rebuild + +```bash +Rscript data-raw/build_county_events_sysdata.R +``` + +Requirements: + +- Network access to census.gov and api.census.gov +- `CENSUS_API_KEY` in the environment (used by tidycensus for decennial + populations, ACS county inventories, and the Connecticut section) +- Dev-only packages: `tidycensus`, `usethis`, `devtools` (not package + Imports; if `renv::snapshot()` picks them up, that is expected for dev + environments) + +Intermediate downloads are cached in `data-raw/cache/` (gitignored). + +### Provenance + +- Event list, effective dates, and part populations: U.S. Census Bureau, + "Changes to Counties and County Equivalent Entities: 1970-Present" + (), + decade pages for the 2000s and 2010s. Published part populations reconcile + exactly with decennial census counts of the source areas; remainders + derived by subtraction are noted per mapping in `factor_derivation`. +- Allocation-factor denominators: Census 2000 / 2010 SF1 county populations + (via tidycensus). +- Valdez-Cordova split factors: Census 2010 block populations apportioned to + 2020 counties via the Census 2010-2020 tabulation block relationship file + (). +- Connecticut planning-region county factors: computed by the package's own + `get_ctdata_crosswalk()` (CT Data Collaborative crosswalk + tidycensus + tract populations), so sysdata and the runtime CT path cannot diverge. +- `effective_vintage_year` is the first Census product vintage (TIGER/ACS + geography year) using the post-change codes. Verified empirically against + ACS 5-year county inventories where those exist (2010s events); for the + 2000s, the January-1 boundary-snapshot convention is applied to the + documented effective dates. Notably, the Valdez-Cordova split (effective + January 2, 2019) first appears in 2020-vintage products. +- The 2007 York County / Newport News, VA territory exchange is modeled as a + one-way transfer of the published NET population (gross flows were not + published). + +### Validation built into the script + +- Allocation factors sum to 1 per source county within each event (1e-9) +- Rolling the 2000 universe forward through the 2000s events reproduces the + Census 2010 FIPS list exactly; likewise 2010 through the 2010s events to + the Census 2020 FIPS list (completeness check on the event set) +- ACS vintage assertions per 2010s event (first vintage using new codes) +- GEOID length and categorical-value checks diff --git a/data-raw/build_county_events_sysdata.R b/data-raw/build_county_events_sysdata.R new file mode 100644 index 0000000..43acb84 --- /dev/null +++ b/data-raw/build_county_events_sysdata.R @@ -0,0 +1,470 @@ +# Build R/sysdata.rda: curated county-change ("county events") data +# +# Rebuild with: Rscript data-raw/build_county_events_sysdata.R +# Requires: CENSUS_API_KEY (tidycensus) and network access to census.gov. +# See data-raw/README.md for provenance details. +# +# Primary source: U.S. Census Bureau, "Changes to Counties and County +# Equivalent Entities: 1970-Present", +# https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html +# (decade pages .../county-changes/2000.html and .../county-changes/2010.html). +# All part populations quoted below are the figures published on those pages. + +library(dplyr) +library(purrr) +library(readr) +library(stringr) +library(tibble) +library(tidyr) + +devtools::load_all(".") + +data_raw_cache <- file.path("data-raw", "cache") +dir.create(data_raw_cache, showWarnings = FALSE, recursive = TRUE) + +# ==== County universes per decennial base ==================================== +# 2010 base: Census 2010 FIPS list (reflects January 1, 2010 boundaries) +# 2020 base: Census 2020 FIPS list (reflects January 1, 2020 boundaries) +# 2000 base: derived by reversing the 2000s events from the 2010 base +# Scope: 50 states + DC + Puerto Rico (matches get_ctdata_crosswalk()'s +# tidycensus universe); island-area territories excluded. + +in_scope_state <- function(state_fips) { + state_fips <= "56" | state_fips == "72" +} + +universe_2010 <- readr::read_csv( + "https://www2.census.gov/geo/docs/reference/codes/files/national_county.txt", + col_names = c("state_abbr", "state_fips", "county_fips", "county_name", "class_fips"), + col_types = readr::cols(.default = readr::col_character())) |> + dplyr::filter(in_scope_state(state_fips)) |> + dplyr::transmute( + decade_base = 2010L, + geoid = stringr::str_c(state_fips, county_fips), + state_fips = state_fips) + +universe_2020 <- readr::read_delim( + "https://www2.census.gov/geo/docs/reference/codes2020/national_county2020.txt", + delim = "|", + col_types = readr::cols(.default = readr::col_character())) |> + dplyr::filter(in_scope_state(STATEFP)) |> + dplyr::transmute( + decade_base = 2020L, + geoid = stringr::str_c(STATEFP, COUNTYFP), + state_fips = STATEFP) + +# Counties created during the 2000s (absent from the 2000 base) and counties +# that existed in 2000 but were dissolved/renamed before 2010 +created_in_2000s <- c("02105", "02195", "02198", "02230", "02275", "08014") +existed_in_2000_only <- tibble::tibble( + decade_base = 2000L, + geoid = c("02201", "02232", "02280", "51560"), + state_fips = c("02", "02", "02", "51")) + +universe_2000 <- universe_2010 |> + dplyr::filter(!geoid %in% created_in_2000s) |> + dplyr::mutate(decade_base = 2000L) |> + dplyr::bind_rows(existed_in_2000_only) |> + dplyr::arrange(geoid) + +county_universes <- dplyr::bind_rows(universe_2000, universe_2010, universe_2020) + +# ==== Decennial base populations for allocation-factor denominators ========== + +pops_2000 <- tidycensus::get_decennial( + geography = "county", variables = "P001001", year = 2000, sumfile = "sf1", + state = c("02", "08", "51")) |> + dplyr::select(geoid = GEOID, pop = value) |> + tibble::deframe() + +pops_2010 <- tidycensus::get_decennial( + geography = "county", variables = "P001001", year = 2010, sumfile = "sf1", + state = c("02", "46", "51")) |> + dplyr::select(geoid = GEOID, pop = value) |> + tibble::deframe() + +# ==== Event registry ========================================================== +# effective_vintage_year is the first Census product vintage (TIGER/ACS +# geography year) whose products use the post-change codes/boundaries. +# Convention: boundary changes appear in the vintage of the first January 1 +# on/after the effective date; the 2015 FIPS code corrections and the 2019 +# Valdez-Cordova split were applied by the Census Bureau to that same year's +# products (verified empirically against ACS 5-year county lists below). + +county_events <- tibble::tribble( + ~event_id, ~effective_vintage_year, ~decade_base, ~event_type, ~state_fips, + ~subcounty_handling, ~reversal_exact_tract, ~reversal_exact_block_group, ~reversal_exact_block, + ~description, ~citation_url, + + "2002_broomfield_co_created", 2002L, 2000L, "part_transfer", "08", + "unsupported", FALSE, FALSE, FALSE, + "Broomfield County, CO (08014) created from parts of Adams (08001), Boulder (08013), Jefferson (08059), and Weld (08123) counties, effective November 15, 2001", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2000.html", + + "2002_clifton_forge_va_merge", 2002L, 2000L, "merge", "51", + "unsupported", FALSE, FALSE, FALSE, + "Clifton Forge independent city, VA (51560) changed to town status and added to Alleghany County (51005), effective July 1, 2001", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2000.html", + + "2008_skagway_hoonah_angoon_split", 2008L, 2000L, "split", "02", + "unsupported", FALSE, FALSE, FALSE, + "Skagway-Hoonah-Angoon Census Area, AK (02232) split into Skagway Municipality (02230) and Hoonah-Angoon Census Area (02105), effective June 20, 2007", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2000.html", + + "2008_york_newport_news_exchange", 2008L, 2000L, "part_transfer", "51", + "unsupported", FALSE, FALSE, FALSE, + "York County, VA (51199) exchanged territory with Newport News city (51700), effective July 1, 2007; modeled as a one-way transfer of the published net population", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2000.html", + + "2009_ak_southeast_reorg", 2009L, 2000L, "split", "02", + "unsupported", FALSE, FALSE, FALSE, + "Wrangell-Petersburg Census Area, AK (02280) split into Wrangell City and Borough (02275) and Petersburg Census Area (02195); Prince of Wales-Outer Ketchikan Census Area (02201) reorganized into Prince of Wales-Hyder Census Area (02198) with parts to Ketchikan Gateway Borough (02130) and Wrangell (02275), effective May 19 / June 1, 2008", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2000.html", + + "2014_bedford_va_merge", 2014L, 2010L, "merge", "51", + "prefix_swap", FALSE, FALSE, FALSE, + "Bedford independent city, VA (51515) changed to town status and added to Bedford County (51019), effective July 1, 2013; census tract 050100 retained its code under the new county (verified against ACS tract inventories), so sub-county GEOIDs are county-prefix relabels. Forward-only at sub-county levels (reversing would require the affected tract list)", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2010.html", + + "2014_petersburg_borough_reorg", 2014L, 2010L, "part_transfer", "02", + "none", FALSE, FALSE, FALSE, + "Petersburg Borough, AK (02195) created from most of Petersburg Census Area (02195) plus a small part of Hoonah-Angoon Census Area (02105); the remainder of the former census area transferred to Prince of Wales-Hyder Census Area (02198), effective January 3, 2013. Census tract sets and codes are unchanged across this event (verified against ACS tract inventories); tract boundaries were revised to follow the new county line, so sub-county crosswalks treat it as identity with a population-comparability caveat", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2010.html", + + "2015_shannon_oglala_lakota_rename", 2015L, 2010L, "rename", "46", + "prefix_swap", TRUE, TRUE, TRUE, + "Shannon County, SD (46113) renamed Oglala Lakota County (46102), effective May 1, 2015; boundaries unchanged", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2010.html", + + "2015_wade_hampton_kusilvak_rename", 2015L, 2010L, "rename", "02", + "prefix_swap", TRUE, TRUE, TRUE, + "Wade Hampton Census Area, AK (02270) renamed Kusilvak Census Area (02158), effective July 1, 2015; boundaries unchanged", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2010.html", + + "2020_valdez_cordova_split", 2020L, 2010L, "split", "02", + "none", FALSE, FALSE, FALSE, + "Valdez-Cordova Census Area, AK (02261) split into Chugach Census Area (02063) and Copper River Census Area (02066), effective January 2, 2019; first reflected in 2020-vintage Census products (verified against ACS county inventories), so sub-county transitions are handled by NHGIS 2010-2020 crosswalks", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes/2010.html", + + "2022_ct_planning_regions", 2022L, 2020L, "split", "09", + "ctdata_runtime", TRUE, TRUE, TRUE, + "Connecticut's eight counties (09001-09015) replaced by nine planning regions (09110-09190) as county equivalents, effective in 2022 Census products; sub-county geographies are exact 1:1 FIPS relabels (reversible), sourced from CT Data Collaborative at runtime", + "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html") + +# ==== County-level event mappings ============================================ +# One row per source-target county pair per event; allocation factors sum to 1 +# per source county within an event. Part populations quoted from the Census +# county-changes pages; denominators are the nearest preceding decennial +# census counts (the published parts reconcile exactly with those counts). + +frac <- function(part, geoid, pops) part / unname(pops[geoid]) + +county_event_mappings <- dplyr::bind_rows( + + # Broomfield: published detached populations by donor county + # Adams 15,870; Boulder 21,512; Jefferson 1,726; Weld 69 (sum = 39,177) + purrr::map_dfr( + list(c("08001", 15870), c("08013", 21512), c("08059", 1726), c("08123", 69)), + \(x) { + donor <- x[1] + part <- as.numeric(x[2]) + tibble::tibble( + event_id = "2002_broomfield_co_created", + source_geoid = donor, + target_geoid = c("08014", donor), + allocation_factor = c(frac(part, donor, pops_2000), 1 - frac(part, donor, pops_2000)), + weighting_factor = "population", + factor_derivation = "Census-published detached population / Census 2000 donor county population") + }), + + tibble::tibble( + event_id = "2002_clifton_forge_va_merge", + source_geoid = "51560", + target_geoid = "51005", + allocation_factor = 1, + weighting_factor = "identity", + factor_derivation = "Entire independent city merged into Alleghany County"), + + # Skagway-Hoonah-Angoon: Skagway Municipality 862; Hoonah-Angoon 2,574 + # (sum = 3,436 = Census 2000 population of 02232) + tibble::tibble( + event_id = "2008_skagway_hoonah_angoon_split", + source_geoid = "02232", + target_geoid = c("02230", "02105"), + allocation_factor = c(frac(862, "02232", pops_2000), frac(2574, "02232", pops_2000)), + weighting_factor = "population", + factor_derivation = "Census-published part populations / Census 2000 population of 02232"), + + # York/Newport News: only the NET exchanged population (293) is published; + # modeled as a one-way York -> Newport News transfer + tibble::tibble( + event_id = "2008_york_newport_news_exchange", + source_geoid = "51199", + target_geoid = c("51700", "51199"), + allocation_factor = c(frac(293, "51199", pops_2000), 1 - frac(293, "51199", pops_2000)), + weighting_factor = "population", + factor_derivation = "Census-published NET exchanged population / Census 2000 population of York County (approximation: gross flows not published)"), + + # Southeast Alaska 2008 reorganization: + # 02280 (Census 2000 pop 6,684): Petersburg CA 4,260 (published); remainder + # 2,424 to Wrangell (derived: 6,684 - 4,260) + # 02201 (Census 2000 pop 6,146): Prince of Wales-Hyder 6,115 (published); + # Ketchikan Gateway 7 (published); remainder 24 (Meyers Chuck area, + # derived: 6,146 - 6,115 - 7) to Wrangell + tibble::tibble( + event_id = "2009_ak_southeast_reorg", + source_geoid = c("02280", "02280", "02201", "02201", "02201"), + target_geoid = c("02195", "02275", "02198", "02130", "02275"), + allocation_factor = c( + frac(4260, "02280", pops_2000), frac(unname(pops_2000["02280"]) - 4260, "02280", pops_2000), + frac(6115, "02201", pops_2000), frac(7, "02201", pops_2000), + frac(unname(pops_2000["02201"]) - 6115 - 7, "02201", pops_2000)), + weighting_factor = "population", + factor_derivation = "Census-published part populations (remainders derived by subtraction) / Census 2000 source populations"), + + tibble::tibble( + event_id = "2014_bedford_va_merge", + source_geoid = "51515", + target_geoid = "51019", + allocation_factor = 1, + weighting_factor = "identity", + factor_derivation = "Entire independent city merged into Bedford County"), + + # Petersburg Borough 2013: 613 (published) from Petersburg CA to Prince of + # Wales-Hyder; 1 (published) from Hoonah-Angoon to Petersburg Borough; + # denominators are Census 2010 counts (02195 = 3,815; 02105 = 2,150) + tibble::tibble( + event_id = "2014_petersburg_borough_reorg", + source_geoid = c("02195", "02195", "02105", "02105"), + target_geoid = c("02195", "02198", "02195", "02105"), + allocation_factor = c( + 1 - frac(613, "02195", pops_2010), frac(613, "02195", pops_2010), + frac(1, "02105", pops_2010), 1 - frac(1, "02105", pops_2010)), + weighting_factor = "population", + factor_derivation = "Census-published part populations / Census 2010 source populations"), + + tibble::tibble( + event_id = "2015_shannon_oglala_lakota_rename", + source_geoid = "46113", + target_geoid = "46102", + allocation_factor = 1, + weighting_factor = "identity", + factor_derivation = "FIPS code change only; boundaries unchanged"), + + tibble::tibble( + event_id = "2015_wade_hampton_kusilvak_rename", + source_geoid = "02270", + target_geoid = "02158", + allocation_factor = 1, + weighting_factor = "identity", + factor_derivation = "FIPS code change only; boundaries unchanged")) + +# ---- Valdez-Cordova split: factors from Census 2010 block populations ------- +# 2010 blocks of 02261 are assigned to their 2020 counties via the official +# 2010-2020 tabulation block relationship file (land-area apportionment for +# the rare split blocks), then weighted by Census 2010 block populations. + +vc_blocks_2010 <- tidycensus::get_decennial( + geography = "block", variables = "P001001", year = 2010, sumfile = "sf1", + state = "02", county = "261") |> + dplyr::transmute(block_2010 = GEOID, pop = value) + +block_rel_zip <- file.path(data_raw_cache, "tab2010_tab2020_st02.zip") +if (!file.exists(block_rel_zip)) { + download.file( + "https://www2.census.gov/geo/docs/maps-data/data/rel2020/t10t20/TAB2010_TAB2020_ST02.zip", + block_rel_zip, mode = "wb", quiet = TRUE) +} + +vc_block_assignment <- readr::read_delim( + block_rel_zip, delim = "|", + col_types = readr::cols(.default = readr::col_character())) |> + dplyr::filter(STATE_2010 == "02", COUNTY_2010 == "261") |> + dplyr::mutate( + block_2010 = stringr::str_c(STATE_2010, COUNTY_2010, TRACT_2010, BLK_2010), + county_2020 = stringr::str_c(STATE_2020, COUNTY_2020), + arealand_int = as.numeric(AREALAND_INT)) |> + dplyr::mutate( + block_land_total = sum(arealand_int), + area_share = dplyr::if_else( + block_land_total > 0, arealand_int / block_land_total, 1 / dplyr::n()), + .by = block_2010) |> + dplyr::summarize(share = sum(area_share), .by = c(block_2010, county_2020)) + +valdez_parts <- vc_block_assignment |> + dplyr::inner_join(vc_blocks_2010, by = "block_2010") |> + dplyr::summarize(part_pop = sum(pop * share), .by = county_2020) + +# Population leaking to counties other than the two successors (from minor +# boundary corrections in the relationship file) must be negligible +leaked_pop <- valdez_parts |> + dplyr::filter(!county_2020 %in% c("02063", "02066")) |> + dplyr::pull(part_pop) |> + sum() +stopifnot(leaked_pop / sum(valdez_parts$part_pop) < 0.001) + +valdez_cordova_mappings <- valdez_parts |> + dplyr::filter(county_2020 %in% c("02063", "02066")) |> + dplyr::transmute( + event_id = "2020_valdez_cordova_split", + source_geoid = "02261", + target_geoid = county_2020, + allocation_factor = part_pop / sum(part_pop), + weighting_factor = "population", + factor_derivation = "Census 2010 block populations apportioned to 2020 counties via the Census 2010-2020 tabulation block relationship file") + +# Sanity check against Census 2020 counts (Chugach 7,102 / Copper River 2,617) +stopifnot(abs( + valdez_cordova_mappings$allocation_factor[valdez_cordova_mappings$target_geoid == "02063"] - + 7102 / (7102 + 2617)) < 0.05) + +# ---- Connecticut planning regions: reuse the package's CTData computation --- +# Guarantees the sysdata factors match get_ctdata_crosswalk("county", 2020, 2022). + +ct_county_crosswalk <- get_ctdata_crosswalk( + geography = "county", source_year = 2020, target_year = 2022, + cache = data_raw_cache) + +ct_mappings <- ct_county_crosswalk |> + dplyr::filter(stringr::str_starts(source_geoid, "09")) |> + dplyr::transmute( + event_id = "2022_ct_planning_regions", + source_geoid = source_geoid, + target_geoid = target_geoid, + allocation_factor = allocation_factor_source_to_target, + weighting_factor = weighting_factor, + factor_derivation = "Population-weighted tract aggregation from CT Data Collaborative crosswalk (see get_ctdata_crosswalk())") + +county_event_mappings <- dplyr::bind_rows( + county_event_mappings, valdez_cordova_mappings, ct_mappings) |> + dplyr::arrange(event_id, source_geoid, target_geoid) + +# ==== Sub-county mappings (explicit_rows events) ============================== +# Populated in a follow-up curation pass; empty schema for now. + +county_event_subcounty_mappings <- tibble::tibble( + event_id = character(), + geography = character(), + source_geoid = character(), + target_geoid = character(), + allocation_factor = double()) + +# ==== Empirical vintage verification (ACS 5-year county inventories) ========= +# Asserts that effective_vintage_year is the first ACS vintage using the +# post-change codes for every 2010s event. + +acs_counties <- function(year, states) { + tidycensus::get_acs( + geography = "county", variables = "B01001_001", year = year, + survey = "acs5", state = states) |> + dplyr::select(GEOID, NAME) +} + +acs13 <- acs_counties(2013, c("VA", "AK")) +acs14 <- acs_counties(2014, c("VA", "AK", "SD")) +acs15 <- acs_counties(2015, c("AK", "SD")) +acs19 <- acs_counties(2019, "AK") +acs20 <- acs_counties(2020, "AK") + +stopifnot( + # Bedford city present through vintage 2013, gone in 2014 + "51515" %in% acs13$GEOID, !"51515" %in% acs14$GEOID, + # Petersburg: Census Area through 2013, Borough from 2014 + any(stringr::str_detect(acs13$NAME, "Petersburg Census Area")), + any(stringr::str_detect(acs14$NAME, "Petersburg Borough")), + # Shannon/Wade Hampton codes through vintage 2014, new codes from 2015 + "46113" %in% acs14$GEOID, !"46113" %in% acs15$GEOID, "46102" %in% acs15$GEOID, + "02270" %in% acs14$GEOID, !"02270" %in% acs15$GEOID, "02158" %in% acs15$GEOID, + # Valdez-Cordova through vintage 2019, Chugach/Copper River from 2020 + "02261" %in% acs19$GEOID, !any(c("02063", "02066") %in% acs19$GEOID), + !"02261" %in% acs20$GEOID, all(c("02063", "02066") %in% acs20$GEOID)) + +# ==== Metadata ================================================================ + +county_events_meta <- list( + known_through_year = 2025L, + built_at = as.character(Sys.time()), + source_documentation_url = "https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html") + +# ==== Validation ============================================================== + +# Factors sum to 1 per source county within each event +factor_sums <- county_event_mappings |> + dplyr::summarize(total = sum(allocation_factor), .by = c(event_id, source_geoid)) +stopifnot(all(abs(factor_sums$total - 1) < 1e-9)) + +# GEOIDs are 5-character county codes +stopifnot( + all(nchar(county_event_mappings$source_geoid) == 5), + all(nchar(county_event_mappings$target_geoid) == 5), + all(nchar(county_universes$geoid) == 5)) + +# Every event has at least one mapping row (CT included via ctdata) +stopifnot(setequal(county_events$event_id, county_event_mappings$event_id)) + +# Rolling each decade's universe forward through its events reproduces the +# next decennial base exactly +roll_universe <- function(universe, mappings) { + sources <- unique(mappings$source_geoid) + targets <- mappings |> + dplyr::distinct(target_geoid) |> + dplyr::transmute( + geoid = target_geoid, + state_fips = stringr::str_sub(target_geoid, 1, 2)) + universe |> + dplyr::select(geoid, state_fips) |> + dplyr::filter(!geoid %in% sources) |> + dplyr::bind_rows(targets) |> + dplyr::distinct(geoid, state_fips) |> + dplyr::arrange(geoid) +} + +events_2000s <- county_events |> dplyr::filter(decade_base == 2000L) +events_2010s <- county_events |> dplyr::filter(decade_base == 2010L) + +rolled_2010 <- roll_universe( + universe_2000, + county_event_mappings |> dplyr::filter(event_id %in% events_2000s$event_id)) +stopifnot(identical( + rolled_2010$geoid, + universe_2010 |> dplyr::arrange(geoid) |> dplyr::pull(geoid))) + +rolled_2020 <- roll_universe( + universe_2010, + county_event_mappings |> dplyr::filter(event_id %in% events_2010s$event_id)) +stopifnot(identical( + rolled_2020$geoid, + universe_2020 |> dplyr::arrange(geoid) |> dplyr::pull(geoid))) + +# Universes contain/exclude the right pre/post-event codes +stopifnot( + all(c("02201", "02232", "02280", "51560") %in% universe_2000$geoid), + !any(c("08014", "02105") %in% universe_2000$geoid), + all(c("08014", "51515", "46113", "02270", "02261") %in% universe_2010$geoid), + !any(c("46102", "02158", "02063") %in% universe_2010$geoid), + all(c("46102", "02158", "02063", "02066") %in% universe_2020$geoid), + !any(c("51515", "46113", "02270", "02261") %in% universe_2020$geoid)) + +# Valid categorical values +stopifnot( + all(county_events$event_type %in% c("rename", "merge", "split", "part_transfer")), + all(county_events$subcounty_handling %in% + c("prefix_swap", "explicit_rows", "ctdata_runtime", "none", "unsupported")), + all(county_events$effective_vintage_year > 2000L), + all(county_events$effective_vintage_year <= county_events_meta$known_through_year)) + +# ==== Write sysdata =========================================================== + +usethis::use_data( + county_events, + county_event_mappings, + county_event_subcounty_mappings, + county_universes, + county_events_meta, + internal = TRUE, + overwrite = TRUE) + +message("sysdata.rda written: ", + nrow(county_events), " events, ", + nrow(county_event_mappings), " county mappings, ", + nrow(county_universes), " universe rows") diff --git a/man/crosswalk-package.Rd b/man/crosswalk-package.Rd new file mode 100644 index 0000000..4bc3cdc --- /dev/null +++ b/man/crosswalk-package.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/crosswalk-package.R +\docType{package} +\name{crosswalk-package} +\alias{crosswalk} +\alias{crosswalk-package} +\title{crosswalk: Streamlined Inter-Temporal and Inter-Geography Crosswalking of Census Data} +\description{ +Provides a standardized interface for accessing and applying geographic crosswalks between United States Census geographies, both across geographies (e.g., tracts to ZIP Code Tabulation Areas) and across time (e.g., 2010 tracts to 2020 tracts). Crosswalks are sourced from the Missouri Census Data Center's Geocorr applications, IPUMS NHGIS, the CT Data Collaborative, and a curated registry of county boundary changes drawn from Census Bureau documentation, and are returned with consistent column names and rich metadata. Also interpolates count and non-count variables from source to target geographies, with diagnostics describing join quality. +} +\details{ +The package standardizes access to geographic crosswalks from four sources: +GeoCorr (Missouri Census Data Center) for same-year geography changes, IPUMS +NHGIS for cross-decade changes, the CT Data Collaborative for Connecticut's +2020-2022 county changes, and a curated registry of county boundary change +events for county crosswalks between any pair of years from 2000 onward. + +Typical usage involves two functions: +\itemize{ +\item \code{\link[=get_crosswalk]{get_crosswalk()}} fetches one or more crosswalks (chaining them +automatically when both geography and year change) +\item \code{\link[=crosswalk_data]{crosswalk_data()}} applies crosswalk(s) to a dataset, interpolating count +and non-count variables and reporting join quality +} + +Two helpers support discovery: +\itemize{ +\item \code{\link[=get_available_crosswalks]{get_available_crosswalks()}} lists every supported combination of source +and target geography and year +\item \code{\link[=list_nhgis_crosswalks]{list_nhgis_crosswalks()}} lists the NHGIS subset with source URLs +} + +To get oriented, see \code{vignette("crosswalk")}. For the interpolation +methodology and guidance on choosing weights, see +\code{vignette("how-interpolation-works")}. +} +\seealso{ +Useful links: +\itemize{ + \item \url{https://ui-research.github.io/crosswalk/} + \item \url{https://github.com/UI-Research/crosswalk} + \item Report bugs at \url{https://github.com/UI-Research/crosswalk/issues} +} + +} +\author{ +\strong{Maintainer}: Will Curran-Groome \email{wcurrangroome@urban.org} + +} +\keyword{internal} diff --git a/man/crosswalk_data.Rd b/man/crosswalk_data.Rd index 97d9fcc..33d264a 100644 --- a/man/crosswalk_data.Rd +++ b/man/crosswalk_data.Rd @@ -34,10 +34,10 @@ Alternatively, a single crosswalk tibble can be provided for backwards compatibility. If NULL, the crosswalk will be fetched using \code{source_geography} and \code{target_geography} parameters.} -\item{source_geography}{Character or NULL. Source geography name. Required if -\code{crosswalk} is NULL. One of c("block", "block group", "tract", "place", -"county", "urban_area", "zcta", "puma", "cd118", "cd119", -"core_based_statistical_area").} +\item{source_geography}{Character or NULL. Source geography name (e.g., +"tract", "zcta"). Required if \code{crosswalk} is NULL; passed to +\code{get_crosswalk()}. See \code{get_crosswalk()} for accepted names and +\code{get_available_crosswalks()} for all supported combinations.} \item{target_geography}{Character or NULL. Target geography name. Required if \code{crosswalk} is NULL. Same options as \code{source_geography}.} @@ -74,16 +74,23 @@ from each step. Default is FALSE, which returns only the final result.} about join quality, including the number of data rows not matching the crosswalk and vice versa. For state-nested geographies (tract, county, block group, etc.), also reports state-level concentration of unmatched rows. Set to FALSE to -suppress these messages. Automatically suppressed when \code{silent = TRUE}.} +suppress these messages; the \code{join_quality} attribute (see the "Join +quality diagnostics" section) is computed and attached to the result +either way. Messages are automatically suppressed when \code{silent = TRUE}.} \item{silent}{Logical. If \code{TRUE}, suppresses all informational messages and -warnings, including join quality diagnostics regardless of \code{show_join_quality}. +warnings, including join quality diagnostics regardless of \code{show_join_quality} +(the \code{join_quality} attribute is still attached to the result). Defaults to \code{getOption("crosswalk.silent", FALSE)}. Set \code{options(crosswalk.silent = TRUE)} to silence all calls by default.} } \value{ If \code{return_intermediate = FALSE} (default), a tibble with data summarized -to the final target geography. +to the final target geography. The target identifier column is named +\code{geoid} (and, when present in the crosswalk, the geography type is in +\code{geography_name}). Data rows whose GEOIDs do not match the crosswalk +cannot be allocated to a target geography and are dropped; use the +\code{join_quality} attribute to inspect them. If \code{return_intermediate = TRUE} and there are multiple crosswalk steps, a list with: \describe{ @@ -91,8 +98,18 @@ If \code{return_intermediate = TRUE} and there are multiple crosswalk steps, a l \item{intermediate}{A named list of intermediate results (step_1, step_2, etc.)} } -The returned tibble(s) include an attribute \code{crosswalk_metadata} from the -underlying crosswalk (access via \code{attr(result, "crosswalk_metadata")}). +The returned tibble(s) carry two attributes: +\describe{ +\item{crosswalk_metadata}{Provenance of the crosswalk that produced the +result (access via \code{attr(result, "crosswalk_metadata")}); see the +"Crosswalk metadata" section of \code{\link[=get_crosswalk]{get_crosswalk()}}.} +\item{join_quality}{Statistics describing how well the data joined to +the crosswalk (access via \code{attr(result, "join_quality")}); see the +"Join quality diagnostics" section below. For multi-step crosswalks, +both attributes describe the \emph{final} step; set +\code{return_intermediate = TRUE} to obtain each step's result with its +own attributes.} +} } \description{ Applies geographic crosswalk weights to transform data from a source geography @@ -115,7 +132,15 @@ the product of the value and the allocation factor across all source geographies that overlap with each target geography. \strong{Non-count variables} (specified in \code{non_count_columns}) are interpolated using -a weighted mean, with the allocation factor serving as the weight. +a weighted mean, with the allocation factor serving as the weight. Note this +is an approximation: the allocation factor reflects each source geography's +share allocated to the target, not the relative size of the source +geographies, so weighted means are most accurate when source units are of +broadly similar size. + +\strong{One row per GEOID}: \code{data} must contain at most one row per GEOID. For +panel data with multiple time periods, split the data by period (e.g., with +\code{purrr::map()}) and crosswalk each subset separately. \strong{Automatic column detection}: If \code{count_columns} and \code{non_count_columns} are both NULL, the function will automatically detect columns based on naming prefixes: @@ -132,8 +157,46 @@ are missing, NA is returned. \strong{Multi-step crosswalks}: When \code{get_crosswalk()} returns multiple crosswalks (for transformations that change both geography and year), this function -automatically applies them in sequence. +automatically applies them in sequence. The attributes on the final result +describe only the final step; to inspect join quality for earlier steps, set +\code{return_intermediate = TRUE} and examine the attributes of each intermediate +tibble. } +\section{Join quality diagnostics}{ + +The \code{join_quality} attribute is a list with the following elements: +\describe{ +\item{n_data_total}{Number of unique GEOIDs in the input data.} +\item{n_data_unmatched}{Number of data GEOIDs with no match in the +crosswalk. These rows cannot be allocated to a target geography and are +dropped from the result.} +\item{pct_data_unmatched}{\code{n_data_unmatched} as a percentage (0-100) of +\code{n_data_total}.} +\item{data_geoids_unmatched}{Character vector of the unmatched data GEOIDs.} +\item{state_analysis_data}{For state-nested geographies with unmatched data +rows, a list describing state-level concentration of the unmatched +GEOIDs: \code{state_counts} (a tibble of unmatched counts and percentages by +state FIPS), \code{top_states} (the three most-affected states), and +\code{is_concentrated} (logical; TRUE when any single state accounts for more +than 15\% of unmatched GEOIDs). NULL when there are no unmatched rows or +state analysis is not applicable.} +\item{n_crosswalk_total}{Number of unique source GEOIDs in the crosswalk.} +\item{n_crosswalk_unmatched}{Number of crosswalk source GEOIDs absent from +the data (e.g., geographies with no observations).} +\item{pct_crosswalk_unmatched}{\code{n_crosswalk_unmatched} as a percentage +(0-100) of \code{n_crosswalk_total}.} +\item{crosswalk_geoids_unmatched}{Character vector of the crosswalk source +GEOIDs absent from the data.} +\item{state_analysis_crosswalk}{As \code{state_analysis_data}, but for crosswalk +GEOIDs absent from the data; otherwise NULL.} +\item{source_geography}{The source geography of the crosswalk step, when +known from its metadata.} +\item{state_analysis_applicable}{Logical; whether state-level analysis is +meaningful for this geography (FALSE for geographies that cross state +lines, such as ZCTAs).} +} +} + \examples{ \dontrun{ # Option 1: Pre-fetched crosswalk @@ -188,3 +251,7 @@ result$intermediate$step_1 # After first crosswalk result$final # Final result } } +\seealso{ +\code{\link[=get_crosswalk]{get_crosswalk()}} to fetch and inspect crosswalks before applying +them; \code{\link[=get_available_crosswalks]{get_available_crosswalks()}} for all supported combinations. +} diff --git a/man/figures/README-unnamed-chunk-6-1.png b/man/figures/README-unnamed-chunk-6-1.png deleted file mode 100644 index 0bc7f209195921ca5bfcdad453db8ea0d5732bd4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29010 zcmeFZXH-*N7cLq=KtaR?C`eHRq$9n91rX`I6O|rFP2aK@D-h1u2W?RoQ=X#~Csz5t6@25$?h{v00K?vm3Ih%X;)NSu6+_Sf{chGwJ*vvw~0%74`W2UKa7XlFt z4b?HRzNT|oI-#nR^JaIzU}k2-Ih;^bnVn|ei}6LAMJarOE7Gp)&fE5t$wV7gftkZ} z8+Pf-cg}yw{vP@L*ysz}byyxIggdoNJ@r(;t-am@g^&F-XV6swr8SY2wsoBo`|}^x zd={o0SEtC|R&p7|OtBDh(H3&g4g4%^o&Wk(-06#VFNJq6H)mKhM>Hz|-ArS{IA)Eh zWol%nV)xhzU8t@=@NzP6G+(si#!m|fj;guD#P8G{Y}=WUCwphUO4xm1`gySl3DLWf zM7dtl6Zek)B~*^2dTs4z*P>;m>1opSBMdGV^u19bsd-*UN0 z+pnkqweV6t@5|Ffl{>K<4lzH1&nN55PB^_R;;|`hK7VXcWpieO8DdgX8|Xl`A^p`% z$Fug@qEk+;NZ)79tid>|^V0!t&R--uT2}8LB5y-KkU9HkemFZ%$pHEOlY_h1W0#qs zSjC3pWoM|KwQ%d^%e~5ZYD+@zRc917pL)&G)yl`Kk;(K)I{#zzW8!w|rX<1~ni`E|(Gps-E4Sipk zZe-ubK`OLixHj}+AK%@~W9IeV+{QI)9p>7zKJM0e&b?h(0H2)1@D^Ec*w;t$UWwZ`0VG~cS~8+0dN)=U-&sSR0lz0-oTGJpTQtf%>5vdH<- zp3xrLj_47Dskk>nFQi6ZH@RlmamneX6)lY0scn2P(V^$K5xOu^3|$a0Jc}dsA@W6| zbSs3=g5Pw>ysPbvzA9`DTE=@}{5Yjy$WbA}^SGm=kwjwb)oZW4giLh5zJj zm5o7%rIOBbKi}9SmLcPpUpKYS&O9M2;^&|~JNG%|7P@+|=2~CoIb^K$9HiKjU$oI) z5qCs`a?h}EOePUMWRuu6EEl8vo=?<^weA%5vzn<}gXBwRy6tGXv3-3qO(;4P(DX%5 zjkYI`5xs5XhG3b3jC$3X9%GL@ytjx3ry-Ro$3(|0Zsg5o!=1oEh+60?TB@i(IKb!A z5Q1PE$SLrN0Q}H`AG|Li{4+wBl0x+7Gr@QKKQBU26A*|jMDhL|Z8rkUcxZkdW)F%Mh}B5Qnf38-i_gmBflEs zcjL9)>l-OJlM9Qup5535m*gMa-x94XldO1p7HUm<1O!}H9agaubFMv73#v73Y`|?1 zoH{T2`&UZfsquLnZ9tp4&rx-D$Q?-g!*-hInLhX&EUk{)7QwsI zl>;Dzd7oqx$Yf)=jppljkVRD{u9HJ1PJMEPX02hXbFomDhP`Iq2{xI1#oadM@cb6J zNvEMCU!1#$<+m#pPD`5DCp#-+JMMj{aw-b>eQdP&2x`f&v=Wpn9jY8naOm;$o`UALyA00sNK1*Dk77!xXsuGK`YJ7I}raN z{gkd#DY^THEh6; z0&Y9R;IVjpcRssd>LDq!ma};}I%1bI)15_I z+Kgv%?Z_f|GJlzDs>)LHc4b9)f9jWv8mA;5xs&Ytfm5>3rgE4Y^Za(FxUg$NuY)46 zw}*td`va`~*MqNJbY7d1+GGn%MWcy&+ZZP>SRg$O84X$!=OLY)+}&8HPn9=Vsa?4Q zyOLKFyKD&0BcM~)_GMXA!7^GcSjp9@kap^cE;RIBB3u2i37Zc zz3y>Tgkjej8dTDp`=@iVl3BpavbLS8#2Q6(&R%A)*diYaJts!SSGxr`AT={yfqcU2!|3p zFmMNh5;<70l1XrH|1q2Lxyx1grO$pRR36tMrCFSnqQS=DI|L_nn?`VHJ}RCGlV~AW z6ShNLN&T49NPz0W8XT_dH9tVMloGh_Ddq6P?r{g+-WjztJ9qeJ-#?ciCgHe z44S9CsG?q^A1>X@@@pI!92X2`@UWgupuR6Ii&d5EJ3tpP!mF1uA4j+u*brnF^}XBy zUME{Amgim3(wnS4@xaIzn=7_DscXK+SmAFK$)T+hKVDqoa?^O5e<_9VIx%N_xi%!1 z+;4l>h*$G`;U(k;L7&rZcOmf>^xoW1rrP*}G!{;os0Dam`sfPA*> zY@)JK2G>9kMAbjbe%G=z@>zq=Q7W&`M##IOKICJARxL>Ne7vnZ7E=Lr+UgQfXgEGv zgC8&gGody%b?g!<$Lz&7TQBy0w9ME%*X~+g`uNAs`*RteJ4SHhx0(sniAyUcU5wJ< z_Oos8gegrvvoA)+5#~#KAL#SZ#Y8t}=~tPLm3LsYt}_M}0Y5>UGJdX%p;}+v6q#(J z&(~Ely-cWyO|Tz+G{9cjkNcS73nJgkQoWTcY2-Zgu7V~HI7l%w9BLU{rnE&>*e5ZP z=3|u6!Hs^U>!mG<xI=#&^!I}Zn4o7zk_)Q^~$X_ zpI?0n@T7wUdp4ub##g@D`WULsDJtGcgUKI760bP!EhG?*ZYmL{Fw4Iwwz&2FVWvM3 zsc^gJu(RY+dL$QwVoMWKv6+#=9j6`Rz!P1OKM?e}-_=2hr}4Q0H}iW-dNqRVjvJ?X z<8z*0xXQ+RSJ3ziCdLSH6<&iWKd@HZ9}ZLtXOQChtjo|y943E@+Al~}VP^5}y*qh4 z?<4lt)mV#}^gG*@)Ap%il!o!1JHtIIBOhybS~e0}uI#*V^IirQ7M_jKgZuECxQtnp zCrTZzHwjmc-j=6P;K+J)iLE6P{Y2X$rPg9Q*AmJVOKQP|_R+ucl-g&wmW-%3A3Zt& zzuS59x}PkX%;^_t^^o*_Je}=A%rJG)M2GLeKr({=HRWi2Qws=T8>~&<1&Q9e!RMd> zPcX{f>dRh`$Kf9RGf9I)$KL}e%Qw6z;l@W&yO$&>_8drzS+zk3O*cyN^YBY zr@&wXKRU$NhMaMR zvW~XcY(=~eoJ8zK`F5*%ym=Vp;Q50=I*G``SAqC`+~~f=!A{>n`3)l;YtW^Q^BzXU{<$+F4>`wsE>x$%5GN5{~vv{ah!*XlMR? z@>AIC^3RGRTr>MA{rjjN8!Ot<{5&6qN8ZLoHyqi+qrAIv;%p@*e6Sce-DXAaS{~W<7L-`byQz`Z3ZA(l=mt1qw&k2xS zIim8AjBn_=(lA0Nl%kgOp(z$O{7Gt9u?`)5`66+Ue=WQlf@o#XA`dIxznM0@7=bI(35m z*1Xp|T~c*PlcwrCC8h;G@3Wp(?b*}*uE~+3)me{Swe9>Rl=DpbE-AaSb-a-6Fh{BS zqkM=e{h6rC#Wy3xYud84`^)O%K69j5(?{gQI}gr03X@nBML*5t?=_3R%ctI*8DT@d zwPMQzr+RLtpUgFr0S|~br6CGbyz{XrwCtRIz2$iQ@J>EvdZgCCogmHO{E~CdvR>d}WD&Vq2}pi6IO-aOzStJk zSqk0By>mtp>4y`Jm$KDDzR#WeRpFZm6Nu8z%Sh28NMs<+_^e$gt21yZf>3??icUf6 zz50=I;JO(M!@$6b9wz4bSr%Wqv3(S}+tspoYBJZkPqhBt3vZ}fN`?O`-`vNV1;H>O zD!<%JIrc%aB@LYtjzcZ(JearBa~(~l1ZA?I0gp=7*1od9fhBBXaEyp6{Sc9ne>E)m zsNktY73bm^wjE93Xw|%p$o6UBits=hUV|1T6p2tNU(c+Ac4twF@TuejRW(2L^(A{| zBkl~J7b(V?5KYD0K{m0qd{j&(dxaXy0^O8eNlzv`ZJd?rW)PcI6cD$xM@#X}SmV+C zWHqt5sJB9%++0wKv)Cz2#q{c7fk%zo7Ja4NVZQ0mO32)1%34AVBMik+i&C%TwlYze zOA|xA$YlQQu(VxbY$j`UkxtWBrd2!Y2YNUNLM>M$4E4L-!6=njrWyKR%w&uzX|$>t zPbE~?&JX3YXKOr~_bhH{AW3bNTDovA7o&yAjsU`zKrmz5mExt zp20Hg?7U=@vSc6hfmN;vnj;g%+XDkC5s~;U#mGo`*iL4f%+OA@uFjnh2Hm2_HxY#T zlhu@89N!xg}!v5uALRE-c#pGkKZ!yS2RIs zjgo99a&0R?VFw>+9}cZ!%X>F=qCeEf??&%k{%@?%;W`G6&p4&G=Ys{5 zG}Sdgk{iyV9osNQrnDAzYx>JNqD*})sOP+xbh|&4LT|{jDC46FKEYO z;thk=Ovq=ChFpO_%|)i~w#o^6d+`jOU*Cy)?p>;Oe%PC4KWJtwH9ll>r+q|xTyp5g zU8x}N0xhS}$H4{*7o30f%_JSZ>R=E^pXypBD745eeaSE#rM&E(e;BAxyeJ{1so3-@ z_S;NF{WdpUW|;6YBx7#5PXw24-!#OmpQF!~zJ7%@RKUFLJbEwXki*1lqgj}8`>h@+ zc^8UQSXk9Te7j>8d2Lv0qyqV7kf5EkN%$LSnQ&2=@AAqGJt@`)z;`vhYhZf0#UQS3 zVbZ$QEzz@?=Y<`=g|5TcCft~=y}pV0bpvX2Wk2O}k}?ycFTYY}424$D8c5!{+1LfDu#80GZ-=DA7y&GAHphlJ0lOOIKCr zpo42YY!<)X&KV~d&KxFsL+sWjT(N6p`gK8Dd9ktJ02fNP$~#wa{vu<^YU%Qen{-mE z53u<~^cCJ3;ZHW779V6$IzO=7iDo*LnL*;2lDv9mCUIf2=O>fq?JC2BA#LRp& zd$Pw9659e_t-lhZ#X)JXd(YQ3v)$}{8KZULua_)h#|I+~hK5e)SrbJ7($_l$;JAZH0aJeS@!+!1>4NiHWcpeUnMK|FBD*+EwgHEX6N$V_kT z&23bhPa!hF_fWb)>rOy}3bk+1vu(+T#8291?VH9#4doxyTei#>1bFiA#}kuZYuaKP zGJ}Rc(J0P8JTO`j_K1OK6XbhTa-P;)vr6>Z7Uhr%Q`pk%&0M&*d{gKBV2ELOVxOD% zr&~F`zm$uTo%NrN?^?E%D1IYr5KE8d?_kw{v_F^qsS%!y=UDdsJ5Rc zQk3ML`|Nk7{dx17Ku?*%hV|9)`av@(;|c4l2Cbabe#>_k8p(Of$QilsS$rg?g>q6A zJ-0V}gwA2(ys32>;)OBnB2@2^2?=je;*8D5FU-R&H6?=`kTO@F?vsmt?pbvpJf^XT zWSCE|Xc!{SNIH9AKFv>CEw#6tPBX{sO-rA2AvtPRi=wg2pia=oaaT^o2)`OljS%hd zoPIgGetDX*@!?1U>)+P)ar?L9J5x50dgNc;#lko{w)QzMxM*DBAmDK`$${Ker1wBU z?vnwuOOj8FG}f}i z!BDy9)R$jqD{2}^j!_8=)B@)u9@H)qN9wy-mPr^$Z(B0^X|SxCyU_A7>{#T{X9#+4 zPVMssFGb+K6+Q18=*m!XI{fOgSGu#^3+)cJxP~%^zTLU<0xqn!gIN165K3s#OhAiH z*5@beoj!T2jIh`|@0!uusiTLb=Xe*W<_9LcZ!1Grgg|~OS{HwLsWNtWqxk8B16sdJ zSfi3ln%Pgb^K{u5q|Cz2;Gxl_2P83p7^z5VgH|QgCeQyMg1R*R!UJb)oZ0z?iT;3u zZ8fr_$4Zh`3*IbkGK8TwQ?jT5&<#sf=dhqJ8kF1v`4a#G{Uynsl1qnY$R}O+53~sJ zzeWy5S_2h^|BgtKo%fsnG}tEa2P%APMnKqmjfTNM;SVOcc}o@&->S*n^9PWW4I@4+ z>uBs|PXBj8X%GTy2BlC*=07N<7`3$k!D5fkj(}^o*uqy|Jrn0>&&7|yAACCa(k(h; z#&)+BdME?pj}CyEigG(TE8!|;6OSk7sSOHZjCDiu)#CyIpg^<=_N=@Dx&uYqj_=$b z%ypEAp-s`-J^y++$ev_x_cxYzAX)ao?nl0I@WIj*3)6teT=K9+4~ui z$CDzT;D44)C4TV67_8j_*+v#^@bY=_C$rX=t7u;+*d9;20<35Mm_%^B#=*j1sd~gr zXB)LTetbq2{Sn2#RSU3#mC7mLJ$p1Guu!RP6+Po;ft~>Uusx@FGA#ju?DQZ?#?{q^ zW1n3x4{;M>9LC<)(RJckFcts|hC(+5WK%MTliYs3iZQ*}c`xN7F&_V$^4^^c>Sm6+ z2k}29E6ahD75B<)#Q{o!rGB%6GY~qq5Y4G4J`}~^_T3-RaJ1tXYwD3>RPW^;`7BF0 z?x(8szB$%Z&%Zh}HU!kGJ8l z#5>CdnO_9J%vAuMATx>PHeJUoVynO=DqJPTwdJan{`o@xYPiGhq$|n;0{}?`*o&7> zuwIJVPeGZ;?6<4s5C1?-<6I=|3wl1aGN8+*+MSvxL+n9<(l6{7I}At(h7BOn^X%jG(>d)@X5ZMw|?)5BE|s00NSJznX)bU^Cj~Hcf~~i%?34@qp~iu9;sa?wj{Sy&T^B$!0sd|R3d<(Rd-;8 zA0^o$&kEs{zy*4=tT%av?Zfx^?nK+P!=E$_PaWsKeIm~e;i+y5il&tjgVr^GWMVRq z#`Fs1u!k)j7CU{+Ksg}#F$ihcs3$C{JnxRAd{Q9cg+hz~!K+xLr1{Ke`YDR^Pm!z}2u6h6JLjb&j4+~I4k+|&$ zk`NkR7n!3aHSS|PBXNXMAQE&wSztD{n~=`t{8c_)6cF5CqF!}GErEw3L`}jvq1sLFr%mxok&RO&dN*e zlrmDhJLTCk|M9yPsjCT}b|Avv5HtG zfP}(<&aT)WLoPDyRT>&o?pq7=AMx3bKo#r9!+^xFzr+p|`B3#3vb6w|YuW}iApDU$ z^lgb9bH>iGCRScwep3g1%O371u~HmhWELv4qMsUBqMFTNM9kQ3DV5Q2e@hh=#5uW8 zcD!7<^-9gFk=}jwUAgb#b2a}8O)(;Pg*bz?)aB2Zl1`r0j963twfU8RjLof2;ojf8 zfzVwOC0t#`Rh_OCI5lWjQFS_wyXdZp=9_&3inuy#>Io;mg+kg|k%%APqW9VR;i910 zLPUD5jGUD?o(@s@F@f*#b8IIR_hhDBbs?CXr;hDxgE0jcF27F+xu0n3{3+*Q zXk(N`GMC;|b1ivn^8)q7bvliN2k4>(pIJq{{VmcjZb#matfsN_H#hZicBV$kzD@@} z_GztnoP2z^!0dZ5qyaEBBg~ASn+vQ%5O0vv$w);am@C-MfYo0PH5H`D{kME4qq&EWQ593&OBZ zWGfjabL+nJl`t8add2R6krX-U)30Z#BZ9V9CnYx74%yfRQwGRiQZz@*}ula^@j7636@vV@JkcQFX?H%Oy1|NoeyeO)F zhZ4GVwV-CT`klnI^`F>)H7GPq$YIX(GZjtTjP znB%#fP`;jZ+F^}WLKfeW&_lVqv%IjzTkj}kXExa@LTytU;i8?=G`uTmB34Q6mQEmB zX$f%7U!1C|nUf<@x_SsGp*O-)ZZoVFDQ2DTZ=3P&Q_n*?Ib#_aQxv3A`m$p5H5`kYTq0lSZ>&W5+lr{Mw7#(h00M%8K5|i`c(z#LMd` z^**_$M3!|*gJ0(*0QPE%(h%6;@F-G$B_lo9}j=kgp3er~kBi5`g?0@ISE!RvR zA!d0=Fr`JcFTEgi`-Zc7o=~G}`>%s)KGsHcv zc?Fq+cP4=x0@XkJhiVFU4ujxA&)Suj%U{4Yw41I*6z*taQ}cX>_Hr!mZk=sUK96~A zl^ua%$P0a%NMTD2RCqj`GA!%CK_WoO%A6lVoHS z?LJ!x)1pH+1JjL`+h-~3=Pw`#&=NyQ%el~sFHxkEsl?3PE?8brWcC7c!?r3Ja?wg}xE z8eUdIUNR&ddXp$AE8v~Bt7rZHl&$*o;)PS5nZjf9TM zJMpszk*_IKR~d7!_JeDS-)a-(qC8=$AH%NT?y?t34U2B-g?h!L!z%`HP{j1Q{}@L| zUwO^Z4hr>j$PadHjy>J9&O?mx8mhHeidsTC@JObL!qzC?DS06BUYYXbq((tvxg*>+ z_bRVd3PusDRC2bF{@v^|mVD^AE&uUBy|xcomVLJm`^XdbbBO~HD1#~o^#P`#Upba) zsQeuvq3J$cZn~kxm(KI4j!~6UFhY zZ?CyqWv=`}B9V>}W4CSmNM-_J*{eBdDw9=6M)X(jJ$Rfr>oU)WZtjQU^+>2~;^4|R z()nSwdkbR^wPH4T=1i70uQYi0?+o?aka6rKFqU|YbILRNl7(R0<2=;Sy?ZBzkY#fI ztm2pFh^_3daKzA}U(})W8yZvRx5ETqN4^T4(QexPq4T9!HK<)yy^}}zM@q(zxlNkP zA;lYUPISz1!ei!Sp$YbsEG-(3wF=*@_w`I*!Ao^qhb~X>@rcf_tA6 zA+eX;m3~hCVMoKTp1<6C-6FCZO$4G`Y#5z8;e-(4)Me#NqBzyF@%D~(PM!sPEOk`c z=yGTNM`vg93X%~U9B!(GAw|v);1M5}q7>i??dd7ZH{lY7S$JO8h0e6Q-RSG%+I7RF zy?9kNQQ{OlyT66pb$M1iq-!+BeFsT#?v8SSB_)~Q4g z7SDDwa_2Tn6(=P}=`8Nu#Brlj`&uOGLDf;iME!;NWUnDP>&owaPvN$(8<#a5P@RUX zikDQhK$_?>OUCE)E@xh;`g{Lv^_&m>J8_z=3SDXMi|Vx=&O?)x<`kycv>bOjx!hIf zbBx6nK_t7dzqFV0NUQ7W;00Ypg7k3#^b<7|uJTD2)ZPtUf~*}Sq0oNfRGw_7FCRnh z5vO}&caR%G$q$VMeLgvARl&cG5TbmvMs;P}SNabl6w2#I1$7JEW@R|9l+IBC*q!IqEM~rl!op1t8b=4HQ;vw8%6Hqy=qeIwqe_ zN4!2$kGqEFvvyOite(5hes*XqF3LB<&@;x!)yVpb47?)OT|*@zqFjYoMEn`pcr1C& z8KB=PALK)n+1!VI=5Qf>u_zZ|!~TW|VNJyG$Yv*ouDw?0px!txy-RICyQHW60sW;R zoba`Qvhdsj)5`B0&S$k#GO|JeTj7vfdsoGg1^cM#&|0jW$)q*q;clmu8vv0RYBq9T(dt3PAe)5+Zs= zr%u;*tR+F?+f2J3AIEpgM@QlwbnEgyk#hP@IE-eA@H_B|guD6Eb-7>v0;?~fh7{TUqBKh+%BdGRt2S6328+zw)Hc#U zx^7n_{B*xLz1RKNvH&h`u|)Z1#{j+@HOQb}Zpg^O$fRIcFC+OCLw&^UWE+r+x!t)Z zoc7#TdcDl078aYCeu#~{KO3-A=?#m47kscP;R0^w4s}|ubboSXK6m1pfh|_a8cH`l z&$3I2Q7G9|yYN!Gg@bZGM=Cy|T*UVW&3_uT1H4#+0;u!DxfOcAh^VK171+s78Sr@5>RD6E1axKANOjskIB0DC{=K^dXU&gnOh8jMQ+f`^c z_i?94oYZDo`(aY#cGZNi*0Sh_W}{9fa_vZaA-NEKvB)S$+vBY7dcczPvg+u5lD?CJ4A-mFkC@-_!im zDGsULETf>;Y-PLhyA@g2Z=Ais{D@`+MUld{mMPZvuyrRR%!0z?vL5@HS*Yu$Jg2Y)+?ql z=>;}JKW@51xUAAzaP`!mJ5(tKyy>_cIjH?$RpKv~UwquuUEtuJ9{FYvQ^vhG`=e+D z!;}~K(sx#;4Y8cJ_6$L)J%%+@MG4=9Ts8>ZIa0etgGPZgSa9K^D&XTrW_QqLMB!G>}S6Vs#JNa z&H!B_H(3M`%ln%QVRyObv~Hg~r`7qefXyM{{BHj1(OW8=zEy{d&$Y239U3&^U1Pi< zb5)}*x`o%AYKNL%l;6P^;S|}Q$-Uso)ZX}>5Ib6*zj0SVP;6hqtemG>lv0tVUVQ$* zu#vAV44TowUUeL}O@4_cDWL<-Y*^TB)=}D_L;u<-Ntwa7sW!5REOkinR-BWMRxopy zQmdKH18Zq<=#sv+n<`I5R}|-(!&H0iMm>%Y&I&0A>X%7vE&TPmtEnRE6?amMdz3O^p7^_Ns)h|)`Z(;opluidGk$YH@daQ<>um)0~}XZggQ%~rO#eu0!y zfoGc@zi4EgD!)Nwxec6=Bld|R7Rt}35*BMg$6j%)S89KAOLN6Lho4=nYi)gi&t21P>m{1XU#8Kx|5N zKI59l5obSMIbP(PbwjD61TO_ccdK^ZnaecRSw)9KH*%l7wM!oL)q&}| z$SE{;2_NW2Mny|4-jJ?^iz8Z#DTBV%I&-1c4Jf zQNQ|okp+O*%Z(56CH{c!y=0)Gx4^Ia#@~y|xnv>ZF-pnKe_?pTJD{kSs@LG=--}`Z z43BbHDqZ~x%yXVDvxXdWORQBYFnT*h4}B1G%&b|f$4>lg4oUFJ04iHbQA)Fh^_hcI&knJpMP0-qKK@ zpTp+`t^if6Mj>jrlL|)#H%`~y`NJ2Sx7PQkJ|k<@lY|L6N52e;P%*6M0gn1Cl1WIgX@zBxHcT$<+wbn3tG#(n&gT=E7ED-&9S3lW%WMvN^ z2UDJFRRd}=lZ4;WpA!xMD6D1&bDR)LKtxh`Rlu^6ic`0en4EglmA1ikZK}|5q5F3U zadW0(ckJ#TDS_AJ?7cy-SdK!V!B36OWojPd#cF3XHRakkGS8^Q{FPCe^*bZ0L}%j~ z7tGH!!KwZXC6iA!KQFk0dgcG#rni@n`=TH9B~X$UU{vK`afpOTOE@pjk_FqI@~+qc z_05|m?^%=1*q@YT(;BAe5cPuHn?Zr)c})tdWNGgSKyR!8FyExYZmeWII<(?P5MA|? zfv?P)B!9RSSpfMP3PI>apQa=T!{ZZVeCvmvs(ujdqo5URb6y$k!y_cCV9f<<@%#n= z`aE{@?BauTSshQ__>*kwo(X5IMncbzrr?^#O+i%vOse!bI;gsSFGyb%)TdPLZ740L zJI%*gjjwAClr?e`|J8elKN<13oV1RpN|yq(-m7X7vDTl_$+6`F3iV8OS10wZn!@6B zxDOUt+^*_W>UWII)%U#1d z8v_YocaOF&Uy*css-kPxpROnrM=1a*vk%Xx6U8_1Gq?VEJ=sD$A7$cY&hY19ZOIS< zYh1^hYk&EuDrzQ3m-EIu_dmwWOdh7V!X^orG8-jEg(^s00W99IeX>1$Ii-ClC9T&d?ZuvcftZ% zzxwq?@>M`+y?%x73)&qvvabR~hyyBd{LLorbI}M87{VJU2^JUo(k6c`4NT&>yTa;a z4%cQXy~=uTY|G5XKjPane0-a99x+M8Ma6(VAUy3X;xPLFD``8-RWa>@tzr(B?3WJq z&Apu{>0W~8Y;33N>*xUCg-;>qgkaiN0zo0qpDG6uYU}sHDR}B~0yI)AfEr?&8BNG| zh7ye{U@i;*IwU9T1-gdtJvr1~Fh9hXp7*z#>Sq}LxOqR>8FCQ6a#5=wnC8)mJ6UsO zLVj(uGP(l7Fs25?=!usXm{%l_beQ1+BT3VdLX%0ri~j8SXA)%RuUkjVXw9k;KTASE zJL|E#N>5S+3J9uKCu@vU7VtDKp8c(O^o4Nb<2}*}hq(^7T6_Ydnke3!{Lf5dl#oJ$ z$O91Al|<)6nPLvsnMy1r1a$P@vH!;MOh=`6M6J%N9w9Y!fb zMC#7cnysmTEfkJ11T5SHpweI!--Xse{XY`5f+y$bK!PEaWCNOZRFTP-*i_5L2i%@a z{p~?JEm$U-H~lV=CV2=7SvxVt0uW|AK!?D^#oMaq)k@hytvEC;^4gF-ue&3 zXz}Ahg>Z&KXoC-;?d|oAvj15+F7>YazBRET*3v_F6z^m`?>9FDxrJ1-Jsj4HIN?LR5*;Dbl&-@gdOM>QO<*#~b9Je({s1Q8w5aa--Jf(^G)*DgWgttfV&k8=hj1ak*l+CT z_q|^d8%jYEg=}N+Wq*2Sn0RYe_+4@P3pJo}e~R9r?K(b;AwYX?`FKBojcY*PRwbwx zs@a%nt$8-nQl*h5-^|$zdj0++A1zeYL&7QGf+VAAr<~t$F;rw~k(x{YktU~Ze(`QD zh0gM1O$B9TE_M>UXkrlOo_0U?9)K`LJ#l}x!lwo6W%Ec8y; zfHY)CfL_^2d>b+~x8W?lzz3usHTuOy6$t8l9^=a1CGQ6&H7?FdFvpfA`P4jK+Eh z@(VoJfi2M~!V@$pn_QN|r;!ezbu7UMboot!0_WM)+AY-?&)IhhT(@){i9h`=D*_)E z6!FD*WsUcM?yqXlQ7R-VFm|$rrwI!@$bCw?gdOpntaQ>|mBf*-9(PcJXE25Dyul+? zG zlqSGAv<$Ai{Ck}`FYw+T?flRa{nG{`+SDXB{@C*AK_vjOn3jB_0WR)8O3I`IfR5-u z|9=FEiy%FEyrhlgc!sx%!`B!%!{{LFm|M`ArQNQS_9{)6rsoR&X zWSMD9?N)aoD3=vp0`SHRD7+sw7e0_qj}x$}u5w(c#?ubq0EVD1wI}XYuu?XUGzH&( zfCkI}z7K6eXvBC%Rc!9Eh;=WMh(*U`SA2&82%?g#Xa8S8BsvrepnP=I2Q(if^n#Y_ zrIv6;dxFJ$gHn78??Ra5W*e)=AEM+)LHqkYfE9`%eT-k(|>?NV4NpwTr1+cLBqocrr{We z`0|{1*wksRBl@vs#gpdOqAx>9|4EiS#-}LS9H4GIUjkot4N#jkaLNH=z<*&A*n{X{ zV@H*d8dv+><-DS5K*v;ND8-IHJsQA*;{v|^FUSi3yLdpVn&`2dvzlMpjn7n!xRCgx zN$`ENh%dG&&$TBlr$O(*#AkmZ%!aauT_j9~P4$CyV$9E1|3q5s*X1p* zzb>38o|nVnANxgekY5mKekE!Nt_8z@spqTv&?*Q|K!ewV7EGa_nrHR-kRfS9OdShCwE2ycC{ez z#@5MBp$$-{@o}#&oa9Zg>#`6-a=Cf`lUFlHF`K`A^*?j=C!zXHGC2t21^6vE4y0{1 z1Ta}8>^k?!ghRnA*q-fJIhpW7FyWY;A4Vq=rUfIzSE9yFQurt^VQx&*KQ`q;V5H#o zt!@w&TnyFg`|DCWpAg9X8D5p;x&7eLQX-WC^pJp$-@Kegb zn%XedpJZ{_pTJ1yojCfF*RTQW*Dvk$j}^r?e9;E*!&lTZCrhu5R~Vt|0)O@8 zeD;0t*o)jP|5$pw!ocoY{@n*KslNh~Wifp7kEKU~k)apg{-_;Lp^HEn9+q7>*({{M zz)0?yID!)uVgbq!ZGD^PWMl}8=v?9_I#HpAz`Qy8o*18uKr^QK4vH-nyyH zl_w)QziGz;2g5$ zp;|oI?zjOUHduuchKHQu9(a|3n`BPI{ z)9ep6cA7r}Pw~sMV51NPCppwY65~4PZp(L`nJuR|{1zuLIZ@>}=|@0h+><0_r22;C zdT$Hy>1(JxlF!Td#|fQpo8#W=OH-&(;;H(ok|;hz@A&ODVyf0%DCH6L$)a&ze_KP_ zE9N*~`F{hDAYagQ$}&aD7QoI_S;V|bc=uaq4ytu5 zWT~CeG}f&?Wmt5Up~a#IBt=yqEFsim;4w-MH1gADR)jMa}0V4w= z)W~!ELQG?<9(gFZtPFELS7c$VQJ}L**c(VrQ;nFcZbnbyyUNVlBlaAbYi-gA&oP9? zWXX4`J^e}b+xF&8iPD&tn$jmajF6Pw3qFSciN>>61^S;FotFpMao_%l^ia2%K*sa* zc)>Ar{rWm<*k4PReCH|tteu~%8Tl;F|5Uf3g<1Ie>e6J&od8;+xQUY>@s+U3mMB*^ zxha^^wJC_gQOe`j)1!lJ$9q9%U+KO4pVFkaDCk{0Pl5-8(4Z{Ej6Cumj+>RRib6Ia zpP2DPEghKO)(EFmy<{#Fk>dfPKPbQxWMmWfc{^%6Jsbv;m zWxNLSC+_1d2MDwsjj|^;;Q`3+VWpV;6IYcE`inD5_|D^n`}=Vzrr_b*5f=6T8&8M6 zdxw3Sj^sw~UBa#TE=PPWX;l2k|ELaVCIT>B6&ZN=QYA@ptiZUcA3&7?-QfJekO|Q9 ziuTxE{@ry)hxlw6gG9i58UX zLU*DG0Nk$4UAr{5+ErrFIdMr%^07tk<|^1n6cAa&*Q6S=Gy&AfRV!2Z89?`*)p_pX z6UfqM59vFR;kMIjs0#a9Zsla@$vQv@3Ww`ZH~Q()+|o%xii&6vjW5e7lK`+y+!ee* z4ITo-C3$Ts-s>YkXQzcZ^vE37N&r9>BjLUABbc#TOI56SdugCG^qN?|oQC<#v_mI1 z0yHv}?;hGp`h&w2$4opiNCc>TM}QJ2o~W(YRGfHL^ylV?%lc*~uU2=c-itWL^pFS9 z8{p)=MFap0QNGCJk~K?KyU`0&eE8m22Ye$n=J|1P(>cfLQFRU%E?@XTu|i39^OT{^XU17y<^3W-D0gl;j}^=;EW#KU!Xqfek+5g-E2Ud&U};KFpV3ibV_Zal#%RiBchZ% zpXk)DxMMKq3Lf7w9Vo)#*&${UQF>plHvm!cTbInpwo> z(9IQW(poEA>~7$M-h2kUJErB$)k_Ntcu*C_|8bT`eRm_=cS0pzuzeC7&8dodadFP> zY}H{_CN&Ys*_T^|y8@QqQk2c!9oFyTkM*oS!q~TRrg$3&O*Mzj(LMvjy(_*tps059 zHJnJ?1^pfq(B~#yYs2OLWX_<|QGmeOEZRMxvnzzR-nznmFzEqC5>EoVf~IdHlnfGc z79MhHpuk9kTo;tiExSy6yVexoDf}^b9lrQ*H3WK?TDLp$4&ME#Pp=X*j0|UL?Q7G0 zo<7Fyc0=Wi%7?z*t(bH6R4od==I~Mrd!wVOPYLOUb4t*iqluWxt7EdDy}yaY9~Rew zsW?hJn|Esz96n_H$*gMF2^<0E$XoX-Vlq{4;7tQ=y9fc=pB+Gh7{UuH-Ebq%syie) zD{LAmE>fYDv9{daYABTY8T8}QfD;sZ42HEiQv==-hJyyzo*=1%#Wcjq=%-a+vreFx zvWz(GvD7aExJ<>EWddKD=}@LM(9w!rm`t|p&WlT`#he&c=FN}Ir<_Wvzys~v$;1!2 zT|A3^z$)^-4~XOxKmOSAKdL(OKq%M0jYl~;BqDx$o=y{a)AS zvDY57{F&*Wm6X}!=9~S_s8j!f)>bK|gNOBBP5q2-Jyj~Ymys9(UJ2iXz|NXq&C6kf z0AA|X&rZ0geg_-~)%7UPw!2~VjU7S6UuHnG+JB4shUQi`c_(W!uO^f5R`qS6dJHzTe9}N@PumTmYELAti^lI%E@YG_;Pe zpJO`dk)fdGyaSpC71DI@Ui5(0eh9pE#I#FfBI=J$N6KN#F&$} z%Z{uEKK+XO#xm$0yw#u^4>|pD1OAck*$XA2Pxx?v*(a*y72oZ>kkgmTSu-8^MK34z zCI|JFI`Ga*IlID(?n9#JdV(K1tCN4;66|SJnbV2iC-kIbvo{-K4wfY69QG07AQ^pn zsQe37mJ!Q7?qt!RO!*O$*82bl@B!~@@8Uib=AdZNXo`j`Sxq%ZCo?xg96E8N-nNIZ z=M!}5gzB)X4}!`TE6u-4!y0NAT~O-OCw^I*1B5JxV5>AHm;KjS!$N*wUbjQfjTt4c zD>wW4r>sAiP}K980C3rC1i1Ne-J_E#yP>U9DLHTNlPG;vWW8I$xvha}s4R+suD^E% zLlJ_ty9Ndk)sgW&{VEUWGw7iRBow63|7trqdBhKFIXXu#ovG$7{j3MjJ?6aO>L878 zn31Hj7vTA=zuRITmlubvy_0KAWq-+w;ivvcO9LOIT)bZlUUIJ?*ed#=y&j-nIW3Dzy$L4j!&VO$4&_?$J>6|({cs${1KDecHeu!2{W%5iew$d zUFuyjHtn4mRClf$m>Jf1){oy^%&`|&6{j3K+MjqYu6?+a88;~`1oiVq_n%E2C!eSu zzpR}OHkG=ua>$J+x!fM-lS4rcC%N;cLXI!r1*pl}mKc;8>h60$wK`b!|8(_MzuwM2 z{!cFe(SSAvrHoLV1eS4#6yUQS6*7;8yQru2)Q}XT_G(9vZYF38H_8zJcJ7VoDKIY|4sCOPOjE~A;x4PLhTU;r`XOm{%Cy(^JD zr*Dv(w$>$`uF9XdV00QkS@05#T2}RiMo&n6z$CFfON9IQxa4vqD)lo)$#_Ry+J293 zD06}0ECFIn{FIHb>81e-oLJL)fiWlgs?26tPQ+N5#b=no#&q?_4#elyJp%3aHil!% zCk%xge)MN?hWuTMveh>S+F;4+0VQQ!*u;gb&~?vMJ~p6a8iBdWXki;Huz!2NetBCq zjJE*2xarG*!Y1?+u_}`TaeIt$dr}R9{knI9o7-`p@3rkxJ&kVIl?KbeuAp`MlpuoZ zjKiB%W~WqkBe6N5_Uoc9YbJ}O(Sc+%b8~gL=_~g&*f&Q!i{RvF?0CU@TvuXcfL0d)+AEfIu zJNTNTadWfgX0aLqKFWtbZ-@m9vHQqM*wQ!B~K!iwljkm+k5~A zsb1EEWB{nr%?lR_7M(oG3->T}5N4JoaPcM^8QELgfCoAdT6x|`zzOp$t&t9(L&9*E z6hs#PP@9p=|HiMBP_aLb4obU-2R2Sub$Mq1TU{MrmQkHTiee1-R7k4W6vSgi)UNID zy=O_0jVF_a0eDBBAvxvX8442j%sE`s#%%p8fQ(=l2`TIPR%$=NN!D{B6ISZ@DAdH> zD5hjaJ(1D~1YL{~uQ{0{`u?sbppf*{oZFZ-JA`#@dM2S>T$NH%Xd{)!{ItBU;qHl- zPq5?db=+dRmcf9kW@Sm&^O=ggiR_^~YNf-$r2 zpb}RQTVQ^#e?}J1sm`XjNcMh~O}1QLh~_hisofI;xd@-^kC+ndg{CWoT%|BG?bqEN z8GMum^*JV(uDP0MZ~Zt-4wmXKlU-w|Fc&fLvWIVD1ifU>ykusfb_+-`bbEr@)vt`E zf1@YoY{!Rrn7QC!#Amzfl!CI7p*X=rxX#!2D5u)T_ia%yIF{{Sp-{AsRd`zfCUmls^L?R>-nLFklB0L)z@k&EW)kFL5g$Odkbxo>3v@V zSpmKFc1QXc$_eWhj(g2x_n!lB94rL3iAluRcERm^WRe6n{put3Z8GU##_b{SSkS%T zuGhjHftu7*=7nOg_hEKLl5$h%2P+3cGe;xR8 z1NIczZXUoFJbyd*r~@}dtL?8!;63a2PS!SUl7vD;*tlbTpF7zRrWR7a0}6M}AplPi zbGV%ssGQU_w>Z|tH*dPrjvwfU<@!bDPHTw^H8BfKXZ@3-Rl6DlC zitl6nC5csttju}(3V>!<$05tA0d*C0yoLV$d12E(1W8KY$#@V!{f#Sqaf9aWLi?Rd zxG@?D4ing$v2K@ebJ!#A1RJrXvsq$G!cFb{4_ZgOzViXjYr!F$rqi-qbmj!GT|QTT zGvCrK92Zai(c_S;RR#hp@kW~FTcdv}A6;K<+LmdI#Xs;`+Ov~qp|=QmGGEzII(K^? zcc-gX$og`?D{^yn;5vaIwgg#f?^hoPmeV`msA6JJZgiCSs?o1CI;o57E9E}2!MU+u z(8+&ZLY~KU)A}#BF7asDTN2!ymS>$w76xC;eeo4;OG{_NcuC>`e}Z~sxRZ1xRNJn1 zP*@?Jd^Q$ev7ei!nf^jt_RrIw^xlonMbg@{2WCR}PflY-lA6N8XaA3txE1Zug&qwn zE^1NAAin8x1urHZuU3q8U!V3ely})F+xP@uA;FcF-?*aQv<1qY5Ir8y=spQ3n*=U6Nbc-WuD{3E*qTS znr+l{9#Kdl^z{qJ>l9}Y#or!FD&n-ofw4)E(#Ev|<^6YAUQEpvm$mb-cB@ZHy&Btx zI67sBgA)PYy$VKnT&2_TzNH5{Y7?;b8a&aUj>#QhnTJU!E>hY6ltmAn zx`xI$QCk-%(bubrK|q=4NyOd)Upa^;?|F3VW6JL*hig4+d0ft{jumXHJifuE+&txN zzl8bL4R98-3Mmbk;HB{5=j0iaZyjSX+{H9;0iWid%PrS&om8LINu87gA9r8CSn%^g z)vhf&)&|le(UjXIb|MzTlOj+Bh6g4kVR&;ImFQFS5RhMN>P`CHR|bawnSBt*XNg9? z5=|PC!-rv|guX*ljvk$q^z4&usuLTJbgVCsDwHz_s9*_nbj+;@Kk)Iz^NATbGHBucI)aWc0*f6@I5K~3Nnhso#x*peDu=Z2= zD~OrgCOtn`PSv;q6ZJ$9D^({(QZfcrT>0~4x9l={4P7ViA;yJffFzKm@i+zKC4Gmq zPArX?3_`|Ym3X(X_rII#g$X!K{&}G3>@KG61z5Y;AxT@7#tNq|+lN5xez~{eM_&$8 zXwom&I+~rcZ%g7$loR`}i?WaIpE0kM!KE9hO^)VWQ~r37EOMKkY(JN@9CFI@m36OU z0y9?SbH@T9+$=dzp{*mqMd**Cb7?wVS_OS}&94NbTzCAom`Jf|S^I&M5~zD9nGm$5 zIysuE*mND*aC}Z=fS!1k6&oQhgonm+nFmD?COHm1-^LonyDq9X(K$Jb4uJ}_M*5f# z!ps5DNQ`q$yNLz-xWu($=S&|%Ur-+`2whI*VPrc(OlIp8%XM1nWJ0^tYV}-bg&^q`rkDX{gqVBtYhoE>3dy~YWD-G@FHg28h+m7crAQtjU3{{ zB2Zm2<>_xsyAc<-5;?zUWsmh0-X&2ZfIW^v#dk18gk(QhUw~eobS)}f>`}ickPB^? z$RVLc;bfh(qaOC;dlUiMVhJ4cjz}oCw%wwpbeCW5h9|peLz=o%l~Sb`IulU^E4#xZ zF{$)V){qR#u5(K$(sOIA+-U3YJPc`nu@;JNGcS(IU zq*t*kxZ6e`zSD?qS*l~(9L;o&&YvS#u=1g_$z7$NU?oUeHr1ITia6wI7+rKFl%b-e zrt*CmZeOOB)p=z4P+E3`07V6{VWTK?o$Z>(E7q|N>l~hKq*{0AdRDGfiVJNcb{57e z2F;dobQQa8u!kw?jtQLH`IIV+P+~~&AV+%q_p3N_Q>oW=6JjzlE14K0@t1%>Ea zl~^;GM+jLAL|Q#x%Swgx40k8bGTimjMt>Z5T`)h(rGtdB*6rVt(rgW%qNH}_|0l}l zMbPgfnGT&oMB-#paHZz83P>9A1ToqeVw|akGeNma$qX>Qm>QnJl{$w%5j-X&To>dZ z{@r#7?cI3&6~_L4mlaJ?X{d_cake&sa#%x?MkVS)bD1V=yQJpzg z#EczJd^}Dzwrq}t+C;TsiG_+6Lq)syYfS1tuDiXvwtl6gNZ4azH*xSySBTs=VFg($ zysTx^i(*dzQbw$ITjsdI_&WZho48(T$b!et5Lu1`zvj}1t0Udt{nw2 z!0FI*@`0)@qa&71P`j0fp^{T37cIIyT3&eMCsnGuN@%hFG-K7rj&;cW)m|lzIh6Q3 zxioNcK<16wK3><$2gBbjcGsjmFls)(6?NHwJ07zV==2 z49yjmW+o#9m`G5k;>vpIj+p$tQm`YBcu9eyHdSBF3d=}5Y+Yg>`C*fD9TAL_58N(J z3P=evAmEZM{bZjsE&K8qA){XBfHICIb9*;8+J(z>(2ALHT4#y8l6?tK+x>-?Xs(Xt zxPe8;k<*A6xv+Z8#pXn{!Yio~_SBrTe8}BAk-LoE>+d~xO-kxh5Z=-sq zt}!##g5`EQ#u$h>6d8O~Jd>K#={pfMdXoMZ_n|KPRql*!=Cg#3TG}%nL8Jnw3lWxN;H*FO#7Ug|ISn!OCt86rV|fvva#;?JUNFLvZFdRAkk=uv|EeRHab*&fi7Cg# zrxAw7tq32dKvt?OBnRUxVt=pmoHh(_9PioEE)jXRG;Pw|51=sLq>BbuufrT&DCL;# zgK0AP@FM9;4CMmKzCP?lM(lns=}S{QcKkVJAu=*G8Bt$gKqQ(bBQQf%R^&T1u|XLq zbns|RU?TG>t+ij%WW%#^#FBW*R*bw@+Lv^`F~nj#=n9t)gOBJzcB`X59@7%!4+ouDf53? zVBvp&nVI74HIaX(XdG~bqU#p>?-WgO*Opw}jV)V=td?=`HM1QS=dI6`n_P?dAFc4( ACIA2c diff --git a/man/get_available_crosswalks.Rd b/man/get_available_crosswalks.Rd index ab05fa7..c8a1d3e 100644 --- a/man/get_available_crosswalks.Rd +++ b/man/get_available_crosswalks.Rd @@ -8,9 +8,62 @@ get_available_crosswalks() } \value{ A tibble with columns: \code{source_geography}, \code{target_geography}, -\code{source_year}, \code{target_year}. +\code{source_year}, \code{target_year}, \code{crosswalk_source}. } \description{ Returns a tibble of all source/target geography and year combinations -supported by \code{get_crosswalk()}. +supported by \code{get_crosswalk()}, along with the crosswalk source that serves +each combination. +} +\details{ +\strong{How to read the year columns.} Years identify the geography \emph{vintages} a +crosswalk translates between, not the only request years \code{get_crosswalk()} +accepts: +\itemize{ +\item \strong{GeoCorr} (same-year geography changes) rows are listed under their +reference years: 2022 for 2020-Census geography and 2018 for 2010-Census +geography. Requests with other year contexts resolve to the matching +version -- years 2020 and later (or no years at all) use GeoCorr 2022, +and years 2010-2019 use GeoCorr 2018. For example, a request for 2021 +tracts -> 2021 ZCTAs is served by the tract -> zcta 2022 row. +\item \strong{County-events} rows enumerate every supported year pair explicitly +(county -> county: any forward pair from 2000 on; tract, block group, and +block: same-decade pairs from 2010 on, including exactly reversible +relabels). +} + +\strong{Nested geographies.} Same-year pairs in which the source geography nests +within the target (e.g., tract -> county) are listed because GeoCorr serves +them, but \code{get_crosswalk()} returns an empty crosswalk with a warning for +these: no crosswalk is needed -- aggregate your data directly instead (e.g., +tracts nest within counties, so tract values can be summed by the first five +GEOID characters). + +\strong{API keys by source.} The \code{crosswalk_source} column indicates which source +\code{get_crosswalk()} uses for a direct request of that combination (multi-step +chains may combine several): +\itemize{ +\item \code{geocorr}: no API key required +\item \code{nhgis}: requires \code{IPUMS_API_KEY} (see \code{\link[=get_crosswalk]{get_crosswalk()}}) +\item \code{ctdata_2020_2022}: county requests require a Census API key +(\code{CENSUS_API_KEY}) and the \code{tidycensus} package; sub-county requests +require neither +\item \code{county_events}: no API key; the data ships with the package and works +offline +} +} +\examples{ +available <- get_available_crosswalks() + +# All combinations that translate data into 2020 tracts +available |> + dplyr::filter(target_geography == "tract", target_year == 2020) + +# County -> county crosswalks support arbitrary year pairs from 2000 on +available |> + dplyr::filter(source_geography == "county", target_geography == "county") +} +\seealso{ +\code{\link[=get_crosswalk]{get_crosswalk()}} to fetch a crosswalk; \code{\link[=list_nhgis_crosswalks]{list_nhgis_crosswalks()}} +for the underlying NHGIS listing. } diff --git a/man/get_crosswalk.Rd b/man/get_crosswalk.Rd index 84d0728..e2a05b7 100644 --- a/man/get_crosswalk.Rd +++ b/man/get_crosswalk.Rd @@ -15,30 +15,45 @@ get_crosswalk( ) } \arguments{ -\item{source_geography}{Character. Source geography name. One of c("block", -"block group", "tract", "place", "county", "urban_area", "zcta", "puma", -"puma12", "puma22", "cd115", "cd116", "cd118", "cd119", "urban_area", -"core_based_statistical_area").} +\item{source_geography}{Character. Source geography name. For same-year +(GeoCorr) crosswalks, one of c("block", "block group", "tract", "county", +"place", "zcta", "puma", "puma12", "puma22", "cd115", "cd116", "cd118", +"cd119"). For cross-decade (NHGIS) crosswalks, sources are limited to +c("block", "block group", "tract"). Use \code{get_available_crosswalks()} for +the full listing of supported combinations.} -\item{target_geography}{Character. Target geography name. One of c("block", -"block group", "tract", "place", "county", "urban_area", "zcta", "puma", -"puma12", "puma22", "cd115", "cd116", "cd118", "cd119", "urban_area", -"core_based_statistical_area").} +\item{target_geography}{Character. Target geography name. For same-year +(GeoCorr) crosswalks, one of c("block", "block group", "tract", "county", +"place", "zcta", "puma", "puma12", "puma22", "cd115", "cd116", "cd118", +"cd119"). For cross-decade (NHGIS) crosswalks, targets additionally +include c("urban_area", "core_based_statistical_area"). Use +\code{get_available_crosswalks()} for the full listing.} \item{source_year}{Character or numeric. Year of the source geography, one of -c(1990, 2000, 2010, 2020).} +c(1990, 2000, 2010, 2020) for decennial years, or c(2011, 2012, 2014, 2015, +2022) for non-census years (limited to block groups, tracts, and counties). +For county -> county crosswalks, any year from 2000 onward is supported; +for same-decade tract/block group/block crosswalks, any year from 2010 +onward (see Details).} \item{target_year}{Character or numeric. Year of the target geography, one of c(1990, 2000, 2010, 2020) for decennial crosswalks, or c(2011, 2012, 2014, 2015, 2022) for non-census year crosswalks (limited to block groups, tracts, -and counties).} +and counties). For county -> county crosswalks, any year from 2000 onward +is supported; for same-decade tract/block group/block crosswalks, any year +from 2010 onward (see Details).} -\item{cache}{Directory path. Where to download the crosswalk to. If NULL (default), -crosswalk is returned but not saved to disk. Individual component crosswalks -are cached separately when provided.} +\item{cache}{Directory path. Where to save the crosswalk. If NULL (default), +the crosswalk is returned but not saved to disk. When provided, individual +component crosswalks are cached separately, and subsequent calls with the +same parameters read from the cached files instead of re-downloading +(see Details).} -\item{weight}{Character. Weighting variable for Geocorr crosswalks. One of -c("population", "housing", "land").} +\item{weight}{Character. Weighting variable used to construct or select +allocation factors. One of c("population", "housing", "land"). Applies to +GeoCorr crosswalks and to the selection of the NHGIS interpolation weight +(block-based NHGIS crosswalks provide a single combined weight, which is +used regardless of this argument).} \item{silent}{Logical. If \code{TRUE}, suppresses all informational messages and warnings. Defaults to \code{getOption("crosswalk.silent", FALSE)}. Set @@ -53,9 +68,13 @@ Single-step transformations have one crosswalk; multi-step have two or more.} \item{message}{A formatted message describing the crosswalk chain} } +For nested same-year requests (e.g., tract -> county), \code{crosswalks$step_1} +is an empty tibble and a warning explains that no crosswalk is needed +(see Details). + Each crosswalk tibble includes an attribute \code{crosswalk_metadata} (access via -\code{attr(result$crosswalks$step_1, "crosswalk_metadata")}) containing comprehensive -information about how the crosswalk was produced. +\code{attr(result$crosswalks$step_1, "crosswalk_metadata")}) documenting how the +crosswalk was produced; see the "Crosswalk metadata" section. Columns in returned crosswalk dataframes (some may not be present depending on source): \describe{ @@ -83,10 +102,10 @@ structure containing one or more crosswalk tibbles. } \details{ This function sources crosswalks from Geocorr 2022, Geocorr 2018, -IPUMS NHGIS, and CT Data Collaborative. Crosswalk weights are from the -original sources and have not been modified; this function merely standardizes -the format of the returned crosswalks and enables easy programmatic access -and caching. +IPUMS NHGIS, CT Data Collaborative, and a curated registry of county +change events. Crosswalk weights are from the original sources and have +not been modified; this function merely standardizes the format of the +returned crosswalks and enables easy programmatic access and caching. \strong{GeoCorr version selection}: For same-year geography crosswalks, the appropriate GeoCorr version is selected automatically based on the year: @@ -117,11 +136,73 @@ Connecticut (county-equivalent planning regions replaced historical counties). For this case, the function combines CT Data Collaborative crosswalks for Connecticut with identity mappings for other states. +\strong{Arbitrary-year county crosswalks (county change events)}: Counties and +county equivalents are occasionally redefined outside decennial censuses +(e.g., the 2013 Bedford, VA merge; the 2015 Shannon -> Oglala Lakota, SD +FIPS change; the Valdez-Cordova, AK split, first reflected in 2020-vintage +products; the 2022 Connecticut planning regions). County -> county +crosswalks are therefore available between ANY pair of years from 2000 +through the curated horizon (e.g., 2014 -> 2019 or 2014 -> 2023), built +from an identity mapping plus the documented change events in the +interval, with population-based allocation factors for splits and +part transfers. County crosswalks are forward-only (source_year < +target_year). For sub-county geographies (tract, block group, block) +affected by these county changes, same-decade crosswalks are available +from 2010 onward (e.g., tract 2014 -> 2019); year pairs spanning a +decennial census are planned automatically as chains through NHGIS +crosswalks (e.g., tract 2014 -> 2023). + +\strong{Nested geographies}: When the source geography nests exactly within +the target geography and the years match (e.g., tract -> county, block +group -> tract), no crosswalk is needed: every source unit belongs to +exactly one target unit. In this case the function warns and returns the +standard list structure with an \emph{empty} crosswalk tibble -- aggregate +your data directly instead (e.g., sum tract values by the first five +GEOID characters to obtain county values). + +\strong{Caching}: When \code{cache} is a directory path, each component crosswalk +is written there as a CSV, and subsequent calls with the same parameters +read the file from disk instead of re-downloading. Cached files never +expire; delete a file to force a fresh download. The metadata attribute +records \code{cache_path} and whether the crosswalk was \code{read_from_cache}. + Note that an IPUMS NHGIS API key is required to access crosswalks from that source. Use \code{usethis::edit_r_environ(scope = "user")} to save your API key to your .Renviron; the name of the key should be "IPUMS_API_KEY". You can obtain a key from: https://account.ipums.org/api_keys. } +\section{Crosswalk metadata}{ + +The \code{crosswalk_metadata} attribute attached to each crosswalk tibble is a +list recording the crosswalk's provenance. Fields with no value for a given +source are NULL. Elements include: +\describe{ +\item{call_parameters}{The arguments the crosswalk was requested with +(\code{source_geography}, \code{target_geography}, \code{source_year}, \code{target_year}, +\code{weight}, \code{cache}).} +\item{data_source, data_source_full_name}{Short code and full name of the +source that produced the crosswalk (e.g., "geocorr", "Geocorr 2022 +(Missouri Census Data Center)").} +\item{download_url, api_endpoint, documentation_url, citation_url, +github_repository}{Links to the underlying data, its documentation, and +citation guidance, where applicable.} +\item{source_geography, target_geography, source_year, target_year}{The +requested geographies and years, plus \code{source_geography_standardized} / +\code{target_geography_standardized} (the source-specific codes they resolved +to) and \code{reference_year} (the source's reference vintage, e.g., the +GeoCorr version).} +\item{weighting_variable}{The weighting variable used to construct or +select allocation factors.} +\item{state_coverage, notes}{Geographic coverage notes and source-specific +caveats.} +\item{retrieved_at, cached, cache_path, read_from_cache}{When the +crosswalk was retrieved, and whether it was written to or read from a +local cache.} +\item{is_multi_step, crosswalk_package_version}{Whether the crosswalk is +part of a multi-step chain, and the package version that produced it.} +} +} + \examples{ \dontrun{ # Same-year crosswalk between geographies (uses Geocorr) @@ -159,5 +240,18 @@ output <- crosswalk_data( crosswalk = result, count_columns = "count_population", return_intermediate = TRUE) + +# Arbitrary-year county crosswalk handling non-decennial county changes +# (e.g., 2015 Shannon -> Oglala Lakota, SD; Valdez-Cordova, AK split) +result <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2014, + target_year = 2023) +} } +\seealso{ +\code{\link[=crosswalk_data]{crosswalk_data()}} to apply the returned crosswalk(s) to data; +\code{\link[=get_available_crosswalks]{get_available_crosswalks()}} for all supported geography/year +combinations; \code{\link[=list_nhgis_crosswalks]{list_nhgis_crosswalks()}} for the NHGIS-specific listing. } diff --git a/man/list_nhgis_crosswalks.Rd b/man/list_nhgis_crosswalks.Rd index fd0d758..b65e36c 100644 --- a/man/list_nhgis_crosswalks.Rd +++ b/man/list_nhgis_crosswalks.Rd @@ -7,15 +7,27 @@ list_nhgis_crosswalks() } \value{ -A tibble with columns: -\itemize{ -\item source_year: Year of the source geography -\item source_geography: Source geography name -\item target_year: Year of the target geography -\item target_geography: Target geography name +A tibble with one row per NHGIS crosswalk and columns: +\describe{ +\item{source_geography}{Source geography name (e.g., "block", "tract")} +\item{source_year}{Year of the source geography (character)} +\item{target_geography}{Target geography name} +\item{target_year}{Year of the target geography (character)} +\item{crosswalk_path}{URL of the underlying NHGIS crosswalk file} } } \description{ -Returns a tibble of all available NHGIS geographic crosswalks with their -corresponding parameters that can be used with get_nhgis_crosswalk(). +Returns a tibble of all IPUMS NHGIS crosswalks available through +\code{get_crosswalk()}. NHGIS provides the package's inter-temporal +(cross-decade) crosswalks; fetching them requires an \code{IPUMS_API_KEY} +(see \code{\link[=get_crosswalk]{get_crosswalk()}}). This listing itself is static and works offline. +} +\examples{ +# All NHGIS crosswalks from 2010 tracts +list_nhgis_crosswalks() |> + dplyr::filter(source_geography == "tract", source_year == "2010") +} +\seealso{ +\code{\link[=get_available_crosswalks]{get_available_crosswalks()}} for supported combinations across all +sources; \code{\link[=get_crosswalk]{get_crosswalk()}} to fetch a crosswalk. } diff --git a/renv/settings.json b/renv/settings.json index 8156db8..f299d93 100644 --- a/renv/settings.json +++ b/renv/settings.json @@ -1,19 +1,20 @@ -{ - "bioconductor.version": null, - "external.libraries": [], - "ignored.packages": [], - "package.dependency.fields": [ - "Imports", - "Depends", - "LinkingTo" - ], - "ppm.enabled": null, - "ppm.ignored.urls": [], - "r.version": null, - "snapshot.type": "explicit", - "use.cache": true, - "vcs.ignore.cellar": true, - "vcs.ignore.library": true, - "vcs.ignore.local": true, - "vcs.manage.ignores": true -} +{ + "bioconductor.version": null, + "external.libraries": [], + "ignored.packages": [], + "package.dependency.fields": [ + "Imports", + "Depends", + "LinkingTo" + ], + "ppm.enabled": null, + "ppm.ignored.urls": [], + "r.version": null, + "snapshot.dev": false, + "snapshot.type": "explicit", + "use.cache": true, + "vcs.ignore.cellar": true, + "vcs.ignore.library": true, + "vcs.ignore.local": true, + "vcs.manage.ignores": true +} diff --git a/tests/testthat/_problems/test-get_county_events_crosswalk-309.R b/tests/testthat/_problems/test-get_county_events_crosswalk-309.R new file mode 100644 index 0000000..230c90c --- /dev/null +++ b/tests/testthat/_problems/test-get_county_events_crosswalk-309.R @@ -0,0 +1,14 @@ +# Extracted from test-get_county_events_crosswalk.R:309 + +# test ------------------------------------------------------------------------- +skip_if_offline() +skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") +skip_if(Sys.getenv("CROSSWALK_RUN_SLOW_TESTS") != "true", + "Set CROSSWALK_RUN_SLOW_TESTS=true to run") +cache_dir <- file.path(tempdir(), "county-events-subcounty-cache") +dir.create(cache_dir, showWarnings = FALSE) +cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("tract", 2021, 2023, cache = cache_dir)) +ct <- cw |> dplyr::filter(state_fips == "09") +expect_true(nrow(ct) > 0) +expect_true(all(stringr::str_sub(ct$target_geoid, 3, 3) == "1")) diff --git a/tests/testthat/test-crosswalk_data.R b/tests/testthat/test-crosswalk_data.R index 9b45351..704c3cc 100644 --- a/tests/testthat/test-crosswalk_data.R +++ b/tests/testthat/test-crosswalk_data.R @@ -804,9 +804,35 @@ test_that("crosswalk_data show_join_quality=FALSE suppresses messages", { show_join_quality = FALSE), message = "did not match") - # join_quality attribute should be NULL when disabled + # join_quality attribute is still computed and attached when messages are off join_quality <- attr(result, "join_quality") - expect_null(join_quality) + expect_type(join_quality, "list") + expect_equal(join_quality$n_data_unmatched, 1) + expect_true("C" %in% join_quality$data_geoids_unmatched) +}) + +test_that("crosswalk_data silent=TRUE still attaches join_quality attribute", { + mock_data <- tibble::tibble( + geoid = c("A", "B", "C"), # C not in crosswalk + count_population = c(1000, 2000, 3000)) + + mock_crosswalk <- tibble::tibble( + source_geoid = c("A", "B"), + target_geoid = c("X", "X"), + allocation_factor_source_to_target = c(0.5, 0.5)) + + expect_no_message( + result <- crosswalk_data( + data = mock_data, + crosswalk = mock_crosswalk, + geoid_column = "geoid", + count_columns = c("count_population"), + silent = TRUE)) + + join_quality <- attr(result, "join_quality") + expect_type(join_quality, "list") + expect_equal(join_quality$n_data_unmatched, 1) + expect_true("C" %in% join_quality$data_geoids_unmatched) }) test_that("crosswalk_data silent when join is perfect", { @@ -969,3 +995,138 @@ test_that("crosswalk_data other columns work correctly with one-to-many mapping" expect_true(all(result$data_year == 2020)) }) +# ============================================================================== +# Guard tests: duplicated crosswalk pairs and duplicated data GEOIDs +# ============================================================================== + +test_that("crosswalk_data errors on crosswalks with multiple rows per source-target pair", { + mock_data <- tibble::tibble( + geoid = c("A"), + count_population = c(100)) + + # Long-format crosswalk: same pair repeated once per weighting factor. + # Summing across these rows would multiply-count values. + mock_crosswalk <- tibble::tibble( + source_geoid = c("A", "A"), + target_geoid = c("X", "X"), + weighting_factor = c("population", "household"), + allocation_factor_source_to_target = c(1, 1)) + + expect_error( + crosswalk_data( + data = mock_data, + crosswalk = mock_crosswalk, + geoid_column = "geoid", + count_columns = c("count_population")), + regexp = "multiple rows per source-target") +}) + +test_that("crosswalk_data errors on duplicated data GEOIDs (panel data)", { + # Panel data: same GEOID appears once per year + mock_data <- tibble::tibble( + geoid = c("A", "A", "B"), + data_year = c(2018, 2019, 2018), + count_population = c(100, 110, 200)) + + mock_crosswalk <- tibble::tibble( + source_geoid = c("A", "B"), + target_geoid = c("X", "X"), + target_geography_name = c("test", "test"), + allocation_factor_source_to_target = c(1, 1)) + + expect_error( + crosswalk_data( + data = mock_data, + crosswalk = mock_crosswalk, + geoid_column = "geoid", + count_columns = c("count_population")), + regexp = "duplicated GEOIDs") +}) + +test_that("crosswalk_data drops unmatched data rows without emitting an NA-geoid row", { + mock_data <- tibble::tibble( + geoid = c("A", "B", "C"), # C not in crosswalk + count_population = c(1000, 2000, 3000)) + + mock_crosswalk <- tibble::tibble( + source_geoid = c("A", "B"), + target_geoid = c("X", "X"), + target_geography_name = c("test", "test"), + allocation_factor_source_to_target = c(1, 1)) + + result <- crosswalk_data( + data = mock_data, + crosswalk = mock_crosswalk, + geoid_column = "geoid", + count_columns = c("count_population"), + silent = TRUE) + + expect_false(any(is.na(result$geoid))) + expect_equal(result$count_population, 3000) +}) + +test_that("crosswalk_data preserves user columns containing 'target_'", { + mock_data <- tibble::tibble( + geoid = c("A"), + count_population = c(1000), + count_target_units = c(50)) + + mock_crosswalk <- tibble::tibble( + source_geoid = c("A"), + target_geoid = c("X"), + target_geography_name = c("test"), + allocation_factor_source_to_target = c(1)) + + result <- crosswalk_data( + data = mock_data, + crosswalk = mock_crosswalk, + geoid_column = "geoid", + count_columns = c("count_population", "count_target_units")) + + expect_true("count_target_units" %in% colnames(result)) + expect_equal(result$count_target_units, 50) +}) + + +# ============================================================================== +# County-events style chains (mock data) +# ============================================================================== + +test_that("a mock events chain preserves counts and weights means", { + # Step 1: rename A -> B (identity factor), C unchanged + step_1 <- tibble::tibble( + source_geoid = c("A", "C"), + target_geoid = c("B", "C"), + target_geography_name = "test", + allocation_factor_source_to_target = c(1, 1)) + + # Step 2: B splits into X (0.7) and Y (0.3); C unchanged + step_2 <- tibble::tibble( + source_geoid = c("B", "B", "C"), + target_geoid = c("X", "Y", "C"), + target_geography_name = "test", + allocation_factor_source_to_target = c(0.7, 0.3, 1)) + + chain <- list(crosswalks = list(step_1 = step_1, step_2 = step_2)) + + data <- tibble::tibble( + source_geoid = c("A", "C"), + count_pop = c(1000, 500), + percent_owner = c(40, 60)) + + result <- crosswalk_data( + data = data, + crosswalk = chain, + geoid_column = "source_geoid", + show_join_quality = FALSE, + silent = TRUE) + + expect_setequal(result$geoid, c("X", "Y", "C")) + # Counts split by allocation and preserved in total + expect_equal(result$count_pop[result$geoid == "X"], 700) + expect_equal(result$count_pop[result$geoid == "Y"], 300) + expect_equal(sum(result$count_pop), sum(data$count_pop)) + # Non-count variables carry through the relabel/split unchanged + expect_equal(result$percent_owner[result$geoid == "X"], 40) + expect_equal(result$percent_owner[result$geoid == "C"], 60) +}) diff --git a/tests/testthat/test-get_available_crosswalks.R b/tests/testthat/test-get_available_crosswalks.R index b409c54..b1fb630 100644 --- a/tests/testthat/test-get_available_crosswalks.R +++ b/tests/testthat/test-get_available_crosswalks.R @@ -8,10 +8,11 @@ test_that("get_available_crosswalks returns a tibble with expected columns", { result <- get_available_crosswalks() expect_s3_class(result, "tbl_df") - expect_equal(ncol(result), 4) + expect_equal(ncol(result), 5) expect_equal( colnames(result), - c("source_geography", "target_geography", "source_year", "target_year")) + c("source_geography", "target_geography", "source_year", "target_year", + "crosswalk_source")) }) test_that("get_available_crosswalks has correct column types", { @@ -21,6 +22,9 @@ test_that("get_available_crosswalks has correct column types", { expect_type(result$target_geography, "character") expect_type(result$source_year, "integer") expect_type(result$target_year, "integer") + expect_type(result$crosswalk_source, "character") + expect_true(all(result$crosswalk_source %in% + c("geocorr", "nhgis", "ctdata_2020_2022", "county_events"))) }) test_that("get_available_crosswalks has no duplicate rows", { @@ -29,6 +33,45 @@ test_that("get_available_crosswalks has no duplicate rows", { expect_equal(nrow(result), nrow(dplyr::distinct(result))) }) +test_that("get_available_crosswalks lists each combination exactly once", { + result <- get_available_crosswalks() + + combos <- result |> + dplyr::count(source_geography, target_geography, source_year, target_year) |> + dplyr::filter(n > 1) + + expect_equal(nrow(combos), 0) +}) + +test_that("get_available_crosswalks routes 2020->2022 pairs to CTData", { + result <- get_available_crosswalks() + + # These combinations are also servable by the county-events engine; the + # listing should mirror the planner, which routes direct requests to CTData + ct_pair <- result |> + dplyr::filter( + source_geography == "county", + target_geography == "county", + source_year == 2020L, + target_year == 2022L) + + expect_equal(ct_pair$crosswalk_source, "ctdata_2020_2022") +}) + +test_that("get_available_crosswalks contains county-events rows", { + result <- get_available_crosswalks() + + county_events_match <- result |> + dplyr::filter( + source_geography == "county", + target_geography == "county", + source_year == 2014L, + target_year == 2023L) + + expect_equal(nrow(county_events_match), 1) + expect_equal(county_events_match$crosswalk_source, "county_events") +}) + test_that("get_available_crosswalks has no same-year self-pairs for Geocorr rows", { result <- get_available_crosswalks() diff --git a/tests/testthat/test-get_county_events_crosswalk.R b/tests/testthat/test-get_county_events_crosswalk.R new file mode 100644 index 0000000..ddc0da7 --- /dev/null +++ b/tests/testthat/test-get_county_events_crosswalk.R @@ -0,0 +1,362 @@ +# ============================================================================== +# Curated data validation (sysdata) +# ============================================================================== + +test_that("curated county events data is internally consistent", { + events <- crosswalk:::county_events + mappings <- crosswalk:::county_event_mappings + universes <- crosswalk:::county_universes + meta <- crosswalk:::county_events_meta + + # Allocation factors sum to 1 per source county within each event + factor_sums <- mappings |> + dplyr::summarize(total = sum(allocation_factor), .by = c(event_id, source_geoid)) + expect_true(all(abs(factor_sums$total - 1) < 1e-9)) + + # GEOIDs are 5-character county codes + expect_true(all(nchar(mappings$source_geoid) == 5)) + expect_true(all(nchar(mappings$target_geoid) == 5)) + expect_true(all(nchar(universes$geoid) == 5)) + + # Every event has county-level mappings, and vice versa + expect_setequal(events$event_id, mappings$event_id) + + # Valid categorical values and vintages + expect_true(all(events$event_type %in% + c("rename", "merge", "split", "part_transfer"))) + expect_true(all(events$subcounty_handling %in% + c("prefix_swap", "explicit_rows", "ctdata_runtime", "none", "unsupported"))) + expect_true(all(events$effective_vintage_year > 2000)) + expect_true(all(events$effective_vintage_year <= meta$known_through_year)) + + # Universes at all three decennial bases + expect_setequal(unique(universes$decade_base), c(2000L, 2010L, 2020L)) +}) + + +# ============================================================================== +# county_universe_at() +# ============================================================================== + +test_that("county_universe_at rolls universes through events", { + u2014 <- crosswalk:::county_universe_at(2014) + u2016 <- crosswalk:::county_universe_at(2016) + u2003 <- crosswalk:::county_universe_at(2003) + u2021 <- crosswalk:::county_universe_at(2021) + u2023 <- crosswalk:::county_universe_at(2023) + + # 2014: Bedford city merged (vintage 2014), Shannon County not yet renamed + expect_false("51515" %in% u2014) + expect_true("46113" %in% u2014) + expect_false("46102" %in% u2014) + + # 2016: renames applied + expect_true("46102" %in% u2016) + expect_false("46113" %in% u2016) + expect_true("02158" %in% u2016) + expect_false("02270" %in% u2016) + + # 2003: Broomfield exists, Clifton Forge gone, AK reorganizations not yet + expect_true("08014" %in% u2003) + expect_false("51560" %in% u2003) + expect_true("02232" %in% u2003) + expect_false("02105" %in% u2003) + + # 2021: Valdez-Cordova split (vintage 2020) applied; CT counties still present + expect_true(all(c("02063", "02066") %in% u2021)) + expect_false("02261" %in% u2021) + expect_true("09001" %in% u2021) + + # 2023: CT planning regions replace counties + expect_true("09110" %in% u2023) + expect_false("09001" %in% u2023) +}) + + +# ============================================================================== +# County-level crosswalks (offline) +# ============================================================================== + +test_that("identity interval returns a full identity crosswalk with a message", { + expect_message( + cw <- crosswalk:::get_county_events_crosswalk("county", 2003, 2005), + "No county boundary or code changes") + + expect_equal(nrow(cw), length(crosswalk:::county_universe_at(2003))) + expect_true(all(cw$source_geoid == cw$target_geoid)) + expect_true(all(cw$allocation_factor_source_to_target == 1)) + expect_true(all(cw$weighting_factor == "identity")) +}) + +test_that("rename interval maps old to new FIPS codes", { + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2014, 2016)) + + expect_equal(cw$target_geoid[cw$source_geoid == "46113"], "46102") + expect_equal(cw$target_geoid[cw$source_geoid == "02270"], "02158") + expect_false("46102" %in% cw$source_geoid) + expect_false("46113" %in% cw$target_geoid) + + # Everything else is identity + others <- cw |> dplyr::filter(!source_geoid %in% c("46113", "02270")) + expect_true(all(others$source_geoid == others$target_geoid)) + + # One row per source-target pair + expect_equal(anyDuplicated(cw[c("source_geoid", "target_geoid")]), 0L) +}) + +test_that("split interval fans out with population factors summing to 1", { + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2018, 2021)) + + valdez <- cw |> dplyr::filter(source_geoid == "02261") + expect_setequal(valdez$target_geoid, c("02063", "02066")) + expect_equal(sum(valdez$allocation_factor_source_to_target), 1) + expect_true(all(valdez$weighting_factor == "population")) + expect_false("02261" %in% cw$target_geoid) +}) + +test_that("multi-event intervals compose factors correctly", { + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2006, 2020)) + + # Factors still sum to 1 per source across composed events + sums <- cw |> + dplyr::summarize( + total = sum(allocation_factor_source_to_target), .by = source_geoid) + expect_true(all(abs(sums$total - 1) < 1e-9)) + + # Skagway-Hoonah-Angoon (02232) fans through the 2007 split and the 2013 + # Petersburg transfer: 02230, 02105, and a sliver reaching 02195 + skagway <- cw |> dplyr::filter(source_geoid == "02232") + expect_setequal(skagway$target_geoid, c("02105", "02195", "02230")) + + # Bedford city merged + expect_equal(cw$target_geoid[cw$source_geoid == "51515"], "51019") +}) + +test_that("county-events crosswalk has the standard column contract", { + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2014, 2016)) + + expect_named(cw, c( + "source_geoid", "target_geoid", + "source_geography_name", "target_geography_name", + "source_year", "target_year", + "allocation_factor_source_to_target", "weighting_factor", "state_fips")) + expect_type(cw$source_geoid, "character") + expect_type(cw$source_year, "character") + expect_type(cw$allocation_factor_source_to_target, "double") + expect_equal(unique(cw$source_year), "2014") + expect_equal(unique(cw$target_year), "2016") + + metadata <- attr(cw, "crosswalk_metadata") + expect_equal(metadata$data_source, "county_events") + expect_true(any(stringr::str_detect(metadata$notes, "Event applied"))) +}) + +test_that("cache round-trip preserves types and leading zeros", { + cache_dir <- file.path(tempdir(), "county-events-test-cache") + dir.create(cache_dir, showWarnings = FALSE) + on.exit(unlink(cache_dir, recursive = TRUE)) + + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2014, 2016, cache = cache_dir)) + cw_cached <- suppressMessages( + crosswalk:::get_county_events_crosswalk("county", 2014, 2016, cache = cache_dir)) + + expect_true(attr(cw_cached, "crosswalk_metadata")$read_from_cache) + expect_type(cw_cached$source_geoid, "character") + expect_true(all(stringr::str_detect(cw_cached$source_geoid[ + stringr::str_starts(cw_cached$source_geoid, "0")], "^0"))) + expect_equal( + cw |> dplyr::arrange(source_geoid, target_geoid), + cw_cached |> dplyr::arrange(source_geoid, target_geoid), + ignore_attr = TRUE) +}) + + +# ============================================================================== +# Validation errors +# ============================================================================== + +test_that("reverse county requests error with a targeted message", { + expect_error( + crosswalk:::get_county_events_crosswalk("county", 2019, 2014), + "forward") +}) + +test_that("out-of-range years error", { + expect_error( + crosswalk:::get_county_events_crosswalk("county", 1999, 2010), + "2000 through") + expect_error( + crosswalk:::get_county_events_crosswalk("county", 2010, 2099), + "2000 through") +}) + +test_that("identical years error", { + expect_error( + crosswalk:::get_county_events_crosswalk("county", 2014, 2014), + "identical") +}) + +test_that("sub-county requests before 2010 error", { + expect_error( + crosswalk:::get_county_events_crosswalk("tract", 2005, 2008), + "2010 and later") +}) + +test_that("sub-county requests spanning a decennial census error", { + expect_error( + crosswalk:::get_county_events_crosswalk("tract", 2014, 2023), + "same tract-definition era") +}) + +test_that("unsupported geographies error", { + expect_error( + crosswalk:::get_county_events_crosswalk("zcta", 2014, 2016), + "not supported") +}) + + +# ============================================================================== +# list_county_events_edges() +# ============================================================================== + +test_that("county edges cover all forward pairs in range", { + edges <- crosswalk:::list_county_events_edges("county") + known_through <- crosswalk:::county_events_meta$known_through_year + n_years <- known_through - 2000 + 1 + + expect_equal(nrow(edges), n_years * (n_years - 1) / 2) + expect_true(all(as.numeric(edges$source_year) < as.numeric(edges$target_year))) + expect_true(nrow(dplyr::filter(edges, source_year == "2014", target_year == "2019")) == 1) + expect_true(nrow(dplyr::filter(edges, source_year == "2014", target_year == "2023")) == 1) +}) + +test_that("sub-county edges respect eras and reversal-exactness", { + edges <- crosswalk:::list_county_events_edges("tract") + + # Forward same-era pairs exist + expect_true(nrow(dplyr::filter(edges, source_year == "2014", target_year == "2019")) == 1) + expect_true(nrow(dplyr::filter(edges, source_year == "2021", target_year == "2023")) == 1) + + # No cross-era pairs + expect_equal( + nrow(dplyr::filter(edges, source_year == "2014", target_year == "2023")), 0) + + # Backward pair over exact relabels (2015 renames) exists + expect_true(nrow(dplyr::filter(edges, source_year == "2017", target_year == "2014")) == 1) + + # Backward pair over non-exact events (2014 Bedford/Petersburg) excluded + expect_equal( + nrow(dplyr::filter(edges, source_year == "2017", target_year == "2013")), 0) + + # No edges before 2010 for sub-county geographies + expect_true(all(as.numeric(edges$source_year) >= 2010)) + + # Unsupported geographies return no edges + expect_equal(nrow(crosswalk:::list_county_events_edges("zcta")), 0) +}) + + +# ============================================================================== +# Sub-county crosswalks (network + API keys required) +# ============================================================================== + +test_that("tract rename interval applies county prefix swaps", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + skip_if(Sys.getenv("CROSSWALK_RUN_SLOW_TESTS") != "true", + "Set CROSSWALK_RUN_SLOW_TESTS=true to run") + + cache_dir <- file.path(tempdir(), "county-events-subcounty-cache") + dir.create(cache_dir, showWarnings = FALSE) + + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("tract", 2014, 2016, cache = cache_dir)) + + shannon <- cw |> dplyr::filter(stringr::str_starts(source_geoid, "46113")) + expect_true(nrow(shannon) > 0) + expect_true(all(stringr::str_starts(shannon$target_geoid, "46102"))) + expect_equal( + stringr::str_sub(shannon$source_geoid, 6), + stringr::str_sub(shannon$target_geoid, 6)) + expect_true(all(shannon$allocation_factor_source_to_target == 1)) + + # Non-affected tracts are identity + other <- cw |> dplyr::filter(!stringr::str_starts(source_geoid, "46113"), + !stringr::str_starts(source_geoid, "02270")) + expect_true(all(other$source_geoid == other$target_geoid)) +}) + +test_that("tract 2021 -> 2023 composes the CT planning region relabels", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + skip_if(Sys.getenv("CROSSWALK_RUN_SLOW_TESTS") != "true", + "Set CROSSWALK_RUN_SLOW_TESTS=true to run") + + cache_dir <- file.path(tempdir(), "county-events-subcounty-cache") + dir.create(cache_dir, showWarnings = FALSE) + + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("tract", 2021, 2023, cache = cache_dir)) + + ct <- cw |> dplyr::filter(state_fips == "09") + expect_true(nrow(ct) > 0) + # CT tracts move from county prefixes (090xx) to planning region prefixes (091xx) + expect_true(all(stringr::str_sub(ct$target_geoid, 3, 3) == "1")) + expect_true(all(ct$allocation_factor_source_to_target == 1)) + + # Non-CT tracts are identity + other <- cw |> dplyr::filter(state_fips != "09") + expect_true(all(other$source_geoid == other$target_geoid)) +}) + + +# ============================================================================== +# Consistency with the CTData county crosswalk (network + API keys required) +# ============================================================================== + +test_that("sysdata CT county factors match get_ctdata_crosswalk", { + skip_if_offline() + skip_if_not_installed("tidycensus") + skip_if(Sys.getenv("CENSUS_API_KEY") == "", "CENSUS_API_KEY not set") + + ctdata <- suppressMessages( + crosswalk:::get_ctdata_crosswalk("county", 2020, 2022)) |> + dplyr::filter(stringr::str_starts(source_geoid, "09")) |> + dplyr::arrange(source_geoid, target_geoid) + + sysdata_ct <- crosswalk:::county_event_mappings |> + dplyr::filter(event_id == "2022_ct_planning_regions") |> + dplyr::arrange(source_geoid, target_geoid) + + expect_equal(ctdata$source_geoid, sysdata_ct$source_geoid) + expect_equal(ctdata$target_geoid, sysdata_ct$target_geoid) + expect_equal( + ctdata$allocation_factor_source_to_target, + sysdata_ct$allocation_factor, + tolerance = 1e-6) +}) + +test_that("tract 2023 -> 2021 reverses the CT relabels exactly", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + skip_if(Sys.getenv("CROSSWALK_RUN_SLOW_TESTS") != "true", + "Set CROSSWALK_RUN_SLOW_TESTS=true to run") + + cache_dir <- file.path(tempdir(), "county-events-subcounty-cache") + dir.create(cache_dir, showWarnings = FALSE) + + cw <- suppressMessages( + crosswalk:::get_county_events_crosswalk("tract", 2023, 2021, cache = cache_dir)) + + ct <- cw |> dplyr::filter(state_fips == "09") + expect_true(nrow(ct) > 0) + # Sources are planning-region prefixed, targets county prefixed + expect_true(all(stringr::str_sub(ct$source_geoid, 3, 3) == "1")) + expect_true(all(stringr::str_sub(ct$target_geoid, 3, 3) == "0")) + expect_true(all(cw$allocation_factor_source_to_target == 1)) +}) diff --git a/tests/testthat/test-get_crosswalk.R b/tests/testthat/test-get_crosswalk.R index 193d80a..9ae982d 100644 --- a/tests/testthat/test-get_crosswalk.R +++ b/tests/testthat/test-get_crosswalk.R @@ -352,3 +352,139 @@ test_that("get_crosswalk routes no-year to GeoCorr 2022", { expect_equal(metadata$data_source, "geocorr") expect_equal(metadata$reference_year, "2022") }) + +test_that("get_crosswalk uses source year for GeoCorr version when target year is NULL", { + skip_if_offline() + + result <- get_crosswalk( + source_geography = "tract", + target_geography = "zcta", + source_year = 2015, + weight = "population") + + metadata <- attr(result$crosswalks$step_1, "crosswalk_metadata") + expect_equal(metadata$data_source, "geocorr") + expect_equal(metadata$reference_year, "2018") +}) + +# ============================================================================== +# NHGIS weight filtering and mass conservation tests +# ============================================================================== + +test_that("get_crosswalk filters NHGIS crosswalks to the requested weight", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + + result <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2010, + target_year = 2020, + weight = "population", + silent = TRUE) + + crosswalk <- result$crosswalks$step_1 + + # Exactly one weighting factor, matching the request + expect_equal(unique(crosswalk$weighting_factor), "population") + + # Exactly one row per source-target pair + expect_equal( + anyDuplicated(crosswalk[c("source_geoid", "target_geoid")]), + 0L) + + # Allocation factors for each source geography should sum to ~1 + allocation_mass <- crosswalk |> + dplyr::summarize( + mass = sum(allocation_factor_source_to_target), + .by = source_geoid) + expect_equal(mean(allocation_mass$mass), 1, tolerance = 0.01) +}) + +test_that("crosswalk_data conserves total counts through an NHGIS crosswalk", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + + result <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2010, + target_year = 2020, + weight = "population", + silent = TRUE) + + # One unit of count per source tract: the crosswalked total must equal the + # number of source tracts (allocation factors sum to 1 per source) + mock_data <- tibble::tibble( + source_geoid = unique(result$crosswalks$step_1$source_geoid), + count_value = 1) + + output <- crosswalk_data( + data = mock_data, + crosswalk = result, + geoid_column = "source_geoid", + count_columns = "count_value", + silent = TRUE) + + expect_equal( + sum(output$count_value), + nrow(mock_data), + tolerance = 0.001) +}) + +test_that("get_crosswalk selects housing weight for NHGIS crosswalks", { + skip_if_offline() + skip_if(Sys.getenv("IPUMS_API_KEY") == "", "IPUMS_API_KEY not set") + + result <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2010, + target_year = 2020, + weight = "housing", + silent = TRUE) + + crosswalk <- result$crosswalks$step_1 + expect_equal(unique(crosswalk$weighting_factor), "housing_all") +}) + +# ============================================================================== +# County-events routing +# ============================================================================== + +test_that("county temporal requests route to the county-events engine", { + result <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2014, + target_year = 2019, + silent = TRUE) + + expect_equal(length(result$crosswalks), 1) + crosswalk <- result$crosswalks$step_1 + metadata <- attr(crosswalk, "crosswalk_metadata") + + expect_equal(metadata$data_source, "county_events") + expect_equal(anyDuplicated(crosswalk[c("source_geoid", "target_geoid")]), 0L) + expect_equal(crosswalk$target_geoid[crosswalk$source_geoid == "46113"], "46102") +}) + +test_that("county 2014 -> 2023 spans the decennial in a single events step", { + result <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2014, + target_year = 2023, + silent = TRUE) + + expect_equal(length(result$crosswalks), 1) + crosswalk <- result$crosswalks$step_1 + + # CT planning regions and the Valdez-Cordova successors are targets + expect_true(all(c("09110", "02063", "02066") %in% crosswalk$target_geoid)) + # Factors sum to 1 per source + sums <- crosswalk |> + dplyr::summarize( + total = sum(allocation_factor_source_to_target), .by = source_geoid) + expect_true(all(abs(sums$total - 1) < 1e-9)) +}) diff --git a/tests/testthat/test-plan_crosswalk_chain.R b/tests/testthat/test-plan_crosswalk_chain.R index 9b06d68..6905c36 100644 --- a/tests/testthat/test-plan_crosswalk_chain.R +++ b/tests/testthat/test-plan_crosswalk_chain.R @@ -590,3 +590,149 @@ test_that("plan_crosswalk_chain composition note covers all steps in 3-step plan expect_true(stringr::str_detect(plan$composition_note, "step2_allocation")) expect_true(stringr::str_detect(plan$composition_note, "step3_allocation")) }) + +# ============================================================================== +# County-events temporal plan tests +# ============================================================================== + +test_that("county temporal requests plan a single county-events step", { + for (years in list(c(2014, 2019), c(2014, 2023), c(2006, 2023))) { + plan <- plan_crosswalk_chain( + source_geography = "county", + target_geography = "county", + source_year = years[1], + target_year = years[2]) + + expect_null(plan$error) + expect_false(plan$is_multi_step) + expect_equal(nrow(plan$steps), 1) + expect_equal(plan$steps$crosswalk_source[1], "county_events") + expect_equal(plan$steps$source_year[1], as.character(years[1])) + expect_equal(plan$steps$target_year[1], as.character(years[2])) + } +}) + +test_that("county 2020 -> 2022 still routes to CTData (back-compat)", { + plan <- plan_crosswalk_chain( + source_geography = "county", + target_geography = "county", + source_year = 2020, + target_year = 2022) + + expect_equal(plan$steps$crosswalk_source[1], "ctdata_2020_2022") +}) + +test_that("reverse county temporal requests error with a targeted message", { + plan <- plan_crosswalk_chain( + source_geography = "county", + target_geography = "county", + source_year = 2019, + target_year = 2014) + + expect_match(plan$error, "forward-only") +}) + +test_that("out-of-range county temporal requests error with the range", { + plan <- plan_crosswalk_chain( + source_geography = "county", + target_geography = "county", + source_year = 1990, + target_year = 2010) + + expect_match(plan$error, "2000 through") +}) + +test_that("tract same-decade temporal requests plan a single county-events step", { + plan <- plan_crosswalk_chain( + source_geography = "tract", + target_geography = "tract", + source_year = 2014, + target_year = 2019) + + expect_null(plan$error) + expect_equal(nrow(plan$steps), 1) + expect_equal(plan$steps$crosswalk_source[1], "county_events") +}) + +test_that("tract 2014 -> 2020 prefers the direct NHGIS crosswalk on ties", { + plan <- plan_crosswalk_chain( + source_geography = "tract", + target_geography = "tract", + source_year = 2014, + target_year = 2020) + + expect_equal(nrow(plan$steps), 1) + expect_equal(plan$steps$crosswalk_source[1], "nhgis") +}) + +test_that("tract 2014 -> 2023 chains NHGIS with a county-events step", { + plan <- plan_crosswalk_chain( + source_geography = "tract", + target_geography = "tract", + source_year = 2014, + target_year = 2023) + + expect_null(plan$error) + expect_true(plan$is_multi_step) + expect_equal(plan$steps$crosswalk_source[1], "nhgis") + expect_equal( + plan$steps$crosswalk_source[nrow(plan$steps)], "county_events") + expect_equal(plan$steps$source_year[1], "2014") + expect_equal(plan$steps$target_year[nrow(plan$steps)], "2023") +}) + +test_that("tract 2017 -> 2023 uses a backward relabel to reach an NHGIS vintage", { + plan <- plan_crosswalk_chain( + source_geography = "tract", + target_geography = "tract", + source_year = 2017, + target_year = 2023) + + expect_null(plan$error) + expect_true(plan$is_multi_step) + # First hop is a county-events relabel backward to an NHGIS-served vintage + expect_equal(plan$steps$crosswalk_source[1], "county_events") + expect_true( + as.numeric(plan$steps$target_year[1]) < as.numeric(plan$steps$source_year[1])) + expect_true("nhgis" %in% plan$steps$crosswalk_source) + expect_equal(plan$steps$target_year[nrow(plan$steps)], "2023") +}) + +test_that("block group 2000s temporal requests error (no path)", { + plan <- plan_crosswalk_chain( + source_geography = "block group", + target_geography = "block group", + source_year = 2004, + target_year = 2016) + + expect_match(plan$error, "No temporal crosswalk path") +}) + +test_that("county source with geography change plans events + geocorr steps", { + plan <- plan_crosswalk_chain( + source_geography = "county", + target_geography = "puma", + source_year = 2014, + target_year = 2019) + + expect_null(plan$error) + expect_true(plan$is_multi_step) + expect_equal(plan$steps$crosswalk_source, c("county_events", "geocorr")) + expect_equal(plan$steps$target_year[1], "2019") +}) + +test_that("find_temporal_path serves county via a tagged county-events hop", { + path <- crosswalk:::find_temporal_path("county", "2014", "2019") + + expect_equal(length(path), 1) + expect_equal(path[[1]]$source_year, "2014") + expect_equal(path[[1]]$target_year, "2019") + expect_equal(path[[1]]$crosswalk_source, "county_events") +}) + +test_that("find_temporal_path tags NHGIS hops", { + path <- crosswalk:::find_temporal_path("tract", "2010", "2020") + + expect_equal(length(path), 1) + expect_equal(path[[1]]$crosswalk_source, "nhgis") +}) diff --git a/vignettes/county-crosswalks.Rmd b/vignettes/county-crosswalks.Rmd new file mode 100644 index 0000000..a849fed --- /dev/null +++ b/vignettes/county-crosswalks.Rmd @@ -0,0 +1,183 @@ +--- +title: "County Crosswalks Between Any Years" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{County Crosswalks Between Any Years} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + message = FALSE, + warning = FALSE) +``` + +Most geographic change in the census system happens on decennial schedules -- +but counties don't wait for the decade. Since 2000, counties and county +equivalents have been created, merged, split, renamed, and transferred between +one another a dozen-plus times. A county panel from, say, 2014 to 2023 that +ignores these events will silently misalign Alaska, South Dakota, Virginia, +and all of Connecticut. + +The `crosswalk` package curates these events from the Census Bureau's +[Substantial Changes to Counties](https://www.census.gov/programs-surveys/geography/technical-documentation/county-changes.html) +documentation and ships them as internal package data, so county -> county +crosswalks work between **any** pair of years from 2000 onward -- offline, +with no API key. + +```{r setup} +library(crosswalk) +library(dplyr) +``` + +## What's covered + +| Event | Type | First vintage with new codes | +|---|---|---| +| Clifton Forge, VA gives up independent-city status | Merge | 2002 | +| Broomfield County, CO created | Split (from parts of four counties) | 2002 | +| Southeast Alaska reorganizations (Skagway, Hoonah-Angoon, Wrangell, Petersburg, Prince of Wales) | Splits/merges | 2008-2009 | +| York County / Newport News, VA territory exchange | Part transfer | 2008 | +| Bedford City, VA rejoins Bedford County | Merge | 2014 | +| Petersburg Borough, AK created | Split/merge | 2014 | +| Shannon County, SD renamed Oglala Lakota County (46113 -> 46102) | FIPS change | 2015 | +| Wade Hampton, AK renamed Kusilvak (02270 -> 02158) | FIPS change | 2015 | +| Valdez-Cordova, AK split into Chugach and Copper River | Split | 2020 | +| Connecticut counties replaced by nine planning regions | Reorganization | 2022 | + +A subtlety worth internalizing: the years above are Census Bureau *product +vintages*, not effective dates. The Valdez-Cordova split took legal effect on +January 2, 2019, but 2019-vintage ACS products still use the old county; the +new codes first appear in 2020-vintage products. The package keys every event +to the first product vintage that uses the post-change codes (verified +empirically against ACS county inventories), because that's what determines +which codes appear in the data you're crosswalking. + +## Basic usage + +Request any forward year pair. The result composes all events in the interval +on top of an identity mapping over the full county universe -- unchanged +counties appear with an allocation factor of 1, so the crosswalk is never +sparse: + +```{r county-basic} +county_14_23 <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2014, + target_year = 2023, + silent = TRUE) + +crosswalk_14_23 <- county_14_23$crosswalks$step_1 + +nrow(crosswalk_14_23) + +## the counties that changed between 2014 and 2023 +crosswalk_14_23 |> + filter(source_geoid != target_geoid) +``` + +Renames (like 46113 -> 46102) carry an allocation factor of 1 with an +`identity` weighting factor. Splits allocate the source county across its +successors with population-based factors: + +```{r split-example} +county_19_21 <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2019, + target_year = 2021, + silent = TRUE) + +## Valdez-Cordova (02261) split into Chugach (02063) and Copper River (02066) +county_19_21$crosswalks$step_1 |> + filter(source_geoid == "02261") +``` + +Applying these crosswalks works exactly as described in +`vignette("crosswalk")`: pass the result to `crosswalk_data()` with your +county-level data. + +## Forward-only, with one exception + +County temporal crosswalks require `source_year < target_year`: + +```{r backward-error, error = TRUE} +get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2021, + target_year = 2019) +``` + +The asymmetry is inherent: composing a *merge* forward is just addition, but +reversing it would require disaggregating the merged county's values back into +its components, which a crosswalk row cannot honestly express without +additional data. Rather than return allocation factors that look precise but +aren't, the package asks you to standardize *forward* to the latest year in +your panel. + +## Sub-county geographies + +County changes ripple into the GEOIDs of everything nested inside a county: +when Shannon County became Oglala Lakota, every tract, block group, and block +in `46113...` was relabeled to `46102...`. The package serves these relabels +for same-decade year pairs from 2010 onward: + +```{r tract-relabel} +tract_14_19 <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2014, + target_year = 2019, + silent = TRUE) + +## identity rows for all unchanged tracts, relabels for the affected ones +tract_14_19$crosswalks$step_1 |> + filter(source_geoid != target_geoid) |> + head() +``` + +Because these sub-county changes are exact 1:1 relabels, they are also the +one case where *backward* temporal crosswalks are offered (e.g., tract +2019 -> 2014) -- reversing a relabel is lossless in a way reversing a merge +is not. + +Year pairs that span a decennial census (e.g., tract 2014 -> 2023) can't be +pure relabels, because tract boundaries themselves changed in 2020. For those, +`get_crosswalk()` automatically plans a chain that routes through NHGIS +cross-decade crosswalks (and so requires an `IPUMS_API_KEY`): + +```{r cross-decade, eval = FALSE} +## planned automatically as: 2014 tracts -[relabel]-> 2010-decade vintage +## -[NHGIS]-> 2020-decade vintage -[relabel]-> 2023 tracts +tract_14_23 <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2014, + target_year = 2023) +``` + +## Connecticut, briefly + +Connecticut's 2022 replacement of its eight counties with nine +planning-region county equivalents is the most sweeping recent change -- +every Connecticut county GEOID changed, and the old and new units don't nest. +Direct 2020 <-> 2022 requests are served by crosswalks from the CT Data +Collaborative (county-level requests fetch county GEOIDs via `tidycensus` and +so need a `CENSUS_API_KEY`); county requests spanning 2022 (like the +2014 -> 2023 example above) incorporate the same change through the curated +event registry, with population-based allocation factors where old counties +split across planning regions. + +## Where the data comes from + +The event registry, GEOID mappings, and year-by-year county universes are +built by `data-raw/build_county_events_sysdata.R` from Census Bureau +documentation and ACS county inventories, and ship as internal package data +(`R/sysdata.rda`). See +[`data-raw/README.md`](https://github.com/UI-Research/crosswalk/blob/main/data-raw/README.md) +for provenance details. diff --git a/vignettes/crosswalk.Rmd b/vignettes/crosswalk.Rmd new file mode 100644 index 0000000..d768b9f --- /dev/null +++ b/vignettes/crosswalk.Rmd @@ -0,0 +1,353 @@ +--- +title: "Getting Started with crosswalk" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Getting Started with crosswalk} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +## chunks that pull data from the Census API only evaluate when a key is set +has_census_key <- Sys.getenv("CENSUS_API_KEY") != "" + +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + message = FALSE, + warning = FALSE) +``` + +## Overview + +Census geographies change. Tracts are redrawn every decade, counties are +occasionally renamed, merged, or split, and many analyses need to move data +between geographies (say, from ZIP Code Tabulation Areas to PUMAs) that don't +nest within one another. A *crosswalk* is a table that relates source +geographies to target geographies, with allocation factors describing what +share of each source unit belongs to each target unit. + +The `crosswalk` package provides a consistent interface to crosswalks from +four sources, plus tooling to apply them to your data: + +| Source | What it covers | API key | +|---|---|---| +| **GeoCorr** (Missouri Census Data Center) | Same-year crosswalks between geographies (GeoCorr 2022 for 2020s geography; GeoCorr 2018 for 2010s geography) | None | +| **IPUMS NHGIS** | Cross-decade crosswalks (e.g., 2010 tracts to 2020 tracts) | `IPUMS_API_KEY` | +| **CT Data Collaborative** | Connecticut's 2020-2022 change from counties to planning regions | `CENSUS_API_KEY` (county requests only) | +| **County events** (curated, ships with the package) | County redefinitions outside decennial censuses, supporting county crosswalks between any years from 2000 on | None | + +Two functions do most of the work: + +- `get_crosswalk()` fetches crosswalk(s), automatically chaining multiple + sources when both geography and year change +- `crosswalk_data()` applies crosswalk(s) to your data + +This vignette walks through the core workflow. The examples that query the +Census Bureau's API require a `CENSUS_API_KEY`; if you're missing outputs +below, that's why. + +```{r setup} +library(crosswalk) +library(dplyr) +``` + +## Fetching your first crosswalk + +Suppose we have county-level data and need it at the PUMA (Public Use +Microdata Area) level. Counties don't nest within PUMAs, so we need a +crosswalk. With no years specified, `get_crosswalk()` uses GeoCorr 2022 +(2020-Census geography): + +```{r fetch-first} +## save fetched crosswalks to a directory so repeat requests read from disk; +## see the Caching section at the end of this vignette +cache_dir <- file.path(tempdir(), "crosswalk-cache") + +county_to_puma <- get_crosswalk( + source_geography = "county", + target_geography = "puma", + weight = "population", + cache = cache_dir) +``` + +`get_crosswalk()` always returns a list with three elements: + +| Element | Description | +|---|---| +| `crosswalks` | A named list of crosswalk tibbles (`step_1`, `step_2`, ...) | +| `plan` | The plan describing each step: its source, geographies, and years | +| `message` | A human-readable description of the crosswalk chain | + +```{r inspect-structure} +county_to_puma$message + +county_to_puma$crosswalks$step_1 |> + head() +``` + +Every crosswalk uses the same standardized columns, whatever its source: + +| Column | Description | +|---|---| +| `source_geoid`, `target_geoid` | Identifiers for the source and target geographies | +| `allocation_factor_source_to_target` | The share of the source unit's `weight` allocated to the target unit | +| `weighting_factor` | The attribute used to compute allocation factors (e.g., `population`, `housing_all`, `land`, `identity`) | + +A county entirely inside one PUMA has a single row with an allocation factor +of 1; a county split across PUMAs has one row per PUMA, with allocation +factors that sum to 1. Crosswalks always contain exactly one row per +source-target pair. + +The `weight` argument ("population", "housing", or "land") determines *how* +split units are apportioned -- population-weighted, housing-unit-weighted, or +area-weighted. Match it to what you're crosswalking: population-based weights +for people-denominated variables, housing for housing-unit variables. See +`vignette("how-interpolation-works")` for details. + +### Where did this crosswalk come from? + +Each crosswalk tibble carries a `crosswalk_metadata` attribute documenting its +provenance -- the source, the parameters used, download URLs, citation +guidance, and more (see `?get_crosswalk` for the full structure): + +```{r metadata} +metadata <- attr(county_to_puma$crosswalks$step_1, "crosswalk_metadata") + +names(metadata) + +metadata$data_source_full_name +``` + +## Applying a crosswalk to data + +`crosswalk_data()` applies a crosswalk to your data. Let's fetch county-level +poverty counts from the American Community Survey and translate them to PUMAs: + +```{r acs-pull, eval = has_census_key} +county_poverty <- tidycensus::get_acs( + year = 2023, + geography = "county", + output = "wide", + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> + select( + source_geoid = GEOID, + count_below_poverty = below_povertyE) + +head(county_poverty) +``` + +Note the column naming: `crosswalk_data()` detects what to do with each column +from its prefix. + +| Prefix | Treatment | +|---|---| +| `count_` | Multiplied by the allocation factor, then summed (counts: population, households, loans, ...) | +| `mean_`, `median_`, `percent_`, `ratio_` | Weighted mean, using the allocation factor as the weight (rates, averages) | + +If you'd rather not rename your columns, pass them explicitly via +`count_columns` and `non_count_columns`. The column holding your GEOIDs is +named by `geoid_column` (default `"source_geoid"`). + +```{r apply, eval = has_census_key, message = TRUE} +puma_poverty <- crosswalk_data( + data = county_poverty, + crosswalk = county_to_puma) + +head(puma_poverty) +``` + +The result has one row per *target* geography: the target identifier is named +`geoid` (plus `geography_name` when the crosswalk provides it). + +## Join quality: did everything match? + +Data rows whose GEOIDs don't appear in the crosswalk can't be allocated +anywhere, so they're dropped -- silently losing data is the main hazard of +crosswalking, which is why `crosswalk_data()` reports on it. The join-quality +messages print as it runs (suppress them with `show_join_quality = FALSE`), +and the same statistics are always attached to the result as the +`join_quality` attribute: + +```{r join-quality, eval = has_census_key} +join_quality <- attr(puma_poverty, "join_quality") + +## what share of data GEOIDs failed to match the crosswalk? +join_quality$pct_data_unmatched + +## which GEOIDs were they? (none, in this case) +head(join_quality$data_geoids_unmatched) +``` + +See `?crosswalk_data` for the attribute's full structure, and +`vignette("how-interpolation-works")` for how to diagnose imperfect joins. + +## Multi-step crosswalks: changing geography *and* year + +No single source provides, say, 2014 counties to 2022 PUMAs. When both the +geography and the year change, `get_crosswalk()` plans a chain: first change +the year (holding geography constant), then change the geography at the target +year. + +```{r multi-step} +county14_to_puma22 <- get_crosswalk( + source_geography = "county", + target_geography = "puma", + source_year = 2014, + target_year = 2022, + weight = "population", + cache = cache_dir) + +county14_to_puma22$message +``` + +You don't need to do anything special to apply a chain -- +`crosswalk_data()` applies each step in sequence: + +```{r multi-step-apply, eval = has_census_key} +county_poverty_2014 <- tidycensus::get_acs( + year = 2014, + geography = "county", + output = "wide", + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> + select( + source_geoid = GEOID, + count_below_poverty = below_povertyE) + +puma22_poverty <- crosswalk_data( + data = county_poverty_2014, + crosswalk = county14_to_puma22) + +head(puma22_poverty) +``` + +The attributes on the returned tibble describe the final step; pass +`return_intermediate = TRUE` to get each step's result (with its own +attributes) alongside the final one. + +### Cross-decade crosswalks (NHGIS) + +Crosswalks that span a decennial census -- 2010 tracts to 2020 tracts, for +example -- come from IPUMS NHGIS and require a free API key. Request one at + and store it in your `.Renviron` as +`IPUMS_API_KEY` (e.g., via `usethis::edit_r_environ()`). Then: + +```{r nhgis, eval = FALSE} +tract_2010_to_zcta_2020 <- get_crosswalk( + source_geography = "tract", + target_geography = "zcta", + source_year = 2010, + target_year = 2020, + weight = "population") + +# Step 1: 2010 tracts -> 2020 tracts (NHGIS) +# Step 2: 2020 tracts -> 2020 ZCTAs (GeoCorr) +``` + +For a complete worked example -- standardizing six years of mortgage data to +2020 tract definitions -- see `vignette("standardizing-longitudinal-data")`. + +## County crosswalks between any pair of years + +Counties are occasionally redefined *outside* decennial censuses: renames +(Shannon County, SD becoming Oglala Lakota County in 2015), merges (Bedford, +VA in 2013), splits (Valdez-Cordova, AK in the 2020 vintage), and Connecticut's +2022 switch to planning regions. The package curates these events from Census +Bureau documentation, so county -> county crosswalks work between **any** pair +of years from 2000 onward (forward in time only), with no API key and no +network access: + +```{r county-events} +county_changes <- get_crosswalk( + source_geography = "county", + target_geography = "county", + source_year = 2014, + target_year = 2023, + silent = TRUE) + +## most counties are unchanged (identity rows); here are some that changed +county_changes$crosswalks$step_1 |> + filter(source_geoid != target_geoid) |> + head() +``` + +See `vignette("county-crosswalks")` for what's covered and how these +crosswalks are constructed. + +## Nested geographies: when you don't need a crosswalk + +If your source geography nests exactly within your target (tracts within +counties, block groups within tracts), there's nothing to crosswalk -- every +source unit belongs to exactly one target. `get_crosswalk()` warns and returns +an empty crosswalk; just aggregate your data directly: + +```{r nested, warning = TRUE} +nested <- get_crosswalk( + source_geography = "tract", + target_geography = "county") +``` + +## What combinations are supported? + +`get_available_crosswalks()` lists every supported combination and which +source serves it: + +```{r available} +available <- get_available_crosswalks() + +available |> + count(crosswalk_source) + +## e.g., everything that translates data out of 2010 tracts +available |> + filter(source_geography == "tract", source_year == 2010) |> + head() +``` + +One nuance: GeoCorr combinations are listed under their reference years (2022 +and 2018), but requests resolve by *decade* -- asking for 2021 tracts -> 2021 +ZCTAs is served by the tract -> zcta 2022 row. See +`?get_available_crosswalks`. + +## Caching + +We passed `cache = cache_dir` to each `get_crosswalk()` call above, so each +component crosswalk was saved as a CSV the first time it was fetched: + +```{r caching} +list.files(cache_dir) + +## a repeat call with the same parameters reads from disk instead of +## re-downloading +refetched <- get_crosswalk( + source_geography = "county", + target_geography = "puma", + weight = "population", + cache = cache_dir, + silent = TRUE) + +attr(refetched$crosswalks$step_1, "crosswalk_metadata")$read_from_cache +``` + +In a real project, use a persistent directory (e.g., +`here::here("crosswalks-cache")`) rather than `tempdir()`. Cached files never +expire; delete a file to force a re-download. + +## Quieting the messages + +Every function accepts `silent = TRUE`, and +`options(crosswalk.silent = TRUE)` silences the whole package. Silencing +suppresses messages only -- the `crosswalk_metadata` and `join_quality` +attributes are always attached. + +## Learn more + +- `vignette("how-interpolation-works")` -- how allocation factors are applied, + choosing a `weight`, the approximation involved in non-count variables, and + diagnosing join quality +- `vignette("county-crosswalks")` -- county redefinitions and arbitrary-year + county crosswalks +- `vignette("standardizing-longitudinal-data")` -- a worked example building a + six-year tract panel from mixed-vintage data diff --git a/vignettes/how-interpolation-works.Rmd b/vignettes/how-interpolation-works.Rmd new file mode 100644 index 0000000..74029c0 --- /dev/null +++ b/vignettes/how-interpolation-works.Rmd @@ -0,0 +1,293 @@ +--- +title: "How Interpolation Works" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{How Interpolation Works} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +## the accuracy-assessment section pulls data from the Census API +has_census_key <- Sys.getenv("CENSUS_API_KEY") != "" + +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + message = FALSE, + warning = FALSE, + fig.width = 7, + fig.height = 4.5) +``` + +Crosswalking produces *estimates*: when a source geography is split across +several target geographies, its values must be apportioned, and every +apportionment rests on assumptions. This vignette explains exactly what +`crosswalk_data()` computes, how to choose the `weight` that drives it, and +how to assess the quality of the results. + +```{r setup} +library(crosswalk) +library(dplyr) +library(tibble) +``` + +## Allocation factors + +Every crosswalk relates source geographies to target geographies with an +*allocation factor*: the share of the source unit's weighting attribute +(population, housing units, or land area) that falls inside each target unit. +Consider a miniature crosswalk in which source unit "A" lies entirely inside +target "X", while source unit "B" is split between "X" (a quarter of B's +population) and "Y" (three quarters): + +```{r toy-crosswalk} +toy_crosswalk <- tribble( + ~ source_geoid, ~ target_geoid, ~ allocation_factor_source_to_target, + "A", "X", 1, + "B", "X", 0.25, + "B", "Y", 0.75) +``` + +Allocation factors for each source unit sum to 1 (all of a source unit's +weight lands somewhere), and the crosswalk has exactly one row per +source-target pair. + +## Interpolating counts + +For count variables (population, households, loan originations, ...), each +source value is multiplied by the allocation factor and the products are +summed within each target geography: + +$$\text{count}_{\text{target}} = \sum_{\text{sources}} \text{count}_{\text{source}} \times \text{allocation factor}_{\text{source} \to \text{target}}$$ + +```{r toy-counts} +toy_data <- tribble( + ~ source_geoid, ~ count_people, + "A", 100, + "B", 400) + +crosswalk_data( + data = toy_data, + crosswalk = toy_crosswalk, + silent = TRUE) +``` + +Target X receives all of A plus a quarter of B (100 + 0.25 x 400 = 200), and +target Y receives the remaining three quarters of B (300). The total (500) is +preserved: counts are neither created nor destroyed, only reapportioned. + +The embedded assumption: the variable being crosswalked is distributed across +the source unit *proportionally to the weighting attribute*. Allocating a +quarter of B's loan originations to X is exactly right only if B's loans are +distributed like B's population. This is why matching the `weight` to your +variable matters (see below) -- and why results for small areas should be +treated as estimates. + +## Interpolating non-count variables + +Means, medians, percentages, and rates can't be summed. For these, +`crosswalk_data()` computes a *weighted mean* of the source values within each +target geography, using the allocation factors as weights: + +```{r toy-noncounts} +toy_medians <- tribble( + ~ source_geoid, ~ median_income, + "A", 50000, + "B", 100000) + +crosswalk_data( + data = toy_medians, + crosswalk = toy_crosswalk, + silent = TRUE) +``` + +Target X's value is (1 x 50000 + 0.25 x 100000) / 1.25 = 60000. + +Two approximations are involved: + +1. **A weighted mean of medians is not a median.** The true median income of + X's residents depends on the full income distributions of A and B, which + the crosswalk does not carry. The same applies to percentages: the exact + percentage for X would weight each source's percentage by its + *denominator* population in X, which is only approximated here. +2. **Allocation factors are shares, not sizes.** The factor 0.25 says "a + quarter of B goes to X" -- it does not say how large B is relative to A. If + B has ten times A's population, B's quarter inside X contains far more + people than all of A, but the weighted mean gives A four times B's weight + (1 vs. 0.25). The approximation is most accurate when source units are of + broadly similar size, as census units within a geography type generally + are. + +When precision matters for a non-count variable, a more accurate route is to +crosswalk its *components* as counts (numerator and denominator separately) +and recompute the ratio at the target level. + +## Choosing a weight + +The `weight` argument controls which attribute the allocation factors are +built from: + +| `weight` | Allocates by | Use for | +|---|---|---| +| `"population"` (default) | Resident population in the overlap | People-denominated variables: population counts, poverty, demographic shares | +| `"housing"` | Housing units in the overlap | Housing-stock variables: units, tenure, vacancy, home loans | +| `"land"` | Land area of the overlap | Land-based variables, or when settlement patterns are irrelevant | + +Guidance: + +- Match the weight to the variable's denominator. Housing-unit weights + allocate a tract's housing stock more faithfully than population weights do + (group quarters hold people but no housing units, for example). +- Population and housing weights are usually similar; land-area weights can + differ dramatically in sparsely settled areas (a target unit covering 90% of + a source's land might hold 5% of its people). +- **NHGIS caveat**: block-based NHGIS crosswalks publish a single combined + interpolation weight, so `weight` does not vary the allocation factors for + those crosswalks. Tract- and block-group-level NHGIS crosswalks do provide + distinct population and housing weights. The `weighting_factor` column of + the returned crosswalk always records what was actually used. +- County-events crosswalks use `"identity"` weights for unchanged and renamed + counties (factor 1) and population-based factors for splits and part + transfers, regardless of `weight`. + +## Assessing accuracy against published values + +For geographies the Census Bureau publishes directly, we can check +crosswalked estimates against observed values. Here we crosswalk ZCTA-level +poverty counts to PUMAs, then compare against the ACS's own PUMA-level +figures. (This section queries the Census API, so it requires a +`CENSUS_API_KEY`.) + +```{r accuracy-setup, eval = has_census_key} +zcta_poverty <- tidycensus::get_acs( + year = 2023, + geography = "zcta", + output = "wide", + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> + select( + source_geoid = GEOID, + count_below_poverty = below_povertyE) + +puma_poverty_crosswalked <- crosswalk_data( + data = zcta_poverty, + source_geography = "zcta", + target_geography = "puma", + weight = "population", + silent = TRUE) +``` + +```{r accuracy-compare, eval = has_census_key} +library(ggplot2) + +puma_poverty_published <- tidycensus::get_acs( + year = 2023, + geography = "puma", + output = "wide", + variables = c(below_poverty = "B17001_002"), + progress_bar = FALSE) |> + select( + puma_geoid = GEOID, + count_below_poverty_published = below_povertyE) + +comparison <- puma_poverty_published |> + left_join(puma_poverty_crosswalked, by = c("puma_geoid" = "geoid")) |> + mutate( + difference_percent = + (count_below_poverty_published - count_below_poverty) / + count_below_poverty_published) + +comparison |> + ggplot() + + geom_histogram(aes(x = difference_percent), bins = 60) + + theme_minimal() + + theme(panel.grid = element_blank()) + + scale_x_continuous(labels = scales::percent) + + labs( + title = "Crosswalked data approximates observed values", + subtitle = "ZCTA-level poverty counts crosswalked to PUMAs vs. published PUMA values", + y = "Number of PUMAs", + x = "Percent difference between published and crosswalked values") +``` + +Most PUMAs land within a few percent of their published values. The residual +error comes from the proportional-distribution assumption above -- and it +shrinks as source units get smaller relative to target units. Crosswalking +from block groups instead of ZCTAs would track published values more closely; +crosswalking from large units to small ones (counties to tracts, say) deserves +much more skepticism than the reverse. + +Also note both sides of this comparison are survey estimates with their own +margins of error. The package does not propagate ACS margins of error through +interpolation. + +## Diagnosing join quality + +Interpolation can only be as good as the join between your data and the +crosswalk. Data GEOIDs missing from the crosswalk are *dropped* (their values +allocate nowhere), so `crosswalk_data()` reports on the join as it runs and +attaches the full statistics to its result as the `join_quality` attribute -- +computed even when messages are silenced: + +```{r join-quality, eval = has_census_key} +join_quality <- attr(puma_poverty_crosswalked, "join_quality") + +## share of data GEOIDs that failed to match the crosswalk (dropped) +join_quality$pct_data_unmatched + +## the unmatched GEOIDs themselves +head(join_quality$data_geoids_unmatched) + +## share of crosswalk GEOIDs with no data (usually benign: geographies +## with no observations in your data) +join_quality$pct_crosswalk_unmatched +``` + +A small unmatched share is common -- GEOID vintages differ subtly across +products, and some datasets use placeholder codes. A *large* share usually +means a vintage mismatch: e.g., 2023-vintage tract data joined to a crosswalk +built for 2010s-era tracts. Checking a few unmatched GEOIDs against the +[Census geocoder](https://geocoding.geo.census.gov/) or the source data's +documentation usually identifies the problem quickly. + +For state-nested geographies (counties, tracts, block groups, blocks), the +attribute also reports whether unmatched GEOIDs concentrate in particular +states (`state_analysis_data`; see `?crosswalk_data`). Concentration is a +diagnostic clue: unmatched GEOIDs spread thinly across many states suggest +scattered data quirks, while a pile-up in one state suggests a systematic +boundary or vintage issue (Connecticut's 2022 county change is a frequent +culprit). + +ZCTAs cross state lines, so state analysis doesn't apply to the example +above. For non-nested geographies, joining unmatched GEOIDs to spatial data is +an effective way to look for patterns: + +```{r spatial-diagnosis, eval = FALSE, purl = FALSE} +## e.g., where are unmatched ZCTAs concentrated? (requires sf + tigris) +zctas_sf <- tigris::zctas(year = 2023, progress_bar = FALSE) +states_sf <- tigris::states(year = 2023, cb = TRUE, progress_bar = FALSE) + +zctas_sf |> + dplyr::filter(GEOID20 %in% join_quality$data_geoids_unmatched) |> + sf::st_intersection(states_sf |> dplyr::select(NAME)) |> + sf::st_drop_geometry() |> + dplyr::count(NAME, sort = TRUE) +``` + +When we ran this diagnosis on the ZCTA -> PUMA example, unmatched ZCTAs were +roughly proportional to state populations -- except for a disproportionate +cluster in Washington, DC corresponding to federal areas and buildings: real +ZCTAs, but ones with no place in a population-weighted crosswalk. + +## Summing up + +- Counts are apportioned by allocation factor and summed; totals are + preserved. +- Non-counts are approximated by allocation-factor-weighted means; when + precision matters, crosswalk numerators and denominators as counts instead. +- Match `weight` to your variable's denominator. +- Check the `join_quality` attribute -- unmatched data rows are dropped, and a + concentrated pattern of unmatched GEOIDs usually signals a vintage mismatch. +- Prefer small source units relative to target units; treat the reverse with + skepticism. diff --git a/vignettes/precompile.R b/vignettes/precompile.R new file mode 100644 index 0000000..2f1f952 --- /dev/null +++ b/vignettes/precompile.R @@ -0,0 +1,15 @@ +# Precompile computationally expensive vignettes +# +# standardizing-longitudinal-data.Rmd is generated from +# standardizing-longitudinal-data.Rmd.orig: knitting the .orig file downloads +# six years of HMDA data and an NHGIS crosswalk (which requires an +# IPUMS_API_KEY), and the resulting .Rmd -- with outputs baked in -- is what +# R CMD build and pkgdown render, so package builds need neither network +# access nor API keys for this vignette. Re-run this script from the package +# root whenever the .orig file changes or its outputs should be refreshed: +# +# source("vignettes/precompile.R") + +knitr::knit( + input = "vignettes/standardizing-longitudinal-data.Rmd.orig", + output = "vignettes/standardizing-longitudinal-data.Rmd") diff --git a/vignettes/standardizing-longitudinal-data.Rmd b/vignettes/standardizing-longitudinal-data.Rmd index ad7a601..86970f0 100644 --- a/vignettes/standardizing-longitudinal-data.Rmd +++ b/vignettes/standardizing-longitudinal-data.Rmd @@ -1,196 +1,332 @@ ---- -title: "Standardizing Longitudinal Data" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{Standardizing Longitudinal Data} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -```{r, include = FALSE} -# Only evaluate chunks if IPUMS API key is available -has_api_key <- nchar(Sys.getenv("IPUMS_API_KEY")) > 10 - -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>", - message = FALSE, - warning = FALSE, - echo = TRUE, - eval = has_api_key) -``` - -## Overview - -A common challenge with longitudinal tract data is that census tract boundaries -change between decennial censuses. Data from before 2020 typically uses 2010 tract -definitions, while more recent data uses 2020 tract definitions. To analyze trends -over time, you need to standardize all years to a consistent tract vintage. - -This vignette demonstrates how the `crosswalk` package efficiently handles this task -using the Urban Institute's [HMDA Neighborhood Summary Files](https://datacatalog.urban.org/dataset/home-mortgage-disclosure-act-neighborhood-summary-files-census-tract-level), -which provide tract-level mortgage lending data from 2018-2023. The 2018-2021 files -use 2010 tract definitions, while 2022-2023 use 2020 tract definitions. - -## Setup - -```{r setup, message=FALSE} -library(crosswalk) -library(dplyr) -library(purrr) -library(readr) -library(tibble) -library(stringr) -library(tidycensus) -library(ggplot2) -``` - -## Step 1: Download the Data - -The Urban Institute publishes annual HMDA tract-level summary files. Let's download -all six years (2018-2023): - -```{r download-data} -## metadata object describing data year/vintage/url -metadata = tribble( - ~ year, ~ vintage, ~ url, - 2018, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2018.csv", - 2019, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2019.csv", - 2020, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2020.csv", - 2021, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2021.csv", - 2022, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2022.csv", - 2023, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2024/12/17/hmda_tract_2023.csv") - -## iterate over the metadata object and read in data for each year -hmda_data <- pmap(metadata, function(url, year, vintage) { - read_csv(url, show_col_types = FALSE) |> - mutate( - vintage = vintage, - data_year = as.integer(year)) }) - -names(hmda_data) = metadata$year %>% as.character() -``` - -Let's inspect the structure of the data: - -```{r inspect-data} -## just view the first ten columns -glimpse(hmda_data[["2018"]] %>% select(1:10)) -``` - -## Step 2: Prepare Data for Crosswalking - -We'll focus on a subset of variables for crosswalking -(total applications by race/ethnicity and median loan amounts). We could explicitly pass the -variables we want to crosswalk to the appropriate parameter (`count_columns` or `non_count_columns`), -but it's easy (and nice practice) to prefix these variables with their unit types ("count" and "median", -respectively), and `crosswalk_data()` will crosswalk each appropriately by default. - -```{r prepare-data} -prepare_hmda <- function(data) { - data |> - rename_with(.cols = matches("^geo20"), .fn = ~ "source_geoid") |> - select( - source_geoid, - vintage, - data_year, - # Count variables: rename with count_ prefix for automatic detection - count_race_white_purchase = race_white_purchase, - count_owner_purchase_originations = owner_purchase_originations, - median_owner_loan_amount = owner_loan_amount_median) |> - mutate(source_geoid = as.character(source_geoid)) } - -hmda_prepared <- map(hmda_data, prepare_hmda) -``` - -## Step 3: Obtain the 2010→2020 Tract Crosswalk - -Next we get our crosswalk (the same for each of our 2010-vintage years of HMDA data), -which contains allocation factors that specify how to distribute values from 2010 tracts -definitions to those for 2020 tracts. - -```{r get-crosswalk} -tract_crosswalk <- get_crosswalk( - source_geography = "tract", - target_geography = "tract", - source_year = 2010, - target_year = 2020, - weight = "population") - -# View the crosswalk plan -tract_crosswalk$message -``` - -## Step 4: Apply the Crosswalk to 2018-2021 Data - -Now we apply the crosswalk to the four years of data that use 2010 tract definitions. -We can see that relatively small, though not insignificant, -fractions of records in our source data do not join to our crosswalk. When this occurs, source -data is effectively lost because it has no associated target geography nor allocation factor -assigned to it. - -```{r apply-crosswalk} -# Years that need crosswalking (2010 vintage) -years_to_crosswalk <- c("2018", "2019", "2020", "2021") - -# Apply crosswalk to each year -hmda_crosswalked <- map_if( - .x = hmda_prepared, - .p = names(hmda_prepared) %in% years_to_crosswalk, - .f = ~ crosswalk_data( - data = .x, - crosswalk = tract_crosswalk, - geoid_column = "source_geoid", - show_join_quality = TRUE)) -``` - -## Step 5: Assess Crosswalking Quality - -Is there anything we can learn about our source data that doesn't join to our crosswalk? -Ideally, every record in our source data maps to a record in our crosswalk. - -```{r, diagnose-crosswalking} -## we see that some observations that don't match have "XXXXXX" in lieu of -## a real tract code--which, from reading the data documentation, we know is -## done to to retain valid observations that, in the source data, do not have -## a valid tract identifier but do have valid county and/or state identifies -hmda_crosswalked |> - map(~ - .x |> - attr("join_quality") |> - pluck("data_geoids_unmatched") |> - head(5)) -``` - -```{r} -## how many source records are we unable to crosswalk each year, excluding -## those with "X" in their GEOIDs? under 30 each year. -hmda_crosswalked |> - map(~ - .x |> - attr("join_quality") |> - pluck("data_geoids_unmatched") %>% - .[!str_detect(., "X")] |> - length()) -``` - -## Result: A Panel Dataset in 2020 Tract Definitions -We now have a single dataframe with all six years of HMDA data standardized to 2020 -tract definitions. Due to changes in tract geographies between decades, we were previously -unable to accurately compare neighborhood changes over time. - -Now, we have apples-to-apples measurements for tracts from 2018 through 2023. - -```{r final-summary} -## there's a little bit of variation year-to-year in terms of which tracts have -## reported HMDA data, but for the majority, we have observations in each of the -## six years: -hmda_combined <- bind_rows(hmda_crosswalked) |> - ## data for years that are crosswalked have slightly different/additional columns - mutate( - geoid = if_else(is.na(geoid), source_geoid, geoid)) |> - count(geoid) |> - count(n) - -head(hmda_combined) -``` +--- +title: "Standardizing Longitudinal Data" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Standardizing Longitudinal Data} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + + + +## Overview + +A common challenge with longitudinal tract data is that census tract boundaries +change between decennial censuses. Data from before 2020 typically uses 2010 tract +definitions, while more recent data uses 2020 tract definitions. To analyze trends +over time, you need to standardize all years to a consistent tract vintage. + +This vignette demonstrates how the `crosswalk` package efficiently handles this task +using the Urban Institute's [HMDA Neighborhood Summary Files](https://datacatalog.urban.org/dataset/home-mortgage-disclosure-act-neighborhood-summary-files-census-tract-level), +which provide tract-level mortgage lending data from 2018-2023. The 2018-2021 files +use 2010 tract definitions, while 2022-2023 use 2020 tract definitions. + +Note: this vignette is precomputed because it downloads six years of HMDA data +and an NHGIS crosswalk (which requires an `IPUMS_API_KEY`). The outputs shown +were generated when the vignette was last knit from its source +(`vignettes/standardizing-longitudinal-data.Rmd.orig`); see +`vignettes/precompile.R`. + +## Setup + + +``` r +library(crosswalk) +library(dplyr) +library(purrr) +library(readr) +library(stringr) +library(tibble) +library(tidyr) +``` + +## Step 1: Download the Data + +The Urban Institute publishes annual HMDA tract-level summary files. Let's download +all six years (2018-2023): + + +``` r +## metadata object describing data year/vintage/url +metadata <- tribble( + ~ year, ~ vintage, ~ url, + 2018, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2018.csv", + 2019, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2019.csv", + 2020, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2020.csv", + 2021, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2021.csv", + 2022, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2022.csv", + 2023, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2024/12/17/hmda_tract_2023.csv") + +## iterate over the metadata object and read in data for each year +hmda_data <- pmap(metadata, function(url, year, vintage) { + read_csv(url, show_col_types = FALSE) |> + mutate( + vintage = vintage, + data_year = as.integer(year)) }) + +names(hmda_data) <- as.character(metadata$year) +``` + +Let's inspect the structure of the data: + + +``` r +## just view the first ten columns +glimpse(hmda_data[["2018"]] |> select(1:10)) +#> Rows: 74,652 +#> Columns: 10 +#> $ geo2010 "00XXXXXXXXX", "01001020100", "0100… +#> $ owner_purchase_originations 2, 21, 11, 41, 64, 185, 44, 68, 41,… +#> $ occupancy_investment_origination 2, 4, 4, 3, 6, 19, 3, 5, 3, 5, 6, 0… +#> $ occupancy_investment_units_1_4 2, 4, 3, 3, 6, 17, 3, 5, 3, 5, 5, 0… +#> $ occupancy_investment_units_5ormore 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 1, 0,… +#> $ income_available 0, 21, 11, 41, 64, 184, 42, 68, 41,… +#> $ race_available 2, 21, 10, 39, 61, 171, 41, 66, 38,… +#> $ race_income_available 0, 21, 10, 39, 61, 170, 39, 66, 38,… +#> $ age_available 2, 21, 11, 41, 64, 185, 44, 68, 41,… +#> $ race_white_purchase 2, 19, 7, 33, 54, 128, 30, 47, 34, … +``` + +## Step 2: Prepare Data for Crosswalking + +We'll focus on a subset of variables for crosswalking +(total applications by race/ethnicity and median loan amounts). We could explicitly pass the +variables we want to crosswalk to the appropriate parameter (`count_columns` or `non_count_columns`), +but it's easy (and nice practice) to prefix these variables with their unit types ("count" and "median", +respectively), and `crosswalk_data()` will crosswalk each appropriately by default. + +Note that non-count variables like `median_owner_loan_amount` are interpolated +as weighted means, which is an approximation -- see +`vignette("how-interpolation-works")` for how to interpret crosswalked +non-count values. + + +``` r +prepare_hmda <- function(data) { + data |> + rename_with(.cols = matches("^geo20"), .fn = ~ "source_geoid") |> + select( + source_geoid, + vintage, + data_year, + # Count variables: rename with count_ prefix for automatic detection + count_race_white_purchase = race_white_purchase, + count_owner_purchase_originations = owner_purchase_originations, + median_owner_loan_amount = owner_loan_amount_median) |> + mutate(source_geoid = as.character(source_geoid)) } + +hmda_prepared <- map(hmda_data, prepare_hmda) +``` + +## Step 3: Obtain the 2010→2020 Tract Crosswalk + +Next we get our crosswalk (the same for each of our 2010-vintage years of HMDA data), +which contains allocation factors that specify how to distribute values from 2010 tracts +definitions to those for 2020 tracts. + + +``` r +tract_crosswalk <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2010, + target_year = 2020, + weight = "population") + +# View the crosswalk plan +tract_crosswalk$message +#> [1] "Single-step crosswalk:\n Step 1: 2010 tract -> 2020 tract (inter-temporal via nhgis)\n\nSingle crosswalk; use allocation_factor_source_to_target directly." +``` + +## Step 4: Apply the Crosswalk to 2018-2021 Data + +Now we apply the crosswalk to the four years of data that use 2010 tract definitions. +`crosswalk_data()` prints join-quality diagnostics as it works (shown below); +relatively small, though not insignificant, +fractions of records in our source data do not join to our crosswalk. When this occurs, source +data is effectively lost because it has no associated target geography nor allocation factor +assigned to it. + + +``` r +# Years that need crosswalking (2010 vintage) +years_to_crosswalk <- c("2018", "2019", "2020", "2021") + +# Apply crosswalk to each year +hmda_crosswalked <- map_if( + .x = hmda_prepared, + .p = names(hmda_prepared) %in% years_to_crosswalk, + .f = ~ crosswalk_data( + data = .x, + crosswalk = tract_crosswalk, + geoid_column = "source_geoid", + show_join_quality = TRUE)) +#> Applying crosswalk step 1 of 1... +#> Join quality: 2,643 of 74,652 unique data GEOIDs (3.5%) did not match the crosswalk. +#> Top states with unmatched data rows: TX (9%, 233 rows), GA (6%, 154 rows), KY (4%, 114 rows) +#> Join quality: 1,993 of 74,002 crosswalk source GEOIDs (2.7%) were not in input data. +#> Top states not in data: PR (47%, 943 rows), NY (7%, 147 rows), CA (5%, 93 rows) +#> (This is expected if your data covers a geographic subset.) +#> Applying crosswalk step 1 of 1... +#> Join quality: 2,447 of 74,485 unique data GEOIDs (3.3%) did not match the crosswalk. +#> Top states with unmatched data rows: TX (10%, 238 rows), GA (6%, 154 rows), KY (4%, 106 rows) +#> Join quality: 1,964 of 74,002 crosswalk source GEOIDs (2.7%) were not in input data. +#> Top states not in data: PR (48%, 942 rows), NY (7%, 143 rows), CA (5%, 90 rows) +#> (This is expected if your data covers a geographic subset.) +#> Applying crosswalk step 1 of 1... +#> Join quality: 2,622 of 74,689 unique data GEOIDs (3.5%) did not match the crosswalk. +#> Top states with unmatched data rows: TX (9%, 239 rows), GA (6%, 154 rows), KY (4%, 115 rows) +#> Join quality: 1,935 of 74,002 crosswalk source GEOIDs (2.6%) were not in input data. +#> Top states not in data: PR (49%, 944 rows), NY (7%, 135 rows), MI (5%, 92 rows) +#> (This is expected if your data covers a geographic subset.) +#> Applying crosswalk step 1 of 1... +#> Join quality: 2,700 of 74,811 unique data GEOIDs (3.6%) did not match the crosswalk. +#> Top states with unmatched data rows: TX (9%, 239 rows), GA (6%, 158 rows), KY (4%, 115 rows) +#> Join quality: 1,891 of 74,002 crosswalk source GEOIDs (2.6%) were not in input data. +#> Top states not in data: PR (50%, 943 rows), NY (7%, 136 rows), CA (4%, 85 rows) +#> (This is expected if your data covers a geographic subset.) +``` + +## Step 5: Assess Crosswalking Quality + +The same diagnostics printed above are attached to each result as the +`join_quality` attribute (see `?crosswalk_data` for its full structure), so we +can work with them programmatically. Is there anything we can learn about our +source data that doesn't join to our crosswalk? +Ideally, every record in our source data maps to a record in our crosswalk. + + +``` r +## we see that some observations that don't match have "XXXXXX" in lieu of +## a real tract code--which, from reading the data documentation, we know is +## done to to retain valid observations that, in the source data, do not have +## a valid tract identifier but do have valid county and/or state identifies +hmda_crosswalked[years_to_crosswalk] |> + map(~ + .x |> + attr("join_quality") |> + pluck("data_geoids_unmatched") |> + head(5)) +#> $`2018` +#> [1] "00XXXXXXXXX" "01001XXXXXX" "01003XXXXXX" "01005XXXXXX" "01007XXXXXX" +#> +#> $`2019` +#> [1] "01001XXXXXX" "01003XXXXXX" "01005XXXXXX" "01007XXXXXX" "01009XXXXXX" +#> +#> $`2020` +#> [1] "01001XXXXXX" "01003XXXXXX" "01005XXXXXX" "01007XXXXXX" "01009XXXXXX" +#> +#> $`2021` +#> [1] "01001XXXXXX" "01003XXXXXX" "01005XXXXXX" "01007XXXXXX" "01009XXXXXX" +``` + + +``` r +## how many source records are we unable to crosswalk each year, excluding +## those with "X" in their GEOIDs? under 30 each year. +hmda_crosswalked[years_to_crosswalk] |> + map(~ + .x |> + attr("join_quality") |> + pluck("data_geoids_unmatched") |> + discard(~ str_detect(.x, "X")) |> + length()) +#> $`2018` +#> [1] 25 +#> +#> $`2019` +#> [1] 27 +#> +#> $`2020` +#> [1] 29 +#> +#> $`2021` +#> [1] 24 +``` + +## Result: A Panel Dataset in 2020 Tract Definitions + +We now have all six years of HMDA data standardized to 2020 tract definitions. +Due to changes in tract geographies between decades, we were previously +unable to accurately compare neighborhood changes over time. + + +``` r +hmda_combined <- bind_rows(hmda_crosswalked) |> + ## crosswalked years name their tract identifier `geoid`; the 2022-2023 + ## years (already in 2020 definitions) still use `source_geoid` + mutate(geoid = if_else(is.na(geoid), source_geoid, geoid)) + +## there's a little bit of variation year-to-year in terms of which tracts have +## reported HMDA data, but the large majority of tracts have observations in +## each of the six years: +hmda_combined |> + count(geoid, name = "n_years_observed") |> + count(n_years_observed) +#> # A tibble: 6 × 2 +#> n_years_observed n +#> +#> 1 1 626 +#> 2 2 2457 +#> 3 3 107 +#> 4 4 559 +#> 5 5 414 +#> 6 6 82672 +``` + +To see the payoff, consider a 2010 tract that was split into multiple 2020 +tracts. Before crosswalking, its 2018-2021 lending activity was reported under +a tract identifier that no longer exists in the 2022-2023 data; after +crosswalking, each of its 2020 successor tracts has an apples-to-apples series +covering all six years: + + +``` r +## find the 2010 tract split across the most 2020 tracts +split_tract <- tract_crosswalk$crosswalks$step_1 |> + summarize(.by = source_geoid, n_targets = n_distinct(target_geoid)) |> + filter(n_targets > 1) |> + arrange(desc(n_targets), source_geoid) |> + slice(1) |> + pull(source_geoid) + +successor_tracts <- tract_crosswalk$crosswalks$step_1 |> + filter(source_geoid == split_tract) |> + pull(target_geoid) + +split_tract +#> [1] "51810990100" + +## owner-occupied purchase originations, by year, for the successor tracts +hmda_combined |> + filter(geoid %in% successor_tracts) |> + select(geoid, data_year, count_owner_purchase_originations) |> + arrange(geoid, data_year) |> + pivot_wider( + names_from = data_year, + values_from = count_owner_purchase_originations) +#> # A tibble: 17 × 7 +#> geoid `2018` `2019` `2020` `2021` `2022` `2023` +#> +#> 1 51810040000 0 0 0 2 NA NA +#> 2 51810041801 51 71 52 61 46 44 +#> 3 51810041803 61.8 63.3 69.0 64.2 47 37 +#> 4 51810041804 68.2 69.7 76.0 70.8 41 37 +#> 5 51810043004 57 51 56 70 49 43 +#> 6 51810043005 40.6 47.3 53.1 56.2 36 24 +#> 7 51810043006 56.5 65.8 74.0 78.1 63 50 +#> 8 51810043200 1.42 2.39 1.42 1.88 NA 1 +#> 9 51810043400 32.0 34.1 38.0 43.0 41 17 +#> 10 51810043600 25.0 35.0 33.0 38.0 33 15 +#> 11 51810043800 69.3 53.3 66.3 64.7 34 31 +#> 12 51810044004 41.1 48.1 51.1 40.2 15 13 +#> 13 51810044008 35.4 38.9 42.7 46.9 19 23 +#> 14 51810045200 10.1 18.2 18.2 14.2 12 11 +#> 15 51810045412 17 19 36 26 21 18 +#> 16 51810046400 65.0 57.0 86.0 75.0 65 40 +#> 17 51810990100 0 0 0 0 0 NA +``` + +Each row is a 2020 tract with a complete 2018-2023 series: the 2018-2021 +values are allocated shares of the original (since-split) 2010 tract, and the +2022-2023 values come directly from the source data. diff --git a/vignettes/standardizing-longitudinal-data.Rmd.orig b/vignettes/standardizing-longitudinal-data.Rmd.orig new file mode 100644 index 0000000..768026c --- /dev/null +++ b/vignettes/standardizing-longitudinal-data.Rmd.orig @@ -0,0 +1,244 @@ +--- +title: "Standardizing Longitudinal Data" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Standardizing Longitudinal Data} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + message = FALSE, + warning = FALSE, + echo = TRUE, + error = FALSE) + +## the HMDA files are large; R's default 60s download timeout is too short +options(timeout = 600) +``` + +## Overview + +A common challenge with longitudinal tract data is that census tract boundaries +change between decennial censuses. Data from before 2020 typically uses 2010 tract +definitions, while more recent data uses 2020 tract definitions. To analyze trends +over time, you need to standardize all years to a consistent tract vintage. + +This vignette demonstrates how the `crosswalk` package efficiently handles this task +using the Urban Institute's [HMDA Neighborhood Summary Files](https://datacatalog.urban.org/dataset/home-mortgage-disclosure-act-neighborhood-summary-files-census-tract-level), +which provide tract-level mortgage lending data from 2018-2023. The 2018-2021 files +use 2010 tract definitions, while 2022-2023 use 2020 tract definitions. + +Note: this vignette is precomputed because it downloads six years of HMDA data +and an NHGIS crosswalk (which requires an `IPUMS_API_KEY`). The outputs shown +were generated when the vignette was last knit from its source +(`vignettes/standardizing-longitudinal-data.Rmd.orig`); see +`vignettes/precompile.R`. + +## Setup + +```{r setup, message=FALSE} +library(crosswalk) +library(dplyr) +library(purrr) +library(readr) +library(stringr) +library(tibble) +library(tidyr) +``` + +## Step 1: Download the Data + +The Urban Institute publishes annual HMDA tract-level summary files. Let's download +all six years (2018-2023): + +```{r download-data} +## metadata object describing data year/vintage/url +metadata <- tribble( + ~ year, ~ vintage, ~ url, + 2018, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2018.csv", + 2019, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2019.csv", + 2020, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2020.csv", + 2021, 2010, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2021.csv", + 2022, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2023/12/20/hmda_tract_2022.csv", + 2023, 2020, "https://urban-data-catalog.s3.amazonaws.com/drupal-root-live/2024/12/17/hmda_tract_2023.csv") + +## iterate over the metadata object and read in data for each year +hmda_data <- pmap(metadata, function(url, year, vintage) { + read_csv(url, show_col_types = FALSE) |> + mutate( + vintage = vintage, + data_year = as.integer(year)) }) + +names(hmda_data) <- as.character(metadata$year) +``` + +Let's inspect the structure of the data: + +```{r inspect-data} +## just view the first ten columns +glimpse(hmda_data[["2018"]] |> select(1:10)) +``` + +## Step 2: Prepare Data for Crosswalking + +We'll focus on a subset of variables for crosswalking +(total applications by race/ethnicity and median loan amounts). We could explicitly pass the +variables we want to crosswalk to the appropriate parameter (`count_columns` or `non_count_columns`), +but it's easy (and nice practice) to prefix these variables with their unit types ("count" and "median", +respectively), and `crosswalk_data()` will crosswalk each appropriately by default. + +Note that non-count variables like `median_owner_loan_amount` are interpolated +as weighted means, which is an approximation -- see +`vignette("how-interpolation-works")` for how to interpret crosswalked +non-count values. + +```{r prepare-data} +prepare_hmda <- function(data) { + data |> + rename_with(.cols = matches("^geo20"), .fn = ~ "source_geoid") |> + select( + source_geoid, + vintage, + data_year, + # Count variables: rename with count_ prefix for automatic detection + count_race_white_purchase = race_white_purchase, + count_owner_purchase_originations = owner_purchase_originations, + median_owner_loan_amount = owner_loan_amount_median) |> + mutate(source_geoid = as.character(source_geoid)) } + +hmda_prepared <- map(hmda_data, prepare_hmda) +``` + +## Step 3: Obtain the 2010→2020 Tract Crosswalk + +Next we get our crosswalk (the same for each of our 2010-vintage years of HMDA data), +which contains allocation factors that specify how to distribute values from 2010 tracts +definitions to those for 2020 tracts. + +```{r get-crosswalk} +tract_crosswalk <- get_crosswalk( + source_geography = "tract", + target_geography = "tract", + source_year = 2010, + target_year = 2020, + weight = "population") + +# View the crosswalk plan +tract_crosswalk$message +``` + +## Step 4: Apply the Crosswalk to 2018-2021 Data + +Now we apply the crosswalk to the four years of data that use 2010 tract definitions. +`crosswalk_data()` prints join-quality diagnostics as it works (shown below); +relatively small, though not insignificant, +fractions of records in our source data do not join to our crosswalk. When this occurs, source +data is effectively lost because it has no associated target geography nor allocation factor +assigned to it. + +```{r apply-crosswalk, message=TRUE} +# Years that need crosswalking (2010 vintage) +years_to_crosswalk <- c("2018", "2019", "2020", "2021") + +# Apply crosswalk to each year +hmda_crosswalked <- map_if( + .x = hmda_prepared, + .p = names(hmda_prepared) %in% years_to_crosswalk, + .f = ~ crosswalk_data( + data = .x, + crosswalk = tract_crosswalk, + geoid_column = "source_geoid", + show_join_quality = TRUE)) +``` + +## Step 5: Assess Crosswalking Quality + +The same diagnostics printed above are attached to each result as the +`join_quality` attribute (see `?crosswalk_data` for its full structure), so we +can work with them programmatically. Is there anything we can learn about our +source data that doesn't join to our crosswalk? +Ideally, every record in our source data maps to a record in our crosswalk. + +```{r, diagnose-crosswalking} +## we see that some observations that don't match have "XXXXXX" in lieu of +## a real tract code--which, from reading the data documentation, we know is +## done to to retain valid observations that, in the source data, do not have +## a valid tract identifier but do have valid county and/or state identifies +hmda_crosswalked[years_to_crosswalk] |> + map(~ + .x |> + attr("join_quality") |> + pluck("data_geoids_unmatched") |> + head(5)) +``` + +```{r} +## how many source records are we unable to crosswalk each year, excluding +## those with "X" in their GEOIDs? under 30 each year. +hmda_crosswalked[years_to_crosswalk] |> + map(~ + .x |> + attr("join_quality") |> + pluck("data_geoids_unmatched") |> + discard(~ str_detect(.x, "X")) |> + length()) +``` + +## Result: A Panel Dataset in 2020 Tract Definitions + +We now have all six years of HMDA data standardized to 2020 tract definitions. +Due to changes in tract geographies between decades, we were previously +unable to accurately compare neighborhood changes over time. + +```{r combine} +hmda_combined <- bind_rows(hmda_crosswalked) |> + ## crosswalked years name their tract identifier `geoid`; the 2022-2023 + ## years (already in 2020 definitions) still use `source_geoid` + mutate(geoid = if_else(is.na(geoid), source_geoid, geoid)) + +## there's a little bit of variation year-to-year in terms of which tracts have +## reported HMDA data, but the large majority of tracts have observations in +## each of the six years: +hmda_combined |> + count(geoid, name = "n_years_observed") |> + count(n_years_observed) +``` + +To see the payoff, consider a 2010 tract that was split into multiple 2020 +tracts. Before crosswalking, its 2018-2021 lending activity was reported under +a tract identifier that no longer exists in the 2022-2023 data; after +crosswalking, each of its 2020 successor tracts has an apples-to-apples series +covering all six years: + +```{r split-tract-example} +## find the 2010 tract split across the most 2020 tracts +split_tract <- tract_crosswalk$crosswalks$step_1 |> + summarize(.by = source_geoid, n_targets = n_distinct(target_geoid)) |> + filter(n_targets > 1) |> + arrange(desc(n_targets), source_geoid) |> + slice(1) |> + pull(source_geoid) + +successor_tracts <- tract_crosswalk$crosswalks$step_1 |> + filter(source_geoid == split_tract) |> + pull(target_geoid) + +split_tract + +## owner-occupied purchase originations, by year, for the successor tracts +hmda_combined |> + filter(geoid %in% successor_tracts) |> + select(geoid, data_year, count_owner_purchase_originations) |> + arrange(geoid, data_year) |> + pivot_wider( + names_from = data_year, + values_from = count_owner_purchase_originations) +``` + +Each row is a 2020 tract with a complete 2018-2023 series: the 2018-2021 +values are allocated shares of the original (since-split) 2010 tract, and the +2022-2023 values come directly from the source data.