diff --git a/NEWS.md b/NEWS.md index 408d8e3..ee92a90 100644 --- a/NEWS.md +++ b/NEWS.md @@ -71,6 +71,27 @@ ## Breaking changes +* **`matchData()`'s arguments are now `dat` and `source`**, replacing + `speciesDat` and `envDat`. + + The function was never specific to species observations or environmental data. + It is a spatiotemporal nearest-feature join between two `sf` point objects, and + works as well for tag positions against a model field, moorings against + satellite retrievals, or one gridded product against another. The old names + described one use of it as though it were the only one. + + **The old names still work**, with a warning, so existing scripts and + `taupatch` keep running. They will be removed in a later version. + + ```r + matchData(observations, env) # positional, unchanged + matchData(dat = observations, source = env) # new names + matchData(speciesDat = obs, envDat = env) # still works, warns + ``` + + One related change: a `source` column whose name collides with one already in + `dat` is now suffixed **`.matched`** rather than `.env`, for the same reason. + * **The `BigelowLab/copernicus` dependency is gone.** It was used in two places, both in `accessEnvDat()`, and both are now internal. This removes the `Remotes:` field and the hand-created `~/.copernicusdata` file that a new diff --git a/R/accessEnvDat.R b/R/accessEnvDat.R index 3eb5e46..8330aed 100644 --- a/R/accessEnvDat.R +++ b/R/accessEnvDat.R @@ -347,7 +347,7 @@ read_day <- function(item, vars) { #' @param n_workers how many days to download at once. See the #' Downloading in parallel section. Use `n_workers = 1` to download one day at #' a time. -#' @return envDat sf object containing requested environmental data from Copernicus Marine Service +#' @return sf object containing requested environmental data from Copernicus Marine Service #' @export accessEnvDat <- function(product_id = NULL, dataset_id = NULL, vars, years = NULL, months = NULL, diff --git a/R/matchData.R b/R/matchData.R index cc126f4..695c676 100644 --- a/R/matchData.R +++ b/R/matchData.R @@ -1,108 +1,158 @@ -#' Match environmental data to species occurrence data +#' Match one set of points to another in space and time #' -#' Joins each species observation to the nearest environmental grid point within -#' the same time period. The time period is the environmental data's own temporal -#' resolution: daily products match on year/month/day, monthly products (e.g. -#' Copernicus `...P1M-m` means) on year/month, and annual products on year alone. +#' Joins each row of `dat` to the nearest feature of `source` within the same +#' time period, and returns `dat` with `source`'s columns added. #' -#' Matching at the environmental data's native resolution matters because a -#' day-exact join against monthly data matches nothing - a monthly product carries -#' one time step per month, while observations fall on arbitrary days. +#' Neither side has to be species observations or environmental data. It is a +#' spatiotemporal nearest-feature join between two `sf` point objects that carry +#' `YEAR`/`MONTH`/`DAY` columns, so it works equally for stations against a +#' covariate grid, tag positions against a model field, moorings against +#' satellite retrievals, or one gridded product against another. #' -#' @param speciesDat species observation data (e.g., presence, density, count); -#' must have spatial and temporal components. Needs -#' year and month columns, plus a day column when -#' matching at daily resolution. -#' @param envDat environmental data accessed using the datamatch::accessEnvDat -#' function to be matched to the species observation data -#' @param temporal_resolution one of "auto" (default), "day", "month", or -#' "year". "auto" infers the resolution from -#' `envDat`'s own time steps. -#' @return `speciesDat` with the matched environmental variables joined on, -#' one row per input observation, plus LON/LAT coordinate columns. -#' Observations in a period with no environmental data get NA -#' for the environmental variables, and a warning is issued. -#' An environmental variable whose name collides with a column -#' already in `speciesDat` is suffixed `.env`. +#' @section Matching in time: +#' The time period is `source`'s own resolution: daily data matches on +#' year/month/day, monthly data (Copernicus `...P1M-m` means, say) on +#' year/month, and annual data on year alone. +#' +#' That matters because a day-exact join against monthly data matches nothing. A +#' monthly product carries one time step per month, while observations fall on +#' arbitrary days. `temporal_resolution` overrides the inference when the data +#' cannot speak for itself. +#' +#' @section What is preserved: +#' One row out per row of `dat`, in the same order, whatever happens. A period +#' `source` does not cover gives `NA` for its columns and a warning naming the +#' periods, rather than dropping those rows — a silent change in row count is a +#' worse outcome than a visible gap. +#' +#' `dat` keeps its own columns. One of `source`'s that collides with a name +#' already in `dat` is suffixed `.matched`, so nothing of `dat`'s is overwritten +#' or renamed. +#' +#' @param dat the points to add columns to: observations, stations, +#' tag positions, anything with coordinates and time. Needs year and month +#' columns, plus a day column when matching at daily resolution. Columns whose +#' names begin with those words are recognised, so `Year` or `obs_month` work. +#' @param source the points to take values from, typically a grid +#' from [accessEnvDat()]. Must carry `YEAR`/`MONTH`/`DAY`. +#' @param temporal_resolution one of `"auto"` (default), `"day"`, +#' `"month"`, or `"year"`. `"auto"` uses the step `accessEnvDat()` recorded on +#' `source`, or infers it from `source`'s time steps. +#' @param speciesDat,envDat deprecated names for `dat` and `source`. Still +#' accepted, with a warning. +#' @return `dat` with `source`'s columns joined on, one row per input +#' row, plus `LON`/`LAT` coordinate columns. +#' @examples +#' \dontrun{ +#' env <- accessEnvDat(vars = "SST", years = 2010, months = 1:12, bounding_box = bb) +#' +#' matched <- matchData(observations, env) +#' +#' # Chains, so several sources land on one table +#' matched <- matchData(matched, chlorophyll) +#' } +#' @seealso [accessEnvDat()] for the usual `source`, [attach_bathymetry()] and +#' [attach_climate_index()] for covariates that are not matched this way #' @export -matchData <- function(speciesDat, envDat, - temporal_resolution = c("auto", "day", "month", "year")) { +matchData <- function(dat, source, + temporal_resolution = c("auto", "day", "month", "year"), + speciesDat = NULL, envDat = NULL) { + + # The old names were specific to one use of a function that was never specific + # to it. Accepted for now because taupatch and any script written against the + # old signature call them by name, and breaking those silently would be worse + # than carrying two lines. + if (!is.null(speciesDat)) { + warning("`speciesDat` is now `dat`. The old name still works but will be ", + "removed.", call. = FALSE) + if (missing(dat)) dat <- speciesDat + } + if (!is.null(envDat)) { + warning("`envDat` is now `source`. The old name still works but will be ", + "removed.", call. = FALSE) + if (missing(source)) source <- envDat + } + if (missing(dat) || missing(source)) { + if (is.null(speciesDat) || is.null(envDat)) { + stop("Both `dat` and `source` are required.", call. = FALSE) + } + } temporal_resolution <- match.arg(temporal_resolution) if (temporal_resolution == "auto") { - temporal_resolution <- detect_temporal_resolution(envDat) + temporal_resolution <- detect_temporal_resolution(source) } match_keys <- switch(temporal_resolution, day = c("YEAR", "MONTH", "DAY"), month = c("YEAR", "MONTH"), year = "YEAR") - speciesDat <- standardize_time_columns(speciesDat, match_keys) + dat <- standardize_time_columns(dat, match_keys) - env_geom <- attr(envDat, "sf_column") - env_vars <- setdiff(names(envDat), c("YEAR", "MONTH", "DAY", env_geom)) + source_geom <- attr(source, "sf_column") + source_vars <- setdiff(names(source), c("YEAR", "MONTH", "DAY", source_geom)) # Both sides need a CRS before they can be reconciled. Without one, # st_transform() fails with "crs not found: is it missing?", which is true but # does not say which object or what to do. Silently assuming a CRS would be # worse: coordinates would be matched as though they were degrees, and every - # observation would join to whichever cell happened to be nearest in a - # meaningless space. - for (side in list(list(x = speciesDat, name = "speciesDat"), - list(x = envDat, name = "envDat"))) { + # row would join to whichever feature happened to be nearest in a meaningless + # space. + for (side in list(list(x = dat, name = "dat"), + list(x = source, name = "source"))) { if (is.na(sf::st_crs(side$x))) { - stop(side$name, " has no coordinate reference system, so it cannot be ", - "matched.\nSet one with sf::st_crs(", side$name, + stop("`", side$name, "` has no coordinate reference system, so it cannot ", + "be matched.\nSet one with sf::st_crs(", side$name, ") <- 4326 for longitude/latitude,\nor the EPSG code the ", "coordinates are actually in.", call. = FALSE) } } - envDat <- sf::st_transform(envDat, sf::st_crs(speciesDat)) + source <- sf::st_transform(source, sf::st_crs(dat)) - # Give environmental variables that share a name with a species column an - # explicit ".env" suffix. Otherwise st_join() disambiguates them as ".x"/".y", - # which both renames the species column and makes the result's column names + # Give a source column that shares a name with one in `dat` an explicit + # ".matched" suffix. Otherwise st_join() disambiguates them as ".x"/".y", + # which both renames the caller's column and makes the result's column names # depend on whether a given period actually matched anything. - collisions <- intersect(env_vars, names(speciesDat)) + collisions <- intersect(source_vars, names(dat)) if (length(collisions) > 0) { - renamed <- paste0(collisions, ".env") - names(envDat)[match(collisions, names(envDat))] <- renamed - env_vars[match(collisions, env_vars)] <- renamed + renamed <- paste0(collisions, ".matched") + names(source)[match(collisions, names(source))] <- renamed + source_vars[match(collisions, source_vars)] <- renamed } - periods <- unique(sf::st_drop_geometry(speciesDat)[match_keys]) + periods <- unique(sf::st_drop_geometry(dat)[match_keys]) matched <- vector("list", nrow(periods)) unmatched_periods <- character() for (i in seq_len(nrow(periods))) { - in_period <- rep(TRUE, nrow(speciesDat)) - env_in_period <- rep(TRUE, nrow(envDat)) + in_period <- rep(TRUE, nrow(dat)) + source_in_period <- rep(TRUE, nrow(source)) for (key in match_keys) { - in_period <- in_period & speciesDat[[key]] == periods[[key]][i] - env_in_period <- env_in_period & envDat[[key]] == periods[[key]][i] + in_period <- in_period & dat[[key]] == periods[[key]][i] + source_in_period <- source_in_period & source[[key]] == periods[[key]][i] } - obs <- speciesDat[in_period, ] - env_slice <- envDat[env_in_period, c(env_vars)] + rows <- dat[in_period, ] + source_slice <- source[source_in_period, c(source_vars)] - if (nrow(env_slice) == 0) { - # st_nearest_feature cannot join against an empty set, so fill the - # environmental columns with NA rather than dropping the observations. - # Dropping them would silently change the row count of the result. - for (v in env_vars) obs[[v]] <- NA + if (nrow(source_slice) == 0) { + # st_nearest_feature cannot join against an empty set, so fill the matched + # columns with NA rather than dropping the rows. Dropping them would + # silently change the row count of the result. + for (v in source_vars) rows[[v]] <- NA unmatched_periods <- c(unmatched_periods, paste(unlist(periods[i, , drop = TRUE]), collapse = "-")) - matched[[i]] <- obs + matched[[i]] <- rows } else { - matched[[i]] <- sf::st_join(obs, env_slice, join = sf::st_nearest_feature) + matched[[i]] <- sf::st_join(rows, source_slice, join = sf::st_nearest_feature) } } if (length(unmatched_periods) > 0) { - warning("No environmental data for ", length(unmatched_periods), - " period(s); environmental variables set to NA for: ", + warning("No data in `source` for ", length(unmatched_periods), + " period(s); matched columns set to NA for: ", paste(utils::head(unmatched_periods, 5), collapse = ", "), if (length(unmatched_periods) > 5) ", ..." else "", call. = FALSE) } @@ -120,7 +170,7 @@ matchData <- function(speciesDat, envDat, matched_data } -#' Infer the temporal resolution of environmental data +#' Infer the temporal resolution of a set of time steps #' #' Reads the resolution off the time steps actually present: more than one day #' within any month means daily data, and more than one month within any year @@ -135,21 +185,22 @@ matchData <- function(speciesDat, envDat, #' fine leaves them unmatched and warns. Pass `temporal_resolution` explicitly to #' override. #' -#' @param envDat environmental data with YEAR/MONTH/DAY columns +#' @param x an object with YEAR/MONTH/DAY columns, typically the +#' `source` side of a match #' @return one of "day", "month", or "year" #' @keywords internal -detect_temporal_resolution <- function(envDat) { +detect_temporal_resolution <- function(x) { # accessEnvDat() knows which dataset it fetched, so it records the step rather # than leaving it to be inferred. Worth trusting over the heuristics below: a # `dates` request of one date per month is genuinely indistinguishable from # monthly data by inspection, and guessing monthly would drop the day from the # match. - recorded <- attr(envDat, "datamatch_step") + recorded <- attr(x, "datamatch_step") if (!is.null(recorded) && recorded %in% c("day", "month", "year")) { return(recorded) } - times <- unique(sf::st_drop_geometry(envDat)[c("YEAR", "MONTH", "DAY")]) + times <- unique(sf::st_drop_geometry(x)[c("YEAR", "MONTH", "DAY")]) days_per_month <- tapply(times$DAY, paste(times$YEAR, times$MONTH), function(d) length(unique(d))) if (any(days_per_month > 1)) return("day") @@ -163,28 +214,28 @@ detect_temporal_resolution <- function(envDat) { "month" } -#' Rename a species dataset's time columns to YEAR/MONTH/DAY +#' Rename a table's time columns to YEAR/MONTH/DAY #' #' Only the columns needed for the requested match keys are required, so monthly #' matching works on data that has no day column at all. #' -#' @param speciesDat species observation data +#' @param dat the table being matched #' @param match_keys the standardized time columns needed, e.g. c("YEAR", "MONTH") -#' @return `speciesDat` with its time columns renamed to YEAR/MONTH/DAY +#' @return `dat` with its time columns renamed to YEAR/MONTH/DAY #' @keywords internal -standardize_time_columns <- function(speciesDat, match_keys) { +standardize_time_columns <- function(dat, match_keys) { # sf's select() method keeps the geometry column "sticky" regardless of the # select criteria, so it must be excluded here - otherwise the rename() below # would rename the geometry column itself and corrupt the sf object's tracked # geometry-column name. - geom_col <- attr(speciesDat, "sf_column") + geom_col <- attr(dat, "sf_column") for (key in match_keys) { - if (key %in% names(speciesDat)) next + if (key %in% names(dat)) next prefix <- tolower(key) candidates <- setdiff( - names(speciesDat |> dplyr::select(dplyr::starts_with(prefix, ignore.case = TRUE))), + names(dat |> dplyr::select(dplyr::starts_with(prefix, ignore.case = TRUE))), geom_col ) # An exact match wins over a mere prefix match, so a dataset carrying both @@ -195,17 +246,17 @@ standardize_time_columns <- function(speciesDat, match_keys) { } if (length(candidates) == 0) { - stop("speciesDat has no column for '", key, "' (looked for names starting with '", + stop("`dat` has no column for '", key, "' (looked for names starting with '", prefix, "'). It is required to match at this temporal resolution.", call. = FALSE) } if (length(candidates) > 1) { - stop("speciesDat has multiple candidate '", key, "' columns: ", + stop("`dat` has multiple candidate '", key, "' columns: ", paste(candidates, collapse = ", "), ". Rename the intended one to '", key, "'.", call. = FALSE) } - names(speciesDat)[names(speciesDat) == candidates] <- key + names(dat)[names(dat) == candidates] <- key } - speciesDat + dat } diff --git a/R/plot.R b/R/plot.R index 893726c..82c691d 100644 --- a/R/plot.R +++ b/R/plot.R @@ -223,7 +223,7 @@ plot_series <- function(env_dat, vars = NULL, fun = mean, spread = TRUE, ...) { #' @return the plotted values, invisibly #' @examples #' \dontrun{ -#' matched <- matchData(speciesDat = observations, envDat = env) +#' matched <- matchData(observations, env) #' #' plot_matched(matched, "SST") #' # Open circles are observations that matched nothing. diff --git a/README.Rmd b/README.Rmd index de68c24..b876af4 100644 --- a/README.Rmd +++ b/README.Rmd @@ -61,7 +61,7 @@ The goal of datamatch is to pull environmental data from Copernicus Marine Servi **Putting it to use** - [Putting it together](#putting-it-together) — a full worked example, four sources onto one table -- [Matching to observations](#matching-to-observations) +- [Matching](#matching) — a general spatiotemporal join, not just observations - [Troubleshooting](#troubleshooting) — what the error messages mean - [Related packages](#related-packages) @@ -121,7 +121,7 @@ env <- accessEnvDat( bounding_box = list(xmin = -76, xmax = -65, ymin = 35, ymax = 45) ) -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` `variable_dictionary()` lists what is available. @@ -149,8 +149,8 @@ phys <- accessEnvDat(vars = c("SST", "SSS", "MLD"), bio <- accessEnvDat(vars = c("CHL", "NO3"), years = 2003:2017, months = 1:12, bounding_box = bb) -matched <- matchData(speciesDat = observations, envDat = phys) -matched <- matchData(speciesDat = matched, envDat = bio) +matched <- matchData(observations, phys) +matched <- matchData(matched, bio) ``` `matchData()` chains. Each call adds that product's columns and leaves the row @@ -208,7 +208,7 @@ describe them. Take the dates from the observations themselves: env <- accessEnvDat(vars = "SST", dates = unique(observations$date), bounding_box = bb) -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` `YYYYMMDD` strings, `YYYY-MM-DD` strings, and `Date` objects are all accepted. @@ -631,7 +631,7 @@ hides the difference between a uniformly warm month and one that is warm inshore and cold offshore. ```{r plot-matched, eval = FALSE} -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) plot_matched(matched[matched$MONTH == 7, ], "SST") ``` @@ -873,8 +873,8 @@ bio <- accessEnvDat(vars = c("CHL", "NO3"), bathy <- fetch_bathymetry(bounding_box = bb) # 3. Join each source to the observations in turn. -matched <- matchData(speciesDat = observations, envDat = phys) -matched <- matchData(speciesDat = matched, envDat = bio) +matched <- matchData(observations, phys) +matched <- matchData(matched, bio) matched <- attach_bathymetry(matched, bathy, c("DEPTH", "SLOPE", "TPI")) matched <- attach_climate_index(matched, c("NAO", "LCR")) @@ -917,20 +917,35 @@ rather than the observations. Regrid first, then match: phys_coarse <- upscale_grid(phys, to = bio) # 0.083° onto the 0.25° BGC grid ``` -## Matching to observations +## Matching -`matchData()` joins environmental data to species observations at the -environmental data's own temporal resolution, inferred from its time steps. This -matters for monthly products: a monthly mean carries one time step per month -while observations fall on arbitrary days, so matching on exact dates would -match nothing. +`matchData(dat, source)` joins each row of `dat` to the nearest feature of +`source` within the same time period, and returns `dat` with `source`'s columns +added. ```{r match, eval = FALSE} -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` -Observations falling in a period with no environmental data are returned with -`NA` values and a warning naming the periods, rather than being dropped silently. +**Neither side has to be observations or environmental data.** It is a +spatiotemporal nearest-feature join between two `sf` point objects carrying +`YEAR`/`MONTH`/`DAY`, so it works equally for stations against a covariate grid, +tag positions against a model field, moorings against satellite retrievals, or +one gridded product against another. The arguments are named `dat` and `source` +for that reason. + +Matching happens at **`source`'s** temporal resolution, inferred from its time +steps. That matters for monthly products: a monthly mean carries one time step +per month while observations fall on arbitrary days, so matching on exact dates +would match nothing. Pass `temporal_resolution` to override. + +Rows falling in a period `source` does not cover are returned with `NA` and a +warning naming the periods, rather than being dropped silently. One row out per +row in, always. A `source` column whose name collides with one already in `dat` +is suffixed `.matched`, so nothing of yours is overwritten. + +> The arguments used to be `speciesDat` and `envDat`. Those still work and warn; +> they will be removed in a later version. ## Troubleshooting diff --git a/README.md b/README.md index d5f39f2..6a04a24 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The goal of datamatch is to pull environmental data from Copernicus Marine Servi **Putting it to use** - [Putting it together](#putting-it-together) — a full worked example, four sources onto one table -- [Matching to observations](#matching-to-observations) +- [Matching](#matching) — a general spatiotemporal join, not just observations - [Troubleshooting](#troubleshooting) — what the error messages mean - [Related packages](#related-packages) @@ -115,7 +115,7 @@ env <- accessEnvDat( bounding_box = list(xmin = -76, xmax = -65, ymin = 35, ymax = 45) ) -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` `variable_dictionary()` lists what is available. @@ -145,8 +145,8 @@ phys <- accessEnvDat(vars = c("SST", "SSS", "MLD"), bio <- accessEnvDat(vars = c("CHL", "NO3"), years = 2003:2017, months = 1:12, bounding_box = bb) -matched <- matchData(speciesDat = observations, envDat = phys) -matched <- matchData(speciesDat = matched, envDat = bio) +matched <- matchData(observations, phys) +matched <- matchData(matched, bio) ``` `matchData()` chains. Each call adds that product's columns and leaves the row @@ -207,7 +207,7 @@ describe them. Take the dates from the observations themselves: env <- accessEnvDat(vars = "SST", dates = unique(observations$date), bounding_box = bb) -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` `YYYYMMDD` strings, `YYYY-MM-DD` strings, and `Date` objects are all accepted. @@ -708,7 +708,7 @@ and cold offshore. ``` r -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) plot_matched(matched[matched$MONTH == 7, ], "SST") ``` @@ -956,8 +956,8 @@ bio <- accessEnvDat(vars = c("CHL", "NO3"), bathy <- fetch_bathymetry(bounding_box = bb) # 3. Join each source to the observations in turn. -matched <- matchData(speciesDat = observations, envDat = phys) -matched <- matchData(speciesDat = matched, envDat = bio) +matched <- matchData(observations, phys) +matched <- matchData(matched, bio) matched <- attach_bathymetry(matched, bathy, c("DEPTH", "SLOPE", "TPI")) matched <- attach_climate_index(matched, c("NAO", "LCR")) @@ -1002,21 +1002,36 @@ rather than the observations. Regrid first, then match: phys_coarse <- upscale_grid(phys, to = bio) # 0.083° onto the 0.25° BGC grid ``` -## Matching to observations +## Matching -`matchData()` joins environmental data to species observations at the -environmental data's own temporal resolution, inferred from its time steps. This -matters for monthly products: a monthly mean carries one time step per month -while observations fall on arbitrary days, so matching on exact dates would -match nothing. +`matchData(dat, source)` joins each row of `dat` to the nearest feature of +`source` within the same time period, and returns `dat` with `source`'s columns +added. ``` r -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) ``` -Observations falling in a period with no environmental data are returned with -`NA` values and a warning naming the periods, rather than being dropped silently. +**Neither side has to be observations or environmental data.** It is a +spatiotemporal nearest-feature join between two `sf` point objects carrying +`YEAR`/`MONTH`/`DAY`, so it works equally for stations against a covariate grid, +tag positions against a model field, moorings against satellite retrievals, or +one gridded product against another. The arguments are named `dat` and `source` +for that reason. + +Matching happens at **`source`'s** temporal resolution, inferred from its time +steps. That matters for monthly products: a monthly mean carries one time step +per month while observations fall on arbitrary days, so matching on exact dates +would match nothing. Pass `temporal_resolution` to override. + +Rows falling in a period `source` does not cover are returned with `NA` and a +warning naming the periods, rather than being dropped silently. One row out per +row in, always. A `source` column whose name collides with one already in `dat` +is suffixed `.matched`, so nothing of yours is overwritten. + +> The arguments used to be `speciesDat` and `envDat`. Those still work and warn; +> they will be removed in a later version. ## Troubleshooting diff --git a/man/accessEnvDat.Rd b/man/accessEnvDat.Rd index 2fb60a2..939e5ba 100644 --- a/man/accessEnvDat.Rd +++ b/man/accessEnvDat.Rd @@ -61,7 +61,7 @@ ten days ahead. See \code{\link[=forecast_variables]{forecast_variables()}} for forecast equivalent and how the identifiers differ.} } \value{ -envDat \if{html}{\out{}} sf object containing requested environmental data from Copernicus Marine Service +\if{html}{\out{}} sf object containing requested environmental data from Copernicus Marine Service } \description{ Downloads a Copernicus dataset over a bounding box and time range, and returns diff --git a/man/detect_temporal_resolution.Rd b/man/detect_temporal_resolution.Rd index 8d42de7..b0b06e7 100644 --- a/man/detect_temporal_resolution.Rd +++ b/man/detect_temporal_resolution.Rd @@ -2,12 +2,13 @@ % Please edit documentation in R/matchData.R \name{detect_temporal_resolution} \alias{detect_temporal_resolution} -\title{Infer the temporal resolution of environmental data} +\title{Infer the temporal resolution of a set of time steps} \usage{ -detect_temporal_resolution(envDat) +detect_temporal_resolution(x) } \arguments{ -\item{envDat}{\if{html}{\out{}} environmental data with YEAR/MONTH/DAY columns} +\item{x}{\if{html}{\out{}} an object with YEAR/MONTH/DAY columns, typically the +\code{source} side of a match} } \value{ one of "day", "month", or "year" diff --git a/man/matchData.Rd b/man/matchData.Rd index e5508a1..1880b10 100644 --- a/man/matchData.Rd +++ b/man/matchData.Rd @@ -2,43 +2,82 @@ % Please edit documentation in R/matchData.R \name{matchData} \alias{matchData} -\title{Match environmental data to species occurrence data} +\title{Match one set of points to another in space and time} \usage{ matchData( - speciesDat, - envDat, - temporal_resolution = c("auto", "day", "month", "year") + dat, + source, + temporal_resolution = c("auto", "day", "month", "year"), + speciesDat = NULL, + envDat = NULL ) } \arguments{ -\item{speciesDat}{\if{html}{\out{}} species observation data (e.g., presence, density, count); -must have spatial and temporal components. Needs -year and month columns, plus a day column when -matching at daily resolution.} +\item{dat}{\if{html}{\out{}} the points to add columns to: observations, stations, +tag positions, anything with coordinates and time. Needs year and month +columns, plus a day column when matching at daily resolution. Columns whose +names begin with those words are recognised, so \code{Year} or \code{obs_month} work.} -\item{envDat}{\if{html}{\out{}} environmental data accessed using the datamatch::accessEnvDat -function to be matched to the species observation data} +\item{source}{\if{html}{\out{}} the points to take values from, typically a grid +from \code{\link[=accessEnvDat]{accessEnvDat()}}. Must carry \code{YEAR}/\code{MONTH}/\code{DAY}.} -\item{temporal_resolution}{\if{html}{\out{}} one of "auto" (default), "day", "month", or -"year". "auto" infers the resolution from -\code{envDat}'s own time steps.} +\item{temporal_resolution}{\if{html}{\out{}} one of \code{"auto"} (default), \code{"day"}, +\code{"month"}, or \code{"year"}. \code{"auto"} uses the step \code{accessEnvDat()} recorded on +\code{source}, or infers it from \code{source}'s time steps.} + +\item{speciesDat, envDat}{deprecated names for \code{dat} and \code{source}. Still +accepted, with a warning.} } \value{ -\if{html}{\out{}} \code{speciesDat} with the matched environmental variables joined on, -one row per input observation, plus LON/LAT coordinate columns. -Observations in a period with no environmental data get NA -for the environmental variables, and a warning is issued. -An environmental variable whose name collides with a column -already in \code{speciesDat} is suffixed \code{.env}. +\if{html}{\out{}} \code{dat} with \code{source}'s columns joined on, one row per input +row, plus \code{LON}/\code{LAT} coordinate columns. } \description{ -Joins each species observation to the nearest environmental grid point within -the same time period. The time period is the environmental data's own temporal -resolution: daily products match on year/month/day, monthly products (e.g. -Copernicus \code{...P1M-m} means) on year/month, and annual products on year alone. +Joins each row of \code{dat} to the nearest feature of \code{source} within the same +time period, and returns \code{dat} with \code{source}'s columns added. } \details{ -Matching at the environmental data's native resolution matters because a -day-exact join against monthly data matches nothing - a monthly product carries -one time step per month, while observations fall on arbitrary days. +Neither side has to be species observations or environmental data. It is a +spatiotemporal nearest-feature join between two \code{sf} point objects that carry +\code{YEAR}/\code{MONTH}/\code{DAY} columns, so it works equally for stations against a +covariate grid, tag positions against a model field, moorings against +satellite retrievals, or one gridded product against another. +} +\section{Matching in time}{ + +The time period is \code{source}'s own resolution: daily data matches on +year/month/day, monthly data (Copernicus \code{...P1M-m} means, say) on +year/month, and annual data on year alone. + +That matters because a day-exact join against monthly data matches nothing. A +monthly product carries one time step per month, while observations fall on +arbitrary days. \code{temporal_resolution} overrides the inference when the data +cannot speak for itself. +} + +\section{What is preserved}{ + +One row out per row of \code{dat}, in the same order, whatever happens. A period +\code{source} does not cover gives \code{NA} for its columns and a warning naming the +periods, rather than dropping those rows — a silent change in row count is a +worse outcome than a visible gap. + +\code{dat} keeps its own columns. One of \code{source}'s that collides with a name +already in \code{dat} is suffixed \code{.matched}, so nothing of \code{dat}'s is overwritten +or renamed. +} + +\examples{ +\dontrun{ +env <- accessEnvDat(vars = "SST", years = 2010, months = 1:12, bounding_box = bb) + +matched <- matchData(observations, env) + +# Chains, so several sources land on one table +matched <- matchData(matched, chlorophyll) +} +} +\seealso{ +\code{\link[=accessEnvDat]{accessEnvDat()}} for the usual \code{source}, \code{\link[=attach_bathymetry]{attach_bathymetry()}} and +\code{\link[=attach_climate_index]{attach_climate_index()}} for covariates that are not matched this way } diff --git a/man/plot_matched.Rd b/man/plot_matched.Rd index a70147c..a52d202 100644 --- a/man/plot_matched.Rd +++ b/man/plot_matched.Rd @@ -40,7 +40,7 @@ reappears: \examples{ \dontrun{ -matched <- matchData(speciesDat = observations, envDat = env) +matched <- matchData(observations, env) plot_matched(matched, "SST") # Open circles are observations that matched nothing. diff --git a/man/standardize_time_columns.Rd b/man/standardize_time_columns.Rd index ad99378..1cadb13 100644 --- a/man/standardize_time_columns.Rd +++ b/man/standardize_time_columns.Rd @@ -2,17 +2,17 @@ % Please edit documentation in R/matchData.R \name{standardize_time_columns} \alias{standardize_time_columns} -\title{Rename a species dataset's time columns to YEAR/MONTH/DAY} +\title{Rename a table's time columns to YEAR/MONTH/DAY} \usage{ -standardize_time_columns(speciesDat, match_keys) +standardize_time_columns(dat, match_keys) } \arguments{ -\item{speciesDat}{\if{html}{\out{}} species observation data} +\item{dat}{\if{html}{\out{}} the table being matched} \item{match_keys}{\if{html}{\out{}} the standardized time columns needed, e.g. c("YEAR", "MONTH")} } \value{ -\code{speciesDat} with its time columns renamed to YEAR/MONTH/DAY +\code{dat} with its time columns renamed to YEAR/MONTH/DAY } \description{ Only the columns needed for the requested match keys are required, so monthly diff --git a/tests/testthat/test-matchData.R b/tests/testthat/test-matchData.R index a166e6c..c73e167 100644 --- a/tests/testthat/test-matchData.R +++ b/tests/testthat/test-matchData.R @@ -1,7 +1,7 @@ # Helpers ----------------------------------------------------------------- # Species observations: points with known lon/lat and YEAR/MONTH/DAY columns. -make_species_dat <- function(year_col = "YEAR", month_col = "MONTH", day_col = "DAY") { +make_observations <- function(year_col = "YEAR", month_col = "MONTH", day_col = "DAY") { df <- data.frame( id = 1:3, lon = c(-70.0, -69.5, -69.0), @@ -24,8 +24,8 @@ make_species_dat <- function(year_col = "YEAR", month_col = "MONTH", day_col = " # matching the two January species points below) with deliberately different # thetao values at the same locations, so a test can confirm matchData() # picks the value for the correct day rather than whichever day happens to -# come first in envDat. -make_env_dat <- function() { +# come first in env. +make_env <- function() { grid <- expand.grid( lon = c(-70.0, -69.5, -69.0), lat = c(42.0, 42.5, 43.0) @@ -41,19 +41,19 @@ make_env_dat <- function() { # Tests --------------------------------------------------------------------- test_that("matchData renames YEAR/MONTH/DAY columns regardless of input naming", { - speciesDat <- make_species_dat(year_col = "Year", month_col = "Month", day_col = "Day") - envDat <- make_env_dat() + observations <- make_observations(year_col = "Year", month_col = "Month", day_col = "Day") + env <- make_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) expect_true(all(c("YEAR", "MONTH", "DAY") %in% names(result))) }) test_that("matchData joins the correct environmental value via nearest feature and matching day", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() + observations <- make_observations() + env <- make_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) # Species point 1: Jan 1 2020, sits exactly on env grid point 1 -> the Jan-1 value expect_equal(result$thetao[result$id == 1], 11) @@ -67,19 +67,19 @@ test_that("matchData joins the correct environmental value via nearest feature a }) test_that("matchData returns one row per species observation", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() + observations <- make_observations() + env <- make_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) - expect_equal(nrow(result), nrow(speciesDat)) + expect_equal(nrow(result), nrow(observations)) }) test_that("matchData assigns LAT and LON correctly (not swapped)", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() + observations <- make_observations() + env <- make_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) row <- result[result$id == 1, ] expect_equal(row$LON, -70.0, tolerance = 1e-6) @@ -87,12 +87,12 @@ test_that("matchData assigns LAT and LON correctly (not swapped)", { }) test_that("matchData drops YEAR/MONTH duplication from the env side", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() + observations <- make_observations() + env <- make_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) - # YEAR/MONTH should appear exactly once each (from speciesDat), not duplicated + # YEAR/MONTH should appear exactly once each (from observations), not duplicated expect_equal(sum(names(result) == "YEAR"), 1) expect_equal(sum(names(result) == "MONTH"), 1) }) @@ -102,7 +102,7 @@ test_that("matchData drops YEAR/MONTH duplication from the env side", { # Monthly products (e.g. Copernicus "...P1M-m" means) carry one time step per # month, conventionally stamped on a single nominal day. Observations still fall # on arbitrary days, so a day-exact join would match nothing. -make_monthly_env_dat <- function(nominal_day = 1) { +make_monthly_env <- function(nominal_day = 1) { grid <- expand.grid( lon = c(-70.0, -69.5, -69.0), lat = c(42.0, 42.5, 43.0) @@ -115,15 +115,15 @@ make_monthly_env_dat <- function(nominal_day = 1) { } test_that("detect_temporal_resolution reads resolution off the time steps", { - expect_equal(detect_temporal_resolution(make_env_dat()), "day") - expect_equal(detect_temporal_resolution(make_monthly_env_dat()), "month") + expect_equal(detect_temporal_resolution(make_env()), "day") + expect_equal(detect_temporal_resolution(make_monthly_env()), "month") }) test_that("detect_temporal_resolution prefers month over year when ambiguous", { # One month of monthly data looks identical to one year of annual data. Falling # back to "month" keeps unmatched observations unmatched (and warned about), # rather than silently matching them to another month's time step. - one_month <- make_monthly_env_dat() + one_month <- make_monthly_env() one_month <- one_month[one_month$MONTH == 1, ] expect_equal(detect_temporal_resolution(one_month), "month") @@ -137,14 +137,14 @@ test_that("detect_temporal_resolution prefers month over year when ambiguous", { }) test_that("matchData matches monthly env data whose nominal day never matches observations", { - speciesDat <- make_species_dat() + observations <- make_observations() # Env data is stamped on day 15; observations are on days 1 and 15. A day-exact # join would drop both January-day-1 and February-day-1 observations entirely. - envDat <- make_monthly_env_dat(nominal_day = 15) + env <- make_monthly_env(nominal_day = 15) - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) - expect_equal(nrow(result), nrow(speciesDat)) + expect_equal(nrow(result), nrow(observations)) expect_false(any(is.na(result$thetao))) # Point 1: Jan, on grid point 1 -> January value at that location expect_equal(result$thetao[result$id == 1], 11) @@ -153,62 +153,62 @@ test_that("matchData matches monthly env data whose nominal day never matches ob }) test_that("matchData still matches per-day when env data is daily", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() + observations <- make_observations() + env <- make_env() # Explicitly requesting month resolution against daily data is ambiguous by # design; auto-detection is what keeps daily data matching per day. - result <- matchData(speciesDat, envDat, temporal_resolution = "day") + result <- matchData(observations, env, temporal_resolution = "day") expect_equal(result$thetao[result$id == 2], 115) }) test_that("matchData works on species data with no day column at monthly resolution", { - speciesDat <- make_species_dat() - speciesDat$DAY <- NULL - envDat <- make_monthly_env_dat() + observations <- make_observations() + observations$DAY <- NULL + env <- make_monthly_env() - result <- matchData(speciesDat, envDat) + result <- matchData(observations, env) - expect_equal(nrow(result), nrow(speciesDat)) + expect_equal(nrow(result), nrow(observations)) expect_false(any(is.na(result$thetao))) }) test_that("matchData resolves an exact day column ahead of a prefix match", { - speciesDat <- make_species_dat() - speciesDat$dayofyear <- c(1, 15, 32) - envDat <- make_env_dat() + observations <- make_observations() + observations$dayofyear <- c(1, 15, 32) + env <- make_env() # Both "DAY" and "dayofyear" start with "day"; the exact match must win rather # than the lookup failing as ambiguous. - expect_no_error(matchData(speciesDat, envDat)) + expect_no_error(matchData(observations, env)) }) test_that("matchData keeps observations in periods with no env data, as NA", { - speciesDat <- make_species_dat() + observations <- make_observations() # Env data covers January only; the February observation has no match. - envDat <- make_monthly_env_dat() - envDat <- envDat[envDat$MONTH == 1, ] + env <- make_monthly_env() + env <- env[env$MONTH == 1, ] - expect_warning(result <- matchData(speciesDat, envDat), "No environmental data") + expect_warning(result <- matchData(observations, env), "No data in `source`") - expect_equal(nrow(result), nrow(speciesDat)) + expect_equal(nrow(result), nrow(observations)) expect_true(is.na(result$thetao[result$id == 3])) expect_false(is.na(result$thetao[result$id == 1])) }) -test_that("matchData does not depend on the order periods appear in speciesDat", { - speciesDat <- make_species_dat() - envDat <- make_env_dat() +test_that("matchData does not depend on the order periods appear in observations", { + observations <- make_observations() + env <- make_env() # Reversing row order puts the chronologically last period first. The previous # implementation initialized its accumulator only on the chronologically first # period, so this ordering made it fail outright. - reversed <- speciesDat[rev(seq_len(nrow(speciesDat))), ] + reversed <- observations[rev(seq_len(nrow(observations))), ] - result <- matchData(reversed, envDat) + result <- matchData(reversed, env) - expect_equal(nrow(result), nrow(speciesDat)) + expect_equal(nrow(result), nrow(observations)) expect_equal(result$thetao[result$id == 2], 115) }) @@ -223,13 +223,13 @@ test_that("a missing CRS is named, on whichever side it is missing", { # neither which object nor what to do about it. expect_error( matchData(sf::st_as_sf(obs, coords = c("lon", "lat")), env), - "speciesDat has no coordinate reference system") + "`dat` has no coordinate reference system") env_no_crs <- env sf::st_crs(env_no_crs) <- NA expect_error( matchData(sf::st_as_sf(obs, coords = c("lon", "lat"), crs = 4326), env_no_crs), - "envDat has no coordinate reference system") + "`source` has no coordinate reference system") }) test_that("projected observations match the same cells as geographic ones", { @@ -246,3 +246,63 @@ test_that("projected observations match the same cells as geographic ones", { expect_equal(suppressWarnings(matchData(projected, env))$SST, suppressWarnings(matchData(geographic, env))$SST) }) + +test_that("the deprecated argument names still work, with a warning", { + # taupatch and any script written against the old signature call these by + # name. Breaking them silently would be worse than carrying the shim. + observations <- make_observations() + env <- make_env() + + # Both names warn, so both have to be caught or the second escapes the test. + expect_warning( + expect_warning(matchData(speciesDat = observations, envDat = env), + "`speciesDat` is now `dat`"), + "`envDat` is now `source`") + + old <- suppressWarnings(matchData(speciesDat = observations, envDat = env)) + new <- matchData(dat = observations, source = env) + + expect_equal(sf::st_drop_geometry(old), sf::st_drop_geometry(new)) +}) + +test_that("the new names work positionally and by name", { + observations <- make_observations() + env <- make_env() + + expect_equal(sf::st_drop_geometry(matchData(observations, env)), + sf::st_drop_geometry(matchData(dat = observations, source = env))) + expect_silent(matchData(observations, env)) +}) + +test_that("a colliding column is suffixed rather than overwriting the caller's", { + # The suffix is ".matched" now: the join is no longer specific to + # environmental data, so ".env" described only one use of it. + observations <- make_observations() + env <- make_env() + observations$thetao <- seq_len(nrow(observations)) * 100 + + result <- matchData(observations, env) + + expect_true(all(c("thetao", "thetao.matched") %in% names(result))) + # The caller's own column is untouched. + expect_equal(result$thetao, seq_len(nrow(observations)) * 100) +}) + +test_that("neither side has to be observations or a covariate grid", { + # The generalisation this rename is about: two gridded products matched to + # each other, with nothing species-shaped involved. + grid_a <- sf::st_as_sf( + data.frame(x = rep(seq(-70, -69, by = 0.5), 2), y = rep(c(42, 43), each = 3), + SST = 1:6, YEAR = 2010L, MONTH = 1L, DAY = 1L), + coords = c("x", "y"), crs = 4326) + grid_b <- sf::st_as_sf( + data.frame(x = c(-69.75, -69.25), y = c(42, 43), + CHL = c(0.5, 0.9), YEAR = 2010L, MONTH = 1L, DAY = 1L), + coords = c("x", "y"), crs = 4326) + + result <- matchData(grid_a, grid_b) + + expect_equal(nrow(result), nrow(grid_a)) + expect_true(all(c("SST", "CHL") %in% names(result))) + expect_false(anyNA(result$CHL)) +})