diff --git a/R/accessors.R b/R/accessors.R index 34c0f56..207bb48 100644 --- a/R/accessors.R +++ b/R/accessors.R @@ -1,6 +1,6 @@ .get_resource_data <- function(x, resource_name) { check_gldp(x) - frictionless::read_resource(x, resource_name = resource_name) + gldp_read_resource(x, resource_name = resource_name) } .set_resource_data <- function(x, value, resource_name, post = identity) { diff --git a/R/add_gldp_resource.R b/R/add_gldp_resource.R index bd25b2d..cc05faf 100644 --- a/R/add_gldp_resource.R +++ b/R/add_gldp_resource.R @@ -44,10 +44,9 @@ add_gldp_resource <- function( # Retrieve full schema (pkg$resources) does not have schema at first pkg_schema <- gldp_profile_schema(version) + # The first branch is the one for the fixed tables. possible_gldp_resources <- - pkg_schema$allOf[[2]]$properties$resources$items$oneOf[[ - 1 - ]]$properties$name$enum + gldp_profile_resource_branches(pkg_schema)[[1]]$properties$name$enum if (!resource_name %in% possible_gldp_resources) { cli_abort(c( @@ -139,17 +138,35 @@ add_gldp_resource <- function( delim = delim ) - # GeoLocator-DP requires `type: "table"` on its tabular resources from v1.1. - # frictionless only sets it from version 2.0, so set it here to stay correct - # on both. Every profile version declares the property, so this is safe for - # older packages too. + # GeoLocator-DP requires `$schema` and `type: "table"` on its tabular resources + # from v1.1. frictionless only sets them from version 2.0, so set them here to + # stay correct on both. Every profile version declares both properties, so this + # is safe for older packages too. + # + # `profile: "tabular-data-resource"` is deliberately left in place: it is the + # Data Package v1 spelling that `$schema` and `type` replace, but frictionless + # 1.3.0 refuses to read a resource without it. resource <- gldp_resource(pkg, resource_name) + changed <- FALSE + + if (!identical(resource[["$schema"]], .gldp_data_resource_profile)) { + resource <- append( + list(`$schema` = .gldp_data_resource_profile), + resource + ) + changed <- TRUE + } + if (!identical(resource$type, "table")) { resource <- append( resource, list(type = "table"), after = which(names(resource) == "name") ) + changed <- TRUE + } + + if (changed) { gldp_resource(pkg, resource_name) <- resource } diff --git a/R/merge_gldp.R b/R/merge_gldp.R index 0410342..5073e2b 100644 --- a/R/merge_gldp.R +++ b/R/merge_gldp.R @@ -224,7 +224,7 @@ merge_gldp_prepare_tags <- function(pkg, resources) { return(NULL) } - tags <- frictionless::read_resource(pkg, resource_name = "tags") + tags <- gldp_read_resource(pkg, resource_name = "tags") if (!"datapackage_id" %in% names(tags)) { tags$datapackage_id <- pkg$id } @@ -246,7 +246,7 @@ merge_gldp_resources <- function(pkgs, resources, resources_by_pkg, tags_by_pkg) if (!resource_name %in% resources_by_pkg[[idx]]) { return(NULL) } - frictionless::read_resource(pkgs[[idx]], resource_name = resource_name) + gldp_read_resource(pkgs[[idx]], resource_name = resource_name) } ) data_parts <- purrr::compact(data_parts) diff --git a/R/read_gldp.R b/R/read_gldp.R index b1cbe82..814d67e 100644 --- a/R/read_gldp.R +++ b/R/read_gldp.R @@ -124,7 +124,7 @@ read_gldp <- function(x = "datapackage.json", force_read = TRUE, drop_measuremen has_parsing_warning <- FALSE df <- tryCatch( withCallingHandlers( - frictionless::read_resource(pkg, resource_name), + gldp_read_resource(pkg, resource_name), warning = function(w) { msg <- conditionMessage(w) if (grepl("One or more parsing issues", msg, fixed = TRUE)) { diff --git a/R/update_gldp.R b/R/update_gldp.R index c3ba31b..09c290b 100644 --- a/R/update_gldp.R +++ b/R/update_gldp.R @@ -45,9 +45,7 @@ update_gldp_order_resources <- function(pkg) { pkg_schema <- gldp_profile_schema(version) - resource_order <- pkg_schema$allOf[[2]]$properties$resources$items$oneOf |> - purrr::map(~ .x$properties$name$enum %||% .x$properties$name$const %||% character(0)) |> - purrr::flatten_chr() + resource_order <- gldp_profile_resource_names(pkg_schema) if (length(resource_order) == 0) { return(pkg) diff --git a/R/upgrade_gldp.R b/R/upgrade_gldp.R index c0cf81e..1ca4c49 100644 --- a/R/upgrade_gldp.R +++ b/R/upgrade_gldp.R @@ -43,7 +43,8 @@ upgrade_gldp <- function(x, to_version = .gldp_default_version) { "v0.3" = upgrade_gldp_v0_3_to_v0_4(x), "v0.4" = upgrade_gldp_v0_4_to_v0_5(x), "v0.5" = upgrade_gldp_v0_5_to_v0_6(x), - "v0.6" = upgrade_gldp_v0_6_to_v1_0(x) + "v0.6" = upgrade_gldp_v0_6_to_v1_0(x), + "v1.0" = upgrade_gldp_v1_0_to_v1_1(x) ) from_version <- gldp_version(x) @@ -62,9 +63,7 @@ normalize_upgraded_resources <- function(x) { # We only touch resources already present in the package and keep custom resources unchanged. target_version <- gldp_version(x) profile <- gldp_profile_schema(target_version) - supported_resources <- profile$allOf[[2]]$properties$resources$items$oneOf |> - purrr::map(~ .x$properties$name$enum %||% .x$properties$name$const %||% character(0)) |> - purrr::flatten_chr() + supported_resources <- gldp_profile_resource_names(profile) resources <- x$resources %||% list() for (resource in resources) { @@ -345,6 +344,16 @@ upgrade_gldp_v0_6_to_v1_0 <- function(x) { x } +#' @noRd +upgrade_gldp_v1_0_to_v1_1 <- function(x) { + # Release: https://github.com/GeoPressure/GeoLocator-DP/releases/tag/v1.1 + # The table schema moved from `resource$$schema` to `resource$schema`, and + # `type` became required. Resources are rebuilt against the v1.1 schemas by + # `normalize_upgraded_resources()`, so only the profile URL changes here. + x[["$schema"]] <- gldp_schema_url("v1.1") + x +} + #' @noRd mutate_resource <- function(pkg, resource_name, fn) { resource <- gldp_resource(pkg, resource_name) diff --git a/R/validate_gldp.R b/R/validate_gldp.R index e7314f3..adea514 100644 --- a/R/validate_gldp.R +++ b/R/validate_gldp.R @@ -213,6 +213,26 @@ validate_gldp_table <- function(data, schema) { schema$fieldsMatch[[1]] ) + # `fieldsMatch` is a rule over the whole set of fields and says nothing about + # which ones are required, so every mode GeoLocator-DP uses would let a table + # leave out a column carrying `required`. The standard has no way to say "the + # required columns must be present, the optional ones may be absent" + # (frictionlessdata/datapackage#1126), so it is checked here instead. + required <- names(schema_fields)[vapply( + schema_fields, + \(f) isTRUE(f$constraints$required), + logical(1) + )] + absent_required <- setdiff(required, names(data)) + + if (length(absent_required) > 0) { + cli_alert_danger( + "{.field {schema$name}} is missing {length(absent_required)} required + column{?s}: {.field {absent_required}}." + ) + valid <- FALSE + } + fields <- intersect(names(schema_fields), names(data)) for (field in fields) { diff --git a/R/validate_gldp_schema.R b/R/validate_gldp_schema.R index 267b47b..0a0b6ad 100644 --- a/R/validate_gldp_schema.R +++ b/R/validate_gldp_schema.R @@ -374,21 +374,31 @@ validate_gldp_item <- function(item, prop, field) { } } - # Check if 'enum' property exists and is specified - if (!is.null(prop$enum)) { - # Check if each item is one of the allowed values in the enum - if (required) { - in_enum <- item %in% unlist(prop$enum) - } else { + # Both `categories` and the `enum` constraint restrict a field to a fixed set + # of values. GeoLocator-DP declares both, `categories` being the Data Package + # v2 spelling that also carries a label per value, so read whichever is there. + allowed <- if (!is.null(prop$categories)) { + vapply( + prop$categories, + \(x) if (is.list(x)) as.character(x$value)[1] else as.character(x)[1], + character(1) + ) + } else if (!is.null(prop$enum)) { + as.character(unlist(prop$enum)) + } + + if (!is.null(allowed)) { + in_allowed <- item %in% allowed + if (!required) { # Allow NA if not required - in_enum <- (item %in% unlist(prop$enum)) | is.na(item) + in_allowed <- in_allowed | is.na(item) } - if (!all(in_enum)) { - invalid_values <- item[!in_enum] + if (!all(in_allowed)) { + invalid_values <- item[!in_allowed] cli_alert_danger( - "{.field {field}} has {sum(!in_enum)} item{?s} that are not in the allowed values: - {.val {glue::glue_collapse(prop$enum, sep = ', ')}}. Invalid value{?s}: + "{.field {field}} has {sum(!in_allowed)} item{?s} that are not in the allowed values: + {.val {glue::glue_collapse(allowed, sep = ', ')}}. Invalid value{?s}: {.val {glue::glue_collapse(unique(invalid_values), sep = ', ')}}" ) valid <- FALSE diff --git a/R/versioning.R b/R/versioning.R index 463d50a..c01930e 100644 --- a/R/versioning.R +++ b/R/versioning.R @@ -1,6 +1,24 @@ # Supported GeoLocator-DP versions. # This vector is the single source of truth for runtime and schema sync. -.gldp_supported_versions <- c("main", "v0.1", "v0.2", "v0.3", "v0.4", "v0.5", "v0.6", "v1.0") +.gldp_supported_versions <- c( + "main", + "v0.1", + "v0.2", + "v0.3", + "v0.4", + "v0.5", + "v0.6", + "v1.0", + "v1.1" +) + +#' Data Package profile of a tabular resource descriptor +#' +#' GeoLocator-DP requires this on its tabular resources from v1.1. +#' +#' @noRd +.gldp_data_resource_profile <- "https://datapackage.org/profiles/2.0/dataresource.json" + .gldp_default_version <- utils::tail(.gldp_supported_versions, 1) #' Get GeoLocator DP version diff --git a/R/zzz.R b/R/zzz.R index 824e657..2653c45 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -85,6 +85,181 @@ gldp_resource <- function(pkg, resource_name) { pkg } +#' Column names of the file backing a resource +#' +#' @param pkg A GeoLocator Data Package object. +#' @param resource A resource descriptor. +#' @return Character vector of column names, or `NULL` when the file cannot be +#' read or the resource has no file. +#' @noRd +gldp_resource_header <- function(pkg, resource) { + path <- as.character(resource$path %||% character(0))[1] + if (is.na(path) || !nzchar(path)) { + return(NULL) + } + + if (!grepl("^https?://", path)) { + directory <- attr(pkg, "directory") + if (is_non_empty_string(directory)) { + path <- file.path(directory, path) + } + if (!file.exists(path)) { + return(NULL) + } + } + + # Only the header is needed, so the rest of the file is never parsed. + tryCatch( + names(readr::read_delim( + path, + delim = as.character(resource$dialect$delimiter %||% ",")[1], + n_max = 0, + show_col_types = FALSE, + progress = FALSE, + locale = readr::locale( + encoding = as.character(resource$encoding %||% "UTF-8")[1] + ) + )), + error = \(e) NULL, + warning = \(w) NULL + ) +} + +#' Match a resource schema to the columns its file actually has +#' +#' @description +#' [frictionless::read_resource()] maps schema fields to data columns by +#' position, because `fieldsMatch` is not implemented +#' (frictionlessdata/frictionless-r#216). A file that leaves out a column its +#' schema declares is therefore read with every later column under the wrong +#' name, and neither frictionless nor GeoLocatoR notices: by the time the table +#' is validated, the names it is checked against are the ones frictionless +#' assigned. +#' +#' Every GeoLocator-DP table declares `fieldsMatch: superset` or `partial`, both +#' of which let a file leave columns out, so this reconciles the schema with the +#' header before reading. Declared fields absent from the file are dropped, the +#' rest are put in file order, and columns the schema does not describe are +#' described as `any` when `fieldsMatch` allows them. +#' +#' @param pkg A GeoLocator Data Package object. +#' @param resource_name Name of the resource to align. +#' @return `pkg`, with that resource's schema matched to its file. +#' @noRd +gldp_align_schema_to_header <- function(pkg, resource_name) { + resource <- gldp_resource(pkg, resource_name) + + # Nothing to align against: a resource carrying its data inline already has + # its column names, and a schema given by URL is not resolved here. + if ( + is.null(resource) || + is.null(resource$path) || + !is.list(resource$schema) || + length(resource$schema$fields) == 0 + ) { + return(pkg) + } + + schema <- resource$schema + declared <- schema_field_names(schema) + header <- gldp_resource_header(pkg, resource) + + if (is.null(header) || identical(header, declared)) { + return(pkg) + } + + fields_match <- as.character(schema$fieldsMatch %||% "exact")[1] + undeclared <- setdiff(header, declared) + + if (length(intersect(header, declared)) == 0) { + cli_abort( + c( + "x" = "None of the columns of {.val {resource_name}} are declared by its + schema.", + "i" = "Columns found: {.field {header}}.", + "i" = "Expected some of: {.field {declared}}." + ), + class = "gldp_error_header_unrelated" + ) + } + + # `subset` and `partial` are the modes that allow a column the schema does not + # define; the others make one an error rather than something to describe. + if (length(undeclared) > 0 && !fields_match %in% c("subset", "partial")) { + cli_abort( + c( + "x" = "{.val {resource_name}} has {length(undeclared)} column{?s} its + schema does not define: {.field {undeclared}}.", + "i" = "{.val {resource_name}} declares {.code fieldsMatch: {fields_match}}, + which allows only the columns its schema defines." + ), + class = "gldp_error_undeclared_columns" + ) + } + + fields <- stats::setNames(schema$fields, declared) + schema$fields <- purrr::map( + header, + \(name) fields[[name]] %||% list(name = name, type = "any") + ) + resource$schema <- schema + gldp_resource(pkg, resource_name) <- resource + + pkg +} + +#' Read a resource, matching its schema to the file by column name +#' +#' @param pkg A GeoLocator Data Package object. +#' @param resource_name Name of the resource to read. +#' @param ... Passed to [frictionless::read_resource()]. +#' @return The resource data. +#' @noRd +gldp_read_resource <- function(pkg, resource_name, ...) { + pkg <- gldp_align_schema_to_header(pkg, resource_name) + frictionless::read_resource(pkg, resource_name, ...) +} + +#' Resource branches declared by a GeoLocator-DP profile +#' +#' @description +#' A profile constrains each resource through a list of alternatives keyed on +#' the resource `name`: one for the fixed tables, one for `params`, one for +#' custom resources. +#' +#' Up to GeoLocator-DP v1.0 that list is a `oneOf`. From v1.1 it is an `allOf` +#' of `if`/`then` pairs, so a validator can report the property at fault rather +#' than only that the resource matched no branch. Both shapes are read here +#' because `upgrade_gldp()` walks packages written against any earlier version. +#' +#' @param profile A GeoLocator-DP profile, as returned by +#' [gldp_profile_schema()]. +#' @return List of branch schemas, tables first. +#' @noRd +gldp_profile_resource_branches <- function(profile) { + items <- profile$allOf[[2]]$properties$resources$items + + if (!is.null(items$oneOf)) { + return(items$oneOf) + } + + purrr::map(items$allOf %||% list(), ~ .x$then) +} + +#' Resource names a GeoLocator-DP profile declares +#' +#' @param profile A GeoLocator-DP profile. +#' @return Character vector of names, in the order the profile declares them. +#' Custom resources contribute nothing, having no fixed name. +#' @noRd +gldp_profile_resource_names <- function(profile) { + gldp_profile_resource_branches(profile) |> + purrr::map( + ~ .x$properties$name$enum %||% .x$properties$name$const %||% character(0) + ) |> + purrr::flatten_chr() +} + #' Locate a resource by name #' #' @param pkg A GeoLocator Data Package object. diff --git a/data-raw/sync_schemas.R b/data-raw/sync_schemas.R index 4ed8be4..86d87f4 100644 --- a/data-raw/sync_schemas.R +++ b/data-raw/sync_schemas.R @@ -10,6 +10,7 @@ # sync_gldp_schemas() source("R/versioning.R") +source("R/zzz.R") sync_gldp_schemas <- function( versions = setdiff(.gldp_supported_versions, "main"), @@ -35,16 +36,7 @@ sync_gldp_schemas <- function( simplifyVector = TRUE ) - one_of <- profile$allOf[[2]]$properties$resources$items$oneOf - resource_names <- unique(unlist(lapply(one_of, function(x) { - if (!is.null(x$properties$name$enum)) { - x$properties$name$enum - } else if (!is.null(x$properties$name$const)) { - x$properties$name$const - } else { - character(0) - } - }))) + resource_names <- unique(gldp_profile_resource_names(profile)) for (resource_name in resource_names) { schema_url <- glue::glue("{base_url}/{resource_name}-table-schema.json") diff --git a/inst/schemas/v1.1/edges-table-schema.json b/inst/schemas/v1.1/edges-table-schema.json new file mode 100644 index 0000000..3b8e154 --- /dev/null +++ b/inst/schemas/v1.1/edges-table-schema.json @@ -0,0 +1,239 @@ +{ + "name": "edges", + "title": "Edges", + "description": "In GeoPressureR, `edges` (graph edges; see more information on the movement model in the [GeoPressureManual](https://geopressure.com/GeoPressureManual/trajectory.html)) are used to extract flight information from GeoPressureR [`paths`](/GeoLocator-DP/geopressurer/paths) using `path2edge()`.", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "type", + "description": "Same as [`paths.type`](/GeoLocator-DP/geopressurer/paths#type). One of: `tag`, `most_likely`, `simulation` or `geopressureviz`.", + "type": "string", + "categories": [ + { + "value": "tag", + "label": "Tag" + }, + { + "value": "most_likely", + "label": "Most likely" + }, + { + "value": "simulation", + "label": "Simulation" + }, + { + "value": "geopressureviz", + "label": "GeoPressureViz" + } + ], + "constraints": { + "required": true, + "enum": ["tag", "most_likely", "simulation", "geopressureviz"] + }, + "example": "tag" + }, + { + "name": "stap_s", + "description": "Identifier of the SOURCE stationary period. Together with [`tag_id`](#tag_id), this forms a foreign key to [`staps`](/GeoLocator-DP/geopressurer/staps) (`tag_id`, `stap_id`).", + "type": "number", + "constraints": { + "required": true + }, + "example": "3" + }, + { + "name": "stap_t", + "description": "Identifier of the TARGET stationary period. Generally `stap_s+1`, except if `include=FALSE` for intermediate `stap_id` values. Together with [`tag_id`](#tag_id), this forms a foreign key to [`staps`](/GeoLocator-DP/geopressurer/staps) (`tag_id`, `stap_id`).", + "type": "number", + "constraints": { + "required": true + }, + "example": "4" + }, + { + "name": "lat_s", + "description": "Latitude of the source stationary period.", + "type": "number", + "constraints": { + "required": true, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude" + }, + { + "name": "lon_s", + "description": "Longitude of the source stationary period.", + "type": "number", + "constraints": { + "required": true, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude" + }, + { + "name": "lat_t", + "description": "Latitude of the target stationary period.", + "type": "number", + "constraints": { + "required": true, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude" + }, + { + "name": "lon_t", + "description": "Longitude of the target stationary period.", + "type": "number", + "constraints": { + "required": true, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude" + }, + { + "name": "j", + "description": "Same as [`paths.j`](/GeoLocator-DP/geopressurer/paths#j). Unique identifier for each trajectory. Only useful for `edges.type='simulation'`. Integer from `1` to `nj` as defined in [`GeoPressureR::graph_simulation()`](https://geopressure.com/GeoPressureR/reference/graph_simulation).", + "type": "integer", + "constraints": { + "required": false, + "minimum": 1 + }, + "example": "1" + }, + { + "name": "start", + "description": "Date and time of the start of the flight. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-01T05:50:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "end", + "description": "Date and time of the end of the flight. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-02T21:18:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "n", + "description": "Flight number.", + "type": "integer", + "constraints": { + "required": false, + "minimum": 1 + }, + "example": "1" + }, + { + "name": "distance", + "description": "Distance of the flight in km.", + "type": "number", + "constraints": { + "required": false, + "minimum": 0 + }, + "unit": "km", + "example": "235" + }, + { + "name": "bearing", + "description": "Direction of the flight in degrees. 0 indicates north and values increase clockwise. In general, this should be 0°-360°, but -180° to 180° is also allowed.", + "type": "number", + "constraints": { + "required": false, + "minimum": -180, + "maximum": 360 + }, + "unit": "degree", + "example": "30" + }, + { + "name": "gs_u", + "description": "The average groundspeed of the entire flight along the longitudinal axis (east-west direction), measured in kilometers per hour (km/h).", + "type": "number", + "constraints": { + "required": false + }, + "unit": "km/h", + "example": "35.5" + }, + { + "name": "gs_v", + "description": "The average groundspeed of the entire flight along the latitudinal axis (north-south direction), measured in kilometers per hour (km/h).", + "type": "number", + "constraints": { + "required": false + }, + "unit": "km/h", + "example": "45.1" + }, + { + "name": "ws_u", + "description": "The average windspeed of the entire flight along the longitudinal axis (east-west direction), measured in kilometers per hour (km/h).", + "type": "number", + "constraints": { + "required": false + }, + "unit": "km/h", + "example": "15.1" + }, + { + "name": "ws_v", + "description": "The average windspeed of the entire flight along the latitudinal axis (north-south direction), measured in kilometers per hour (km/h).", + "type": "number", + "constraints": { + "required": false + }, + "unit": "km/h", + "example": "5.5" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + }, + { + "fields": ["tag_id", "stap_s"], + "reference": { + "resource": "staps", + "fields": ["tag_id", "stap_id"] + } + }, + { + "fields": ["tag_id", "stap_t"], + "reference": { + "resource": "staps", + "fields": ["tag_id", "stap_id"] + } + } + ], + "fieldsMatch": "superset" +} diff --git a/inst/schemas/v1.1/geolocator-dp-profile.json b/inst/schemas/v1.1/geolocator-dp-profile.json new file mode 100644 index 0000000..b2d3c86 --- /dev/null +++ b/inst/schemas/v1.1/geolocator-dp-profile.json @@ -0,0 +1,173 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Geolocator Data Package", + "description": "Data exchange format for geolocator data.", + "type": "object", + "allOf": [ + { + "$ref": "https://datapackage.org/profiles/2.0/datapackage.json" + }, + { + "required": ["$schema", "resources"], + "properties": { + "$schema": { + "title": "Profile", + "description": "URL of the GeoLocator DP profile providing the version information. Use a short Git ref in the URL path. URLs using `refs/tags` are also accepted. See [Data Package specification](https://datapackage.org/standard/data-package/#dollar-schema).", + "type": "string", + "format": "uri", + "pattern": "^https://raw\\.githubusercontent\\.com/GeoPressure/GeoLocator-DP/(?:(?:v\\d+\\.\\d+(?:\\.\\d+)?)|dev|(?:refs/tags/(?:v\\d+\\.\\d+(?:\\.\\d+)?|dev)))/geolocator-dp-profile\\.json$", + "examples": [ + "{\n \"$schema\": \"https://raw.githubusercontent.com/GeoPressure/GeoLocator-DP/v1.0/geolocator-dp-profile.json\"\n}\n" + ] + }, + "resources": { + "title": "Data Resources", + "description": "GeoLocator DP supports tabular resources with a fixed table schema (`tags`, `observations`, `measurements`, `staps`, `twilights`, `paths`, `edges`, `pressurepaths`), a `params` JSON resource, and additional custom resources with non-reserved names. See [Data Package specification](https://datapackage.org/standard/data-resource/).", + "type": "array", + "minItems": 3, + "items": { + "type": "object", + "required": ["name"], + "allOf": [ + { + "if": { + "required": ["name"], + "properties": { + "name": { + "enum": [ + "tags", + "observations", + "measurements", + "staps", + "twilights", + "paths", + "edges", + "pressurepaths" + ] + } + } + }, + "then": { + "required": ["$schema", "name", "path", "type", "schema"], + "properties": { + "name": { + "description": "Identifier of the resource. See [Data Package specification](https://datapackage.org/standard/data-resource/#name).", + "enum": [ + "tags", + "observations", + "measurements", + "staps", + "twilights", + "paths", + "edges", + "pressurepaths" + ] + }, + "type": { + "description": "Type of the resource. Only `table` is allowed. See [Data Package specification](https://datapackage.org/standard/data-resource/#type).", + "enum": ["table"] + }, + "path": { + "description": "Path or URL to a CSV resource file (`.csv`) or a zipped CSV file (`.csv.gz`, `.csv.zip`, `.zip`). See [Data Package specification](https://datapackage.org/standard/data-resource/#path).", + "type": "string", + "pattern": "^.+\\.(csv|csv\\.gz|csv\\.zip|zip)(\\?.*)?(#.*)?$" + }, + "$schema": { + "description": "Profile of the resource descriptor, which must be the Data Package v2 data resource profile. Implementations read this property to determine the version of the standard a resource follows. See [Data Package specification](https://datapackage.org/standard/data-resource/#dollar-schema).", + "const": "https://datapackage.org/profiles/2.0/dataresource.json" + }, + "schema": { + "description": "Table Schema of the resource, either the URL of the used GeoLocator DP Table Schema version or an inline Table Schema descriptor. See [Data Package specification](https://datapackage.org/standard/data-resource/#schema).", + "oneOf": [ + { + "type": "string", + "format": "uri", + "pattern": "^https://raw\\.githubusercontent\\.com/GeoPressure/GeoLocator-DP/(?:(?:v\\d+\\.\\d+(?:\\.\\d+)?)|dev|(?:refs/tags/(?:v\\d+\\.\\d+(?:\\.\\d+)?|dev)))/.*-table-schema\\.json$" + }, + { + "type": "object", + "required": ["fields"] + } + ] + } + } + } + }, + { + "if": { + "required": ["name"], + "properties": { + "name": { + "const": "params" + } + } + }, + "then": { + "required": ["name", "path"], + "properties": { + "name": { + "description": "GeoPressureR parameters resource. This JSON file stores the parameters used to build trajectories in the GeoPressureR workflow (the `param` object, organized as `param$$`).", + "const": "params" + }, + "path": { + "description": "Path or URL to a JSON file containing GeoPressureR parameters (typically `params.json`).", + "type": "string", + "pattern": "^.+\\.json(\\?.*)?(#.*)?$" + } + } + } + }, + { + "if": { + "required": ["name"], + "properties": { + "name": { + "not": { + "enum": [ + "tags", + "observations", + "measurements", + "staps", + "twilights", + "paths", + "edges", + "pressurepaths", + "params" + ] + } + } + } + }, + "then": { + "required": ["name", "path"], + "properties": { + "name": { + "description": "Identifier of a custom resource. Must not reuse reserved GeoLocator DP resource names.", + "type": "string", + "not": { + "enum": [ + "tags", + "observations", + "measurements", + "staps", + "twilights", + "paths", + "edges", + "pressurepaths", + "params" + ] + } + }, + "path": { + "description": "Path or URL to the custom resource file." + } + } + } + } + ] + } + } + } + } + ] +} diff --git a/inst/schemas/v1.1/measurements-table-schema.json b/inst/schemas/v1.1/measurements-table-schema.json new file mode 100644 index 0000000..00864a0 --- /dev/null +++ b/inst/schemas/v1.1/measurements-table-schema.json @@ -0,0 +1,158 @@ +{ + "name": "measurements", + "title": "Measurements", + "description": "`measurements.csv` is a [tabular data](https://datapackage.org/standard/glossary/#tabular-data) resource from a GeoLocator Data Package containing raw data from sensors on all tags.\n\n{:.alert .alert-warning}\nNote that data from all sensors and all tags are contained in a single table!", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "sensor", + "description": "Sensor type.\n - `light`: Light-level values measured by a solar geolocator. Range and units may vary by provider and tag settings.\n - `pressure`: Atmospheric pressure in hPa (or mbar).\n - `activity`: Measure of movement performed by the bird, computed from acceleration sensors. Range and computation method may vary by tag provider, model, settings, and placement on the animal. In general, higher values indicate more activity. Typically computed as the cumulative sum of the absolute difference of acceleration along the z-axis. See [doi:10.1186/s40462-018-0137-1](https://doi.org/10.1186/s40462-018-0137-1) (SOI), [doi:10.1111/jav.01068](https://doi.org/10.1111/jav.01068) (Lund), and the [migratetech report](https://www.migratetech.co.uk/IntigeoSummary.pdf) for more details.\n - `mean_acceleration_z`: Mean acceleration along the z-axis, computed from acceleration sensors. Mainly used by SOI devices. This metric is sometimes used as a proxy for pitch and was named `pitch` in GeoLocator DP versions earlier than v1.0.\n - `temperature_external`: Temperature measured by the tag sensor pointing outward from the animal (air temperature). This typically corresponds to the main external temperature sensor. Units: degrees Celsius.\n - `temperature_internal`: Temperature measured by the tag sensor located between the tag and the animal (body temperature). Units: degrees Celsius.\n - `acceleration_x`: Raw acceleration values provided by the tag for the X axis. Units: m/s^2.\n - `acceleration_y`: Raw acceleration values provided by the tag for the Y axis. Units: m/s^2.\n - `acceleration_z`: Raw acceleration values provided by the tag for the Z axis. Units: m/s^2.\n - `magnetic_x`: Magnetic field strength measured by the magnetometer for the X axis. Units: microtesla.\n - `magnetic_y`: Magnetic field strength measured by the magnetometer for the Y axis. Units: microtesla.\n - `magnetic_z`: Magnetic field strength measured by the magnetometer for the Z axis. Units: microtesla.\n - `wet_count`: Wet count during a measurement interval from the wet-dry sensor. Units are usually device-specific.\n - `conductivity`: Conductivity measured by the wet-dry sensor. Unit varies by model.", + "type": "string", + "categories": [ + { + "value": "light", + "label": "Light", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000139/" + }, + { + "value": "pressure", + "label": "Pressure", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000055/" + }, + { + "value": "activity", + "label": "Activity", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000114/" + }, + { + "value": "mean_acceleration_z", + "label": "Mean acceleration (z-axis)" + }, + { + "value": "temperature_internal", + "label": "Internal temperature", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000256/" + }, + { + "value": "temperature_external", + "label": "External temperature", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000104/" + }, + { + "value": "acceleration_x", + "label": "Acceleration (x-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000006/" + }, + { + "value": "acceleration_y", + "label": "Acceleration (y-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000007/" + }, + { + "value": "acceleration_z", + "label": "Acceleration (z-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000008/" + }, + { + "value": "magnetic_x", + "label": "Magnetic field (x-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000258/" + }, + { + "value": "magnetic_y", + "label": "Magnetic field (y-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000259/" + }, + { + "value": "magnetic_z", + "label": "Magnetic field (z-axis)", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000260/" + }, + { + "value": "wet_count", + "label": "Wet count", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000213/" + }, + { + "value": "conductivity", + "label": "Conductivity", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000071/" + } + ], + "constraints": { + "required": true, + "enum": [ + "light", + "pressure", + "activity", + "mean_acceleration_z", + "temperature_internal", + "temperature_external", + "acceleration_x", + "acceleration_y", + "acceleration_z", + "magnetic_x", + "magnetic_y", + "magnetic_z", + "wet_count", + "conductivity" + ] + }, + "skos:relatedMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/", + "example": "pressure" + }, + { + "name": "datetime", + "description": "Date and time of the measurement. Accepts any valid [ISO 8601](https://wikipedia.org/wiki/ISO_8601) datetime string, including datetime in local time (`YYYY-MM-DDThh:mm`), in UTC (`YYYY-MM-DDThh:mmZ`) or any specific timezone (`YYYY-MM-DDThh:mm+hh:mm`).", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-01T22:00:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "value", + "description": "Value of the measurement taken by the sensor.", + "type": "number", + "constraints": { + "required": true + }, + "example": "12.3" + }, + { + "name": "label", + "description": "Labeling of pressure and acceleration data according to the [labeling chapter of the GeoPressureManual](https://geopressure.com/GeoPressureManual/labelling-tracks.html).", + "type": "string", + "constraints": { + "required": false, + "pattern": "^(flight|discard|elev_\\d+|)$" + }, + "example": "elev_23", + "skos:relatedMatch": [ + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000156", + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000157" + ] + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + } + ], + "fieldsMatch": "superset" +} diff --git a/inst/schemas/v1.1/observations-table-schema.json b/inst/schemas/v1.1/observations-table-schema.json new file mode 100644 index 0000000..61ef02f --- /dev/null +++ b/inst/schemas/v1.1/observations-table-schema.json @@ -0,0 +1,495 @@ +{ + "name": "observations", + "title": "observations", + "description": "`observations.csv` is a [tabular data](https://datapackage.org/standard/glossary/#tabular-data) resource from a GeoLocator Data Package containing relevant [events](http://rs.tdwg.org/dwc/terms/Event) (datetime and location) that typically happen in the field and provide important information for analysis. These observations should include at least equipment and, when appropriate, retrieval events, as well as ringing controls or direct sightings.\n\n{:.alert .alert-info}\nYou can think of `observations` as your log entries in your ringing book.\n\nAn observation is described at minimum by `datetime`, location (`latitude` and `longitude`), `ring_number`, and `observation_type` (e.g., equipment, retrieval, or capture). In addition, `sex` and `age_class` are required (use `U` for unknown `sex` and `0` for unknown `age_class`).\n\nAlso include ringing events for any control group (i.e., birds ringed but not equipped with a tag).\n\n **Example:** [`observations.csv`](https://github.com/GeoPressure/GeoLocator-DP/blob/main/example/observations.csv)", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "ring_number", + "description": "Ring number associated with the observed bird. This is the bird-level identifier used to connect observations over time. It is not a strict foreign key because the same `ring_number` can appear in multiple `tags` rows (e.g., multiple deployments on the same bird).", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000022", + "example": "F58445" + }, + { + "name": "tag_id", + "description": "Unique identifier of the tag. `tag_id` is NOT required in this table so that you can enter any relevant ringed species (e.g., control group). Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id). When provided, it should refer to a deployment associated with the same bird (`ring_number`).", + "type": "string", + "constraints": { + "required": false + }, + "example": "18LX" + }, + { + "name": "observation_type", + "description": "Type of observation.\n - `equipment`: Attachment of the tag to the bird. If you equip a new tag at the same time as retrieving an old one from the same bird, create two dedicated rows in the dataset: one for retrieval and one for equipment.\n - `retrieval`: Retrieval of the data, generally through recapture and tag removal, but it can also happen through remote download readout methods (see [`tags.readout_method`](/GeoLocator-DP/core/tags#readout_method)).\n - `capture`: General case of capturing a bird while neither deploying nor retrieving a tag. This option covers (1) recapturing a bird without a device (choose `missing` in `device_status`), (2) a capture before `equipment` or after `retrieval`, and (3) captures of individuals from the control group. Recapture/retrap are not hard-coded in the dataset, but inferred from consecutive captures of the same `ring_number`.\n - `sighting`: Known presence of the bird on a date and location outside ringing (e.g., direct field observation with a color ring).\n - `other`: Any other observation type. Avoid using this; if used, always provide [`observation_comments`](/GeoLocator-DP/core/observations#observation_comments).", + "type": "string", + "categories": [ + { + "value": "equipment", + "label": "Equipment" + }, + { + "value": "retrieval", + "label": "Retrieval" + }, + { + "value": "capture", + "label": "Capture" + }, + { + "value": "sighting", + "label": "Sighting" + }, + { + "value": "other", + "label": "Other" + } + ], + "constraints": { + "required": true, + "enum": ["equipment", "retrieval", "capture", "sighting", "other"] + }, + "example": "equipment" + }, + { + "name": "datetime", + "description": "Date and optionally time of the observation. Accepts any valid [ISO 8601](https://wikipedia.org/wiki/ISO_8601) date or datetime string, including date-only (`YYYY-MM-DD`), full datetime in local time (`YYYY-MM-DDThh:mm`), in UTC (`YYYY-MM-DDThh:mmZ`), or in any specific timezone (`YYYY-MM-DDThh:mm+hh:mm`).", + "skos:broadMatch": "http://rs.tdwg.org/dwc/terms/eventDate", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-03-01T22:00:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000352" + }, + { + "name": "latitude", + "description": "Latitude of the observation location in decimal degrees, using the WGS84 datum.", + "skos:closeMatch": [ + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000074", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000078" + ], + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude", + "type": "number", + "constraints": { + "required": true, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442" + }, + { + "name": "longitude", + "description": "Longitude of the observation location in decimal degrees, using the WGS84 datum.", + "skos:closeMatch": [ + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000075", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000079" + ], + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude", + "type": "number", + "constraints": { + "required": true, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995" + }, + { + "name": "location_name", + "description": "Given name of the observation location. It might be helpful to keep a human-readable name of the location to differentiate the location of different observations more easily.", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/locality", + "type": "string", + "constraints": { + "required": false + }, + "example": "Mwamba center--Nest box #2" + }, + { + "name": "device_status", + "description": "Status of the tag itself (not the data inside the tag):\n - `present`: Tag on the bird.\n - `none`: No tag on the bird and this is expected (e.g., pre-equipment, post-removal, or control group). Only for `observation_type='capture'`.\n - `missing`: Tag lost by the bird. Only for `observation_type='capture'`. If a new tag is deployed on a bird that lost its previous tag, create two rows: one `capture`-`missing` and one `equipment`-`present`.\n - `unknown`: Only for `observation_type='sighting'`.\n\nCan only be `present` for `observation_type` `equipment` or `retrieval`.", + "type": "string", + "categories": [ + { + "value": "present", + "label": "Present" + }, + { + "value": "none", + "label": "None" + }, + { + "value": "missing", + "label": "Missing" + }, + { + "value": "unknown", + "label": "Unknown" + } + ], + "skos:relatedMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000084", + "constraints": { + "required": false, + "enum": ["present", "none", "missing", "unknown"] + }, + "example": "present" + }, + { + "name": "observer", + "description": "The name of the individual or group who made the observation.", + "type": "string", + "constraints": { + "required": false + }, + "skos:exactMatch": [ + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000372/", + "http://rs.tdwg.org/dwc/terms/recordedBy" + ], + "example": "Yann Rime" + }, + { + "name": "catching_method", + "description": "Catching method according to [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=12). Most records use `M` (mist net), `N` (on nest) or `Z` (unknown).", + "type": "string", + "categories": [ + { + "value": "-", + "label": "Not applicable, the bird was not caught" + }, + { + "value": "A", + "label": "Actively triggered trap (by ringer)" + }, + { + "value": "B", + "label": "Trap automatically triggered by the bird" + }, + { + "value": "C", + "label": "Cannon net or rocket net" + }, + { + "value": "D", + "label": "Dazzling" + }, + { + "value": "F", + "label": "Caught in flight by anything other than a static mist net" + }, + { + "value": "G", + "label": "Net just under the water surface, lifted as waterfowl swim over it" + }, + { + "value": "H", + "label": "By hand, with or without hook or noose" + }, + { + "value": "L", + "label": "Clap net" + }, + { + "value": "M", + "label": "Mist net" + }, + { + "value": "N", + "label": "On nest, any method" + }, + { + "value": "O", + "label": "Any other system" + }, + { + "value": "P", + "label": "Phut net" + }, + { + "value": "R", + "label": "Rounded up whilst flightless" + }, + { + "value": "S", + "label": "Bal-chatri or other snare device" + }, + { + "value": "T", + "label": "Helgoland trap or duck decoy" + }, + { + "value": "U", + "label": "Dutch net for Pluvialis apricaria" + }, + { + "value": "V", + "label": "Roosting in cavity" + }, + { + "value": "W", + "label": "Passive walk-in or maze trap" + }, + { + "value": "Z", + "label": "Unknown" + } + ], + "constraints": { + "required": false, + "enum": [ + "-", + "A", + "B", + "C", + "D", + "F", + "G", + "H", + "L", + "M", + "N", + "O", + "P", + "R", + "S", + "T", + "U", + "V", + "W", + "Z" + ] + }, + "skos:relatedMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000397", + "example": "M" + }, + { + "name": "age_class", + "description": "Life stage of the bird according to the age class in [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=14). The code describes the plumage of the bird rather than its age in years.", + "type": "string", + "categories": [ + { + "value": "0", + "label": "Age unknown" + }, + { + "value": "1", + "label": "Pullus: nestling or chick, unable to fly freely" + }, + { + "value": "2", + "label": "Full-grown, year of hatching unknown" + }, + { + "value": "3", + "label": "First-year" + }, + { + "value": "4", + "label": "After first-year" + }, + { + "value": "5", + "label": "2nd year" + }, + { + "value": "6", + "label": "After 2nd year" + }, + { + "value": "7", + "label": "3rd year" + }, + { + "value": "8", + "label": "After 3rd year" + }, + { + "value": "9", + "label": "4th year" + }, + { + "value": "A", + "label": "After 4th year" + }, + { + "value": "B", + "label": "5th year" + }, + { + "value": "C", + "label": "After 5th year" + }, + { + "value": "D", + "label": "6th year" + }, + { + "value": "E", + "label": "After 6th year" + }, + { + "value": "F", + "label": "7th year" + }, + { + "value": "G", + "label": "After 7th year" + }, + { + "value": "H", + "label": "8th year" + } + ], + "constraints": { + "required": true, + "enum": [ + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H" + ] + }, + "skos:closeMatch": [ + "http://rs.tdwg.org/dwc/terms/lifeStage", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000018" + ], + "example": "1" + }, + { + "name": "sex", + "description": "Sex of the bird according to [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=13) as male `M`, female `F` or unknown `U`.", + "skos:closeMatch": [ + "http://rs.tdwg.org/dwc/terms/sex", + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000023" + ], + "type": "string", + "categories": [ + { + "value": "U", + "label": "Unknown" + }, + { + "value": "M", + "label": "Male" + }, + { + "value": "F", + "label": "Female" + } + ], + "constraints": { + "required": true, + "enum": ["U", "M", "F"] + }, + "example": "F" + }, + { + "name": "condition", + "description": "Condition of the bird following a simplified version of the [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=23). Assumed to be `alive` if not provided.", + "type": "string", + "categories": [ + { + "value": "alive", + "label": "Alive and probably healthy" + }, + { + "value": "unhealthy", + "label": "Found sick, wounded or otherwise unhealthy" + }, + { + "value": "dead", + "label": "Dead" + }, + { + "value": "unknown", + "label": "Unknown" + } + ], + "constraints": { + "required": false, + "enum": ["alive", "unhealthy", "dead", "unknown"] + }, + "example": "alive", + "skos:closeMatch": "http://rs.tdwg.org/dwc/terms/vitality" + }, + { + "name": "mass", + "description": "Mass of the tagged bird. See [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=27). Expressed in grams.", + "type": "number", + "constraints": { + "required": false, + "minimum": 0 + }, + "unit": "grams", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000019", + "example": "23.3" + }, + { + "name": "wing_length", + "description": "Maximum chord measurement. See [EURING Exchange Code](https://euring.org/files/documents/E2020ExchangeCodeV201.pdf#page=26). Expressed in millimeters.", + "type": "number", + "constraints": { + "required": false, + "minimum": 0 + }, + "unit": "mm", + "example": "15.4" + }, + { + "name": "additional_metric", + "description": "A list of additional measurements, facts, characteristics, or assertions about the record. Recommended best practice is to use a key:value encoding schema for a data interchange format such as JSON.", + "type": "string", + "constraints": { + "required": false + }, + "skos:exactMatch": ["http://rs.tdwg.org/dwc/terms/dynamicProperties"], + "skos:narrowMatch": [ + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000356", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000354" + ], + "example": "{'RP3':32, 'fat':2, 'tarsusLengthInMeters':0.014, 'weightInGrams':120}" + }, + { + "name": "observation_comments", + "description": "Comments or notes about the observation.", + "skos:broadMatch": [ + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000396", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000077", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000082", + "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000083" + ], + "type": "string", + "constraints": { + "required": false + }, + "example": "" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + } + ], + "fieldsMatch": "superset" +} diff --git a/inst/schemas/v1.1/paths-table-schema.json b/inst/schemas/v1.1/paths-table-schema.json new file mode 100644 index 0000000..6a7d825 --- /dev/null +++ b/inst/schemas/v1.1/paths-table-schema.json @@ -0,0 +1,106 @@ +{ + "name": "paths", + "title": "Paths", + "description": "A GeoPressureR `path` contains positions along a bird trajectory.\n\n{:.alert .alert-info}\nIn GeoPressureR, there are 4 path types: `tag`, `most_likely`, `simulation`, and `geopressureviz`. See [`paths.type`](#type) for more details.\n\n`path` should always be used in combination with [`staps`](/GeoLocator-DP/geopressurer/staps), joined by the composite key (`tag_id`, `stap_id`).", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "type", + "description": "One of:\n - `tag`: Created with [`tag2path()`](https://geopressure.com/GeoPressureR/reference/tag2path) from a likelihood map.\n - `most_likely`: Created with [`graph_most_likely()`](https://geopressure.com/GeoPressureR/reference/graph_most_likely.html).\n - `simulation`: Created by [`graph_simulation()`](https://geopressure.com/GeoPressureR/reference/graph_simulation).\n - `geopressureviz`: Created by [`geopressureviz()`](https://geopressure.com/GeoPressureR/reference/geopressureviz).", + "type": "string", + "categories": [ + { + "value": "tag", + "label": "Tag" + }, + { + "value": "most_likely", + "label": "Most likely" + }, + { + "value": "simulation", + "label": "Simulation" + }, + { + "value": "geopressureviz", + "label": "GeoPressureViz" + } + ], + "constraints": { + "required": true, + "enum": ["tag", "most_likely", "simulation", "geopressureviz"] + }, + "example": "most_likely" + }, + { + "name": "stap_id", + "description": "Identifier of the stationary period. Together with [`tag_id`](#tag_id), this forms a foreign key to [`staps`](/GeoLocator-DP/geopressurer/staps) (`tag_id`, `stap_id`).", + "type": "number", + "constraints": { + "required": true + }, + "example": "3" + }, + { + "name": "lat", + "description": "Latitude of the stationary period.", + "type": "number", + "constraints": { + "required": false, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude" + }, + { + "name": "lon", + "description": "Longitude of the stationary period.", + "type": "number", + "constraints": { + "required": false, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude" + }, + { + "name": "j", + "description": "Unique identifier for each trajectory. Only useful for `paths.type='simulation'`. Integer from `1` to `nj` as defined in [`GeoPressureR::graph_simulation()`](https://geopressure.com/GeoPressureR/reference/graph_simulation).", + "type": "integer", + "constraints": { + "required": false, + "minimum": 1 + }, + "example": "1" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + }, + { + "fields": ["tag_id", "stap_id"], + "reference": { + "resource": "staps", + "fields": ["tag_id", "stap_id"] + } + } + ], + "fieldsMatch": "superset" +} diff --git a/inst/schemas/v1.1/pressurepaths-table-schema.json b/inst/schemas/v1.1/pressurepaths-table-schema.json new file mode 100644 index 0000000..2efa3be --- /dev/null +++ b/inst/schemas/v1.1/pressurepaths-table-schema.json @@ -0,0 +1,109 @@ +{ + "name": "pressurepaths", + "title": "Pressurepaths", + "description": "{:.alert .alert-info}\nA GeoPressureR `pressurepath` contains ERA5 variables along the bird trajectory at hourly intervals, using the exact bird location (including flight segments). [Read more about pressurepaths in the GeoPressureManual](https://geopressure.com/GeoPressureManual/pressurepath.html)\n\nSimilarly to [`path`](/GeoLocator-DP/geopressurer/paths), there are 4 `pressurepath` types: `tag`, `most_likely`, `simulation`, and `geopressureviz`. See [`pressurepaths.type`](#type) for more details.\n\n`pressurepath` should be used in combination with [`staps`](/GeoLocator-DP/geopressurer/staps) and [`paths`](/GeoLocator-DP/geopressurer/paths). Joins are performed on `tag_id` and, for stationary rows, `stap_id`.\n\n{:.alert .alert-warning}\nOnly `tag_id` is defined as a strict foreign key here. `stap_id` can be decimal during migration segments, so it cannot be enforced as a strict foreign key to `staps` in all rows.\n\nThe definition below does not list ERA5 variables, which can (and should!) be added to the table as additional columns (e.g., `'altitude'` or `'surface_pressure'`). Read more about these variables in [the documentation of `pressurepath_create()`](https://geopressure.com/GeoPressureR/reference/pressurepath_create.html#arg-variable).", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "datetime", + "description": "Date and time of the pressure measurement. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-01T22:00:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "stap_id", + "description": "Identifier of the stationary-period index for the trajectory. Integer values correspond to `staps.stap_id` within the same `tag_id`; decimal values indicate pressure measurements taken during migration between two integer stationary periods. For this reason, `stap_id` is not a strict foreign key in this table.", + "type": "number", + "constraints": { + "required": true + }, + "example": "3" + }, + { + "name": "type", + "description": "One of:\n - `tag`: Created with [`tag2path()`](https://geopressure.com/GeoPressureR/reference/tag2path) from a likelihood map.\n - `most_likely`: Created with [`graph_most_likely()`](https://geopressure.com/GeoPressureR/reference/graph_most_likely.html).\n - `simulation`: Created by [`graph_simulation()`](https://geopressure.com/GeoPressureR/reference/graph_simulation).\n - `geopressureviz`: Created by [`geopressureviz()`](https://geopressure.com/GeoPressureR/reference/geopressureviz).", + "type": "string", + "categories": [ + { + "value": "tag", + "label": "Tag" + }, + { + "value": "most_likely", + "label": "Most likely" + }, + { + "value": "simulation", + "label": "Simulation" + }, + { + "value": "geopressureviz", + "label": "GeoPressureViz" + } + ], + "constraints": { + "required": true, + "enum": ["tag", "most_likely", "simulation", "geopressureviz"] + }, + "example": "most_likely" + }, + { + "name": "lat", + "description": "Latitude of the trajectory position at this timestamp.", + "type": "number", + "constraints": { + "required": true, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude" + }, + { + "name": "lon", + "description": "Longitude of the trajectory position at this timestamp.", + "type": "number", + "constraints": { + "required": true, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude" + }, + { + "name": "j", + "description": "Only useful for `pressurepaths.type='simulation'`. Unique identifier for each trajectory, integer from `1` to `nj` as defined in [`GeoPressureR::graph_simulation()`](https://geopressure.com/GeoPressureR/reference/graph_simulation).", + "type": "integer", + "constraints": { + "required": false, + "minimum": 1 + }, + "example": "1" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + } + ], + "fieldsMatch": "partial" +} diff --git a/inst/schemas/v1.1/staps-table-schema.json b/inst/schemas/v1.1/staps-table-schema.json new file mode 100644 index 0000000..c120504 --- /dev/null +++ b/inst/schemas/v1.1/staps-table-schema.json @@ -0,0 +1,83 @@ +{ + "name": "staps", + "title": "Stationary periods", + "description": "A stationary period (`stap`) is [a key concept in the GeoPressureR workflow](https://geopressure.com/GeoPressureManual/tag-object#label-tag-into-stationary-periods) that discretizes a bird trajectory in time and space. During a single `stap`, the bird is assumed to remain at the same location (+/- tens of kilometers). [`flight`](https://geopressure.com/GeoPressureR/reference/stap2flight) is the converse of `stap`, with `flight.start` corresponding to a `stap.end`, and vice versa.\n\n{:.alert .alert-info}\n`stap_id` is indexed within each `tag_id` (it is not globally unique), so the table key is the composite pair (`tag_id`, `stap_id`). `staps` are typically estimated by [`GeoPressureR::tag_label_stap()`](https://geopressure.com/GeoPressureR/reference/tag_label_stap) based on the `'flight'` labels assigned to pressure or acceleration measurements ([`measurements.label`](/GeoLocator-DP/core/measurements#label)). See the [GeoPressureManual](https://geopressure.com/GeoPressureManual/labelling-tracks) for more details.", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "stap_id", + "description": "Identifier of the stationary period for a given `tag_id`.", + "type": "number", + "constraints": { + "required": true, + "minimum": 1 + }, + "example": "3" + }, + { + "name": "start", + "description": "Date and time of the start of the stationary period. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-01T05:50:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "end", + "description": "Date and time of the end of the stationary period. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-02T21:18:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "known_lat", + "description": "Latitude of the known stationary period. This value is typically assigned by [`GeoPressureR::tag_set_map()`](https://geopressure.com/GeoPressureR/reference/tag_set_map).", + "type": "number", + "constraints": { + "required": false, + "minimum": -90, + "maximum": 90 + }, + "example": "52.70442", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLatitude" + }, + { + "name": "known_lon", + "description": "Longitude of the known stationary period. This value is typically assigned by [`GeoPressureR::tag_set_map()`](https://geopressure.com/GeoPressureR/reference/tag_set_map).", + "type": "number", + "constraints": { + "required": false, + "minimum": -180, + "maximum": 180 + }, + "example": "23.84995", + "skos:exactMatch": "http://rs.tdwg.org/dwc/terms/decimalLongitude" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + } + ], + "primaryKey": ["tag_id", "stap_id"], + "fieldsMatch": "superset" +} diff --git a/inst/schemas/v1.1/tags-table-schema.json b/inst/schemas/v1.1/tags-table-schema.json new file mode 100644 index 0000000..1908f0f --- /dev/null +++ b/inst/schemas/v1.1/tags-table-schema.json @@ -0,0 +1,129 @@ +{ + "name": "tags", + "title": "Tags", + "description": "`tags.csv` is a [tabular data](https://datapackage.org/standard/glossary/#tabular-data) resource from a GeoLocator Data Package. Each tag listed in this table is assumed to have been deployed only once on a single bird. This table contains information about the device, the deployment, and the equipped bird. Data collected on the bird that may change over time (e.g., `age_class`) should be provided in the [`observations`](/GeoLocator-DP/core/observations) table.\n\nAdditional data can be provided by following the [Movebank tag attribute dictionary](https://www.movebank.org/cms/movebank-content/movebank-attribute-dictionary#tag_attributes) and [Movebank deployment attribute dictionary](https://www.movebank.org/cms/movebank-content/movebank-attribute-dictionary#deployment_attributes).\n\n **Example:** [`tags.csv`](https://github.com/GeoPressure/GeoLocator-DP/blob/main/example/tags.csv)", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag.", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000181/", + "example": "18LX" + }, + { + "name": "datapackage_id", + "description": "Optional identifier of the data package for this tag, typically corresponding to the `datapackage.id`. This field is mostly used when [merging multiple data packages](https://geopressure.com/GeoLocatoR/reference/merge_gldp.html).", + "type": "string", + "constraints": { + "required": false + }, + "example": "https://doi.org/10.5281/zenodo.11207081" + }, + { + "name": "ring_number", + "description": "Ring number associated with the tagged bird. This value is not unique at table level because the same bird can receive multiple tag deployments (`tag_id`).", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000022", + "example": "F58445" + }, + { + "name": "scientific_name", + "description": "Scientific name of the tagged species.", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": [ + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000024", + "http://rs.tdwg.org/dwc/terms/scientificName" + ], + "example": "Tachymarptis melba" + }, + { + "name": "manufacturer", + "description": "The company or person that produced the tag. When appropriate, please use the terms `'Swiss Ornithological Institute'`, `'Migrate Technology Limited'`, or `'Lund University'`.", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000183", + "example": "Swiss Ornithological Institute" + }, + { + "name": "model", + "description": "Model/design/version of the tag. Any device specification that can identify it.", + "type": "string", + "constraints": { + "required": true + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000185", + "example": "GDL2.3" + }, + { + "name": "firmware", + "description": "The tag firmware and version used during the deployment.", + "type": "string", + "constraints": { + "required": false + }, + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000380", + "example": "v1.0" + }, + { + "name": "weight", + "description": "Weight of the device with harness. Expressed in grams.", + "type": "number", + "constraints": { + "required": false, + "minimum": 0 + }, + "unit": "grams", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000409/", + "example": "1.32" + }, + { + "name": "attachment_type", + "description": "The way a tag is attached to an animal. Use values from the [Movebank `attachment type` concept](https://vocab.nerc.ac.uk/collection/MVB/current/MVB000052) if possible. Assumed to be `harness` if not provided. You can also provide the material used.", + "type": "string", + "constraints": { + "required": false + }, + "example": "leg-loop-harness", + "skos:exactMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000052" + }, + { + "name": "readout_method", + "type": "string", + "description": "The way the data are received from the tag. Use values from the [Movebank `tag readout method` concept](https://vocab.nerc.ac.uk/collection/MVB/current/MVB000188/) if possible. Assumed to be `tag-retrieval` if not provided.", + "constraints": { + "required": false + }, + "skos:relatedMatch": "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000188", + "example": "tag-retrieval" + }, + { + "name": "tag_comments", + "description": "Comments or notes about the tag, deployment, or animal. Use this field to provide additional information that does not fit in other fields, such as data-quality issues, deployment/retrieval details, or analysis notes (e.g., labeling effort, quality checks, or issues encountered).", + "type": "string", + "constraints": { + "required": false + }, + "skos:narrowMatch": [ + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000179", + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000012", + "https://vocab.nerc.ac.uk/collection/MVB/current/MVB000082" + ], + "example": "" + } + ], + "missingValues": ["", "NA"], + "primaryKey": ["tag_id"], + "fieldsMatch": "partial" +} diff --git a/inst/schemas/v1.1/twilights-table-schema.json b/inst/schemas/v1.1/twilights-table-schema.json new file mode 100644 index 0000000..a94befb --- /dev/null +++ b/inst/schemas/v1.1/twilights-table-schema.json @@ -0,0 +1,56 @@ +{ + "name": "twilights", + "title": "Twilights", + "description": "Table containing twilight data inferred from light data for all tags. Twilight data is typically computed with [`GeoPressureR::twilight_create()`](https://geopressure.com/GeoPressureR/reference/twilight_create.html).", + "$schema": "https://datapackage.org/profiles/2.0/tableschema.json", + "fields": [ + { + "name": "tag_id", + "description": "Unique identifier of the tag. Foreign key to [`tags.tag_id`](/GeoLocator-DP/core/tags#tag_id).", + "type": "string", + "constraints": { + "required": true + }, + "example": "18LX" + }, + { + "name": "twilight", + "description": "Date and time of the twilight event. Formatted as an ISO 8601 string with timezone designator `YYYY-MM-DDThh:mm:ssZ`.", + "type": "datetime", + "constraints": { + "required": true + }, + "example": "2020-04-01T22:00:00Z", + "skos:closeMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200" + }, + { + "name": "rise", + "description": "Boolean value to indicate if the twilight is a sunrise (true) or sunset (false).", + "type": "boolean", + "constraints": { + "required": true + }, + "example": "true" + }, + { + "name": "label", + "description": "Labeling of twilight data according to the [GeoPressureManual](https://geopressure.com/GeoPressureManual/light-map.html#manual-labelling-of-twilight). The value `discard` indicates that the twilight should not be used for geopositioning. Any other value can be used without affecting the workflow; setting it to `stap_id` typically helps visualize which twilights belong to the same stationary period in the training set.", + "type": "string", + "constraints": { + "required": false + }, + "example": "discard" + } + ], + "missingValues": ["", "NA"], + "foreignKeys": [ + { + "fields": ["tag_id"], + "reference": { + "resource": "tags", + "fields": ["tag_id"] + } + } + ], + "fieldsMatch": "superset" +} diff --git a/tests/testthat/test-add_gldp_resource.R b/tests/testthat/test-add_gldp_resource.R index cfb3e21..3a86bc6 100644 --- a/tests/testthat/test-add_gldp_resource.R +++ b/tests/testthat/test-add_gldp_resource.R @@ -2,22 +2,40 @@ library(testthat) library(GeoLocatoR) test_that("add_gldp_resource warns when a superset resource drops columns", { - # `tags` declares `fieldsMatch: superset`, so a column its schema does not + # `staps` declares `fieldsMatch: superset`, so a column its schema does not # define cannot be kept. Warn rather than lose it silently. data <- tibble::tibble( tag_id = "a", - ring_number = "r1", - scientific_name = "Turdus merula", + stap_id = 1, my_custom_note = "hello" ) expect_warning( - pkg <- add_gldp_resource(create_gldp(), "tags", data), + pkg <- add_gldp_resource(create_gldp(), "staps", data), class = "gldp_warning_undeclared_columns_dropped" ) expect_false("my_custom_note" %in% names(pkg$resources[[1]]$data)) }) +test_that("add_gldp_resource keeps undeclared columns on tags", { + # `tags` declares `fieldsMatch: partial`, so extra attributes a user carries + # alongside the standard columns are kept and described in the schema. + data <- tibble::tibble( + tag_id = "a", + ring_number = "r1", + movebank_tag_id = 12345 + ) + + expect_no_warning(add_gldp_resource(create_gldp(), "tags", data)) + + pkg <- add_gldp_resource(create_gldp(), "tags", data) + resource <- pkg$resources[[1]] + expect_true("movebank_tag_id" %in% names(resource$data)) + expect_true( + "movebank_tag_id" %in% vapply(resource$schema$fields, \(f) f$name, character(1)) + ) +}) + test_that("add_gldp_resource keeps undeclared columns for a partial resource", { # `pressurepaths` declares `fieldsMatch: partial`, so extra columns are kept # and described in the schema. @@ -52,3 +70,19 @@ test_that("add_gldp_resource marks resources as tabular", { expect_equal(pkg$resources[[1]]$type, "table") }) + +test_that("add_gldp_resource sets the data resource profile", { + # GeoLocator-DP requires `$schema` from v1.1, and frictionless only sets it + # from version 2.0. Implementations read it to tell which version of the + # Data Package standard a resource follows. + data <- tibble::tibble(tag_id = "a", ring_number = "r1") + pkg <- suppressWarnings(add_gldp_resource(create_gldp(), "tags", data)) + resource <- pkg$resources[[1]] + + expect_equal( + resource[["$schema"]], + "https://datapackage.org/profiles/2.0/dataresource.json" + ) + # `$schema` first, as frictionless 2.0 writes it. + expect_equal(names(resource)[1], "$schema") +}) diff --git a/tests/testthat/test-gldp_read_resource.R b/tests/testthat/test-gldp_read_resource.R new file mode 100644 index 0000000..bafa5e4 --- /dev/null +++ b/tests/testthat/test-gldp_read_resource.R @@ -0,0 +1,82 @@ +library(testthat) +library(GeoLocatoR) + +# Write a one-resource package to disk, then replace its CSV, so the file no +# longer has the columns its schema declares. +local_package_with_csv <- function(resource_name, data, csv_lines, env = parent.frame()) { + directory <- withr::local_tempdir(.local_envir = env) + pkg <- suppressWarnings(add_gldp_resource(create_gldp(), resource_name, data)) + suppressMessages(frictionless::write_package(pkg, directory)) + writeLines(csv_lines, file.path(directory, paste0(resource_name, ".csv"))) + suppressWarnings(frictionless::read_package(file.path(directory, "datapackage.json"))) +} + +test_that("a resource that leaves out a column keeps its remaining names", { + # frictionless maps schema fields to columns by position, so without matching + # by name `ring_number` is read as `datapackage_id`, the second declared + # field (frictionlessdata/frictionless-r#216). + pkg <- local_package_with_csv( + "tags", + tibble::tibble(tag_id = "a", ring_number = "r1"), + c("tag_id,ring_number", "a,r1") + ) + + data <- gldp_read_resource(pkg, "tags") + + expect_identical(names(data), c("tag_id", "ring_number")) + expect_identical(data$ring_number, "r1") +}) + +test_that("a resource read out of order is matched by name", { + pkg <- local_package_with_csv( + "tags", + tibble::tibble(tag_id = "a", ring_number = "r1"), + c("ring_number,tag_id", "r1,a") + ) + + data <- gldp_read_resource(pkg, "tags") + + expect_identical(data$tag_id, "a") + expect_identical(data$ring_number, "r1") +}) + +test_that("undeclared columns are kept when fieldsMatch allows them", { + # `tags` declares `fieldsMatch: partial`. + pkg <- local_package_with_csv( + "tags", + tibble::tibble(tag_id = "a", ring_number = "r1"), + c("tag_id,ring_number,movebank_tag_id", "a,r1,12345") + ) + + data <- gldp_read_resource(pkg, "tags") + + expect_true("movebank_tag_id" %in% names(data)) + expect_identical(data$tag_id, "a") +}) + +test_that("undeclared columns are reported when fieldsMatch forbids them", { + # `staps` declares `fieldsMatch: superset`. + pkg <- local_package_with_csv( + "staps", + tibble::tibble(tag_id = "a", stap_id = 1), + c("tag_id,stap_id,my_extra", "a,1,9") + ) + + expect_error( + gldp_read_resource(pkg, "staps"), + class = "gldp_error_undeclared_columns" + ) +}) + +test_that("a file sharing no column with its schema is reported", { + pkg <- local_package_with_csv( + "tags", + tibble::tibble(tag_id = "a", ring_number = "r1"), + c("alpha,beta", "1,2") + ) + + expect_error( + gldp_read_resource(pkg, "tags"), + class = "gldp_error_header_unrelated" + ) +}) diff --git a/tests/testthat/test-upgrade_gldp.R b/tests/testthat/test-upgrade_gldp.R index 046acfa..b2bd5c7 100644 --- a/tests/testthat/test-upgrade_gldp.R +++ b/tests/testthat/test-upgrade_gldp.R @@ -311,3 +311,33 @@ test_that("upgrade_gldp rejects a resource holding both path and data", { class = "gldp_error_resource_both_path_data" ) }) + +test_that("upgrade_gldp migrates a v1.0 package to v1.1", { + pkg <- list( + "$schema" = "https://raw.githubusercontent.com/GeoPressure/GeoLocator-DP/v1.0/geolocator-dp-profile.json", + resources = list( + list( + name = "tags", + data = data.frame( + tag_id = "tag-1", + ring_number = "A-123", + scientific_name = "Hirundo rustica", + stringsAsFactors = FALSE + ) + ) + ) + ) + class(pkg) <- c("geolocatordp", "datapackage", "list") + attr(pkg, "directory") <- "." + + upgraded <- suppressMessages(suppressWarnings(upgrade_gldp(pkg, to_version = "v1.1"))) + + expect_equal(gldp_version(upgraded), "v1.1") + expect_equal(upgraded$`$schema`, gldp_schema_url("v1.1")) + + # v1.1 requires `type` and the table schema in `schema`, not in `$schema`. + tags_res <- purrr::detect(upgraded$resources, \(r) r$name == "tags") + expect_equal(tags_res$type, "table") + expect_type(tags_res$schema, "list") + expect_false(is.null(tags_res$schema$fields)) +}) diff --git a/tests/testthat/test-validate_gldp_helpers.R b/tests/testthat/test-validate_gldp_helpers.R index dfacccf..dbeff69 100644 --- a/tests/testthat/test-validate_gldp_helpers.R +++ b/tests/testthat/test-validate_gldp_helpers.R @@ -163,3 +163,84 @@ test_that("check_type and check_format handle NA values correctly", { expect_false(check_format(c("2023-12-25", NA, "invalid"), "date", "test_field")) }) }) + +test_that("gldp_profile_resource_names reads both profile shapes", { + # GeoLocator-DP up to v1.0 lists the resource alternatives under `oneOf`, from + # v1.1 under `allOf` as `if`/`then` pairs. Both must yield the same names, or + # upgrading a package written against an older version drops resources. + tables <- c( + "tags", + "observations", + "measurements", + "staps", + "twilights", + "paths", + "edges", + "pressurepaths" + ) + + for (version in c("v1.0", "v1.1")) { + profile <- gldp_profile_schema(version) + expect_identical( + gldp_profile_resource_names(profile), + c(tables, "params"), + info = version + ) + } +}) + +test_that("schema validation reads allowed values from categories", { + # GeoLocator-DP v1.1 declares `categories` alongside the `enum` constraint. + # A schema that dropped `enum` must still constrain its values. + schema <- list( + name = "observations", + fields = list( + list( + name = "sex", + type = "string", + categories = list( + list(value = "U", label = "Unknown"), + list(value = "M", label = "Male"), + list(value = "F", label = "Female") + ), + constraints = list(required = TRUE) + ) + ) + ) + + expect_true(suppressMessages(validate_gldp_table( + tibble::tibble(sex = c("M", "F")), + schema + ))) + expect_false(suppressMessages(validate_gldp_table( + tibble::tibble(sex = c("M", "female")), + schema + ))) +}) + +test_that("schema validation flags a missing required column", { + # `fieldsMatch` says nothing about which columns are required, so every mode + # GeoLocator-DP uses would otherwise accept a table without its primary key + # (frictionlessdata/datapackage#1126). + schema <- list( + name = "tags", + fieldsMatch = "partial", + fields = list( + list(name = "tag_id", type = "string", constraints = list(required = TRUE)), + list(name = "ring_number", type = "string", constraints = list(required = TRUE)), + list(name = "tag_comments", type = "string") + ) + ) + + # An optional column may be left out. + expect_true(suppressMessages(validate_gldp_table( + tibble::tibble(tag_id = "a", ring_number = "r1"), + schema + ))) + + # A required one may not. + expect_false(suppressMessages(validate_gldp_table( + tibble::tibble(ring_number = "r1"), + schema + ))) +})