From 4efd65dd523da821d20d6719315b80fad02f3e5f Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:52:35 -0700 Subject: [PATCH 1/8] Remove numbering from comments This makes it easier to add/remove steps. --- augur/subsample.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/augur/subsample.py b/augur/subsample.py index 8237ce9de..8a83e6c59 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -150,10 +150,10 @@ def run(args: argparse.Namespace) -> None: support is adopted: """ - # 1. Parse and validate config. + # Parse and validate config. config = _parse_config(args.config, args.config_section) - # 2. Construct argument lists for augur filter. + # Construct argument lists for augur filter. defaults = config.get("defaults") samples: List[Sample] = [] @@ -172,7 +172,7 @@ def run(args: argparse.Namespace) -> None: if (value := getattr(args, cli_option)) is not None: _add_to_args(final_filter_args, filter_option, value) - # 3. Run augur filter. + # Run augur filter. if len(samples) == 1: # A single sample is translated to a single augur filter call. From e867ba13518b77d089d1f31de5874b9aa8ebf1c6 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:09:26 -0700 Subject: [PATCH 2/8] Mark filepath values in schema This will be used in a future commit. --- augur/data/schema-subsample-config.json | 25 +++++++++++++++++-------- devel/regenerate-subsample-schema | 9 +++++---- docs/_ext/augur_subsample_helpers.py | 4 ++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/augur/data/schema-subsample-config.json b/augur/data/schema-subsample-config.json index 5be5627d9..4fec1b7b2 100644 --- a/augur/data/schema-subsample-config.json +++ b/augur/data/schema-subsample-config.json @@ -16,12 +16,14 @@ "exclude": { "oneOf": [ { - "type": "string" + "type": "string", + "format": "filepath" }, { "type": "array", "items": { - "type": "string" + "type": "string", + "format": "filepath" } } ], @@ -58,12 +60,14 @@ "include": { "oneOf": [ { - "type": "string" + "type": "string", + "format": "filepath" }, { "type": "array", "items": { - "type": "string" + "type": "string", + "format": "filepath" } } ], @@ -136,12 +140,14 @@ "exclude": { "oneOf": [ { - "type": "string" + "type": "string", + "format": "filepath" }, { "type": "array", "items": { - "type": "string" + "type": "string", + "format": "filepath" } } ], @@ -178,12 +184,14 @@ "include": { "oneOf": [ { - "type": "string" + "type": "string", + "format": "filepath" }, { "type": "array", "items": { - "type": "string" + "type": "string", + "format": "filepath" } } ], @@ -263,6 +271,7 @@ }, "group_by_weights": { "type": "string", + "format": "filepath", "description": "TSV file defining weights for grouping. Path must be relative to the\nworking directory. Requirements:\n\n(1) Lines starting with '#' are treated as comment lines.\n(2) The first non-comment line must be a header row.\n(3) There must be a numeric ``weight`` column (weights can take on any\n non-negative values).\n(4) Other columns must be a subset of grouping columns, with\n combinations of values covering all combinations present in the\n metadata.\n(5) This option only applies when grouping columns and a total sample\n size are provided.\n(6) This option can only be used when probabilistic sampling is allowed.\n\nNotes:\n\n(1) Any grouping columns absent from this file will be given equal\n weighting across all values *within* groups defined by the other\n weighted columns.\n(2) An entry with the value ``default`` under all columns will be\n treated as the default weight for specific groups present in the\n metadata but missing from the weights file. If there is no default\n weight and the metadata contains rows that are not covered by the\n given weights, augur filter will exit with an error." }, "probabilistic_sampling": { diff --git a/devel/regenerate-subsample-schema b/devel/regenerate-subsample-schema index 1950888e3..1576a398c 100755 --- a/devel/regenerate-subsample-schema +++ b/devel/regenerate-subsample-schema @@ -47,10 +47,10 @@ def create_schema(): default_options = { "exclude": { "oneOf": [ - {"type": "string"}, + {"type": "string", "format": "filepath"}, { "type": "array", - "items": {"type": "string"} + "items": {"type": "string", "format": "filepath"} } ], "description": descriptions["exclude"] @@ -76,10 +76,10 @@ def create_schema(): }, "include": { "oneOf": [ - {"type": "string"}, + {"type": "string", "format": "filepath"}, { "type": "array", - "items": {"type": "string"} + "items": {"type": "string", "format": "filepath"} } ], "description": descriptions["include"] @@ -148,6 +148,7 @@ def create_schema(): }, "group_by_weights": { "type": "string", + "format": "filepath", "description": descriptions["group_by_weights"] }, "probabilistic_sampling": { diff --git a/docs/_ext/augur_subsample_helpers.py b/docs/_ext/augur_subsample_helpers.py index 2df0410c8..d70e3d325 100644 --- a/docs/_ext/augur_subsample_helpers.py +++ b/docs/_ext/augur_subsample_helpers.py @@ -260,8 +260,8 @@ def _format_type_info(self, prop_def): # Special case: string or array of strings one_of_options = prop_def['oneOf'] if (len(one_of_options) == 2 and - one_of_options[0] == {"type": "string"} and - one_of_options[1] == {"type": "array", "items": {"type": "string"}}): + one_of_options[0]["type"] == "string" and + (one_of_options[1]["type"] == "array" and one_of_options[1]["items"]["type"] == "string")): return nodes.paragraph('', 'string(s)') # Multiple types via list From 4d42c4be486a60e6b158042328f3ac9dea94e8e3 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:12:36 -0700 Subject: [PATCH 3/8] Load schema outside of config parsing Preparing to use the schema in another function. --- augur/subsample.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/augur/subsample.py b/augur/subsample.py index 8a83e6c59..110018cfe 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -150,8 +150,9 @@ def run(args: argparse.Namespace) -> None: support is adopted: """ - # Parse and validate config. - config = _parse_config(args.config, args.config_section) + # Load schema, parse and validate config. + schema_validator = load_json_schema("schema-subsample-config.json") + config = _parse_config(args.config, args.config_section, schema_validator) # Construct argument lists for augur filter. @@ -217,7 +218,7 @@ def run(args: argparse.Namespace) -> None: sample.remove_output_strains() -def _parse_config(filename: str, config_section: Optional[List[str]] = None) -> Dict[str, Any]: +def _parse_config(filename: str, config_section: Optional[List[str]], schema) -> Dict[str, Any]: # Create a custom YAML loader to treat timestamps as strings. class CustomLoader(yaml.SafeLoader): pass @@ -245,7 +246,6 @@ def string_constructor(loader, node): # Validate against schema. try: - schema = load_json_schema("schema-subsample-config.json") validate_json(config, schema, filename) except ValidateError as e: raise AugurError(e) From 8e2cc7a83ba33257e3143fc896cc7e879019d16d Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Fri, 24 Oct 2025 14:07:22 -0700 Subject: [PATCH 4/8] Add CLI option for custom search paths Previously, the only search path was implicitly the current working directory. Make this explicit and add an option to search other paths before it. This means the notes in option descriptions mentioning cwd as search path are no longer accurate for augur subsample. Since the descriptions are shared with augur filter which does not allow custom search paths, I figured it'd be best to remove the notes entirely. --- augur/data/schema-subsample-config.json | 10 +- augur/filter/arguments.py | 9 +- augur/subsample.py | 168 +++++++++++++++++- .../functional/subsample/cram/advanced-yaml.t | 4 +- .../functional/subsample/cram/include-file.t | 90 +++++++++- .../subsample/cram/include-value-types.t | 4 +- 6 files changed, 266 insertions(+), 19 deletions(-) diff --git a/augur/data/schema-subsample-config.json b/augur/data/schema-subsample-config.json index 4fec1b7b2..cd417e62d 100644 --- a/augur/data/schema-subsample-config.json +++ b/augur/data/schema-subsample-config.json @@ -27,7 +27,7 @@ } } ], - "description": "File(s) with list of strains to exclude. Paths must be relative to the\nworking directory." + "description": "File(s) with list of strains to exclude." }, "exclude_all": { "type": "boolean", @@ -71,7 +71,7 @@ } } ], - "description": "File(s) with list of strains to include regardless of priorities,\nsubsampling, or absence of an entry in sequences. Paths must be relative\nto the working directory." + "description": "File(s) with list of strains to include regardless of priorities,\nsubsampling, or absence of an entry in sequences." }, "include_where": { "oneOf": [ @@ -151,7 +151,7 @@ } } ], - "description": "File(s) with list of strains to exclude. Paths must be relative to the\nworking directory." + "description": "File(s) with list of strains to exclude." }, "exclude_all": { "type": "boolean", @@ -195,7 +195,7 @@ } } ], - "description": "File(s) with list of strains to include regardless of priorities,\nsubsampling, or absence of an entry in sequences. Paths must be relative\nto the working directory." + "description": "File(s) with list of strains to include regardless of priorities,\nsubsampling, or absence of an entry in sequences." }, "include_where": { "oneOf": [ @@ -272,7 +272,7 @@ "group_by_weights": { "type": "string", "format": "filepath", - "description": "TSV file defining weights for grouping. Path must be relative to the\nworking directory. Requirements:\n\n(1) Lines starting with '#' are treated as comment lines.\n(2) The first non-comment line must be a header row.\n(3) There must be a numeric ``weight`` column (weights can take on any\n non-negative values).\n(4) Other columns must be a subset of grouping columns, with\n combinations of values covering all combinations present in the\n metadata.\n(5) This option only applies when grouping columns and a total sample\n size are provided.\n(6) This option can only be used when probabilistic sampling is allowed.\n\nNotes:\n\n(1) Any grouping columns absent from this file will be given equal\n weighting across all values *within* groups defined by the other\n weighted columns.\n(2) An entry with the value ``default`` under all columns will be\n treated as the default weight for specific groups present in the\n metadata but missing from the weights file. If there is no default\n weight and the metadata contains rows that are not covered by the\n given weights, augur filter will exit with an error." + "description": "TSV file defining weights for grouping. Requirements:\n\n(1) Lines starting with '#' are treated as comment lines.\n(2) The first non-comment line must be a header row.\n(3) There must be a numeric ``weight`` column (weights can take on any\n non-negative values).\n(4) Other columns must be a subset of grouping columns, with\n combinations of values covering all combinations present in the\n metadata.\n(5) This option only applies when grouping columns and a total sample\n size are provided.\n(6) This option can only be used when probabilistic sampling is allowed.\n\nNotes:\n\n(1) Any grouping columns absent from this file will be given equal\n weighting across all values *within* groups defined by the other\n weighted columns.\n(2) An entry with the value ``default`` under all columns will be\n treated as the default weight for specific groups present in the\n metadata but missing from the weights file. If there is no default\n weight and the metadata contains rows that are not covered by the\n given weights, augur filter will exit with an error." }, "probabilistic_sampling": { "type": "boolean", diff --git a/augur/filter/arguments.py b/augur/filter/arguments.py index 416db6683..e9df997dc 100644 --- a/augur/filter/arguments.py +++ b/augur/filter/arguments.py @@ -43,8 +43,7 @@ "2010-XX-01")."""), "exclude": dedent("""\ - File(s) with list of strains to exclude. Paths must be relative to the - working directory."""), + File(s) with list of strains to exclude."""), "exclude_where": dedent("""\ Exclude sequences matching these conditions. Ex: "host=rat" or @@ -57,8 +56,7 @@ "include": dedent("""\ File(s) with list of strains to include regardless of priorities, - subsampling, or absence of an entry in sequences. Paths must be relative - to the working directory."""), + subsampling, or absence of an entry in sequences."""), "include_where": dedent("""\ Include sequences with these values. ex: host=rat. Multiple values are @@ -104,8 +102,7 @@ when a total sample size is provided."""), "group_by_weights": dedent("""\ - TSV file defining weights for grouping. Path must be relative to the - working directory. Requirements: + TSV file defining weights for grouping. Requirements: (1) Lines starting with '#' are treated as comment lines. (2) The first non-comment line must be a header row. diff --git a/augur/subsample.py b/augur/subsample.py index 110018cfe..b942be84d 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -12,12 +12,14 @@ import tempfile import yaml from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from textwrap import dedent from typing import Any, Dict, List, Optional, Tuple, Union from augur import filter as augur_filter from augur.argparse_ import ExtendOverwriteDefault, SKIP_AUTO_DEFAULT_IN_HELP from augur.errors import AugurError from augur.io.metadata import DEFAULT_DELIMITERS, DEFAULT_ID_COLUMNS -from augur.io.print import print_err +from augur.io.print import print_err, indented_list from augur.utils import augur from augur.validate import load_json_schema, validate_json, ValidateError @@ -111,6 +113,13 @@ def register_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.A config_group = parser.add_argument_group("Configuration options", "options related to configuration") config_group.add_argument("--config", metavar="FILE", required=True, help="augur subsample config file. The expected config options must be defined at the top level, or within a specific section using --config-section." + SKIP_AUTO_DEFAULT_IN_HELP) config_group.add_argument("--config-section", metavar="KEY", nargs="+", action=ExtendOverwriteDefault, help="Use a section of the file given to --config by listing the keys leading to the section. Provide one or more keys. (default: use the entire file)" + SKIP_AUTO_DEFAULT_IN_HELP) + config_group.add_argument("--search-paths", "--search-path", metavar="DIR", nargs="+", action=ExtendOverwriteDefault, + help=dedent(f"""\ + One or more directories to search for relative filepaths specified + in the config file. If a file exists in multiple directories, only + the file from the first directory will be used. Specified + directories will be considered before the default (current working + directory)""" + SKIP_AUTO_DEFAULT_IN_HELP)) config_group.add_argument('--nthreads', metavar="N", type=int, default=1, help="Number of CPUs/cores/threads/jobs to utilize at once. For augur subsample, this means the number of samples to run simultaneously. Individual samples are limited to a single thread. The final augur filter call can take advantage of multiple threads.") config_group.add_argument('--seed', metavar="N", type=int, help="random number generator seed for reproducible outputs (with same input data)." + SKIP_AUTO_DEFAULT_IN_HELP) @@ -154,6 +163,10 @@ def run(args: argparse.Namespace) -> None: schema_validator = load_json_schema("schema-subsample-config.json") config = _parse_config(args.config, args.config_section, schema_validator) + # Resolve filepaths. + search_paths = _get_search_paths(args.search_paths) + config = _resolve_filepaths(config, search_paths, schema_validator.schema) + # Construct argument lists for augur filter. defaults = config.get("defaults") @@ -252,6 +265,159 @@ def string_constructor(loader, node): return config +def _get_search_paths(from_cli: List[str]) -> List[Path]: + """ + Returns the paths to search for relative filepaths in config. + """ + default = [ + Path.cwd(), + ] + + if from_cli: + return [ + *(Path(p) for p in from_cli), + *default, + ] + + return default + + +def _resolve_filepaths( + config: Dict[str, Any], + search_paths: List[Path], + schema: Dict[str, Any], + root_schema: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Resolve filepaths in config. + + Recursively walks the config alongside the schema to determine which fields + contain filepaths and resolves them. + """ + if root_schema is None: + root_schema = schema + + # Get properties schema for current section + properties = schema.get("properties", {}) + pattern_properties = schema.get("patternProperties", {}) + + for key, value in config.items(): + prop_schema = properties.get(key) + + if not prop_schema and pattern_properties: + # Use first pattern property schema (for dynamic keys like samples) + prop_schema = next(iter(pattern_properties.values())) + + # Get referenced property schema + if ref := prop_schema.get("$ref"): + prop_schema = _get_referenced_schema(ref, root_schema) + + # Resolve filepath + if _is_filepath(prop_schema): + if isinstance(value, list): + config[key] = [str(_resolve_filepath(Path(v), search_paths)) for v in value] + elif isinstance(value, str): + config[key] = str(_resolve_filepath(Path(value), search_paths)) + + # Recurse into config section + elif isinstance(value, dict): + config[key] = _resolve_filepaths(value, search_paths, prop_schema, root_schema) + + return config + + +def _get_referenced_schema( + ref: str, + root_schema: Dict[str, Any], +) -> Dict[str, Any]: + """ + Resolve a JSON schema reference. Example: '#/$defs/sampleProperties' + """ + keys = ref.lstrip("#/").split("/") + schema = root_schema + for key in keys: + schema = schema[key] + return schema + + +def _is_filepath(prop_schema: Dict[str, Any]) -> bool: + """ + Check if the property schema declares it is a filepath. + """ + # Direct 'format: filepath' + if prop_schema.get("format") == "filepath": + return True + + # Check oneOf variants for 'format: filepath' + if "oneOf" in prop_schema: + for variant in prop_schema["oneOf"]: + if variant.get("format") == "filepath": + return True + + return False + + +def _resolve_filepath( + path: Path, + search_paths: List[Path], +) -> Path: + """ + Resolve a filepath by searching through multiple directories. + + If the path is already absolute, verify it exists and return it. + + >>> import tempfile + >>> from pathlib import Path + >>> tmpdir1 = Path(tempfile.mkdtemp()).resolve() + >>> tmpdir2 = Path(tempfile.mkdtemp()).resolve() + >>> absolute_path = tmpdir1 / "file.txt" + >>> with open(absolute_path, "w") as f: _ = f.write("test") + >>> _resolve_filepath(absolute_path, []) == absolute_path + True + + Otherwise, try resolving it relative to each directory in search_paths, in order. + Return the first path that exists. + + >>> with open(tmpdir2 / "file.txt", "w") as f: _ = f.write("test") + >>> result = _resolve_filepath(Path("file.txt"), [tmpdir1, tmpdir2]) + >>> result == tmpdir1 / "file.txt" + True + + If an absolute path doesn't exist, raise an error. + + >>> _resolve_filepath(Path("/nonexistent/file.txt"), [tmpdir1, tmpdir2]) + Traceback (most recent call last): + ... + augur.errors.AugurError: File '/nonexistent/file.txt' does not exist. + + If the relative path doesn't exist anywhere, raise an error. + + >>> _resolve_filepath(Path("nonexistent.txt"), [tmpdir1, tmpdir2]) + Traceback (most recent call last): + ... + augur.errors.AugurError: File 'nonexistent.txt' not resolvable from any of the following paths: + + ... + """ + # Absolute path + if path.is_absolute(): + if not path.exists(): + raise AugurError(f"File {str(path)!r} does not exist.") + return path + + # Relative path + for search_path in search_paths: + resolved_path = (search_path / path).resolve() + if resolved_path.exists(): + return resolved_path + + # File not found + raise AugurError(dedent(f"""\ + File {str(path)!r} not resolvable from any of the following paths: + + {indented_list([str(p) for p in search_paths], ' ' + ' ')}""")) + + def _merge_options(sample_options: Dict[str, Any], defaults: Optional[Dict[str, Any]]) -> Dict[str, Any]: """ Merge sample options with default options, with sample options taking precedence. diff --git a/tests/functional/subsample/cram/advanced-yaml.t b/tests/functional/subsample/cram/advanced-yaml.t index 279dc88d9..cdd409db2 100644 --- a/tests/functional/subsample/cram/advanced-yaml.t +++ b/tests/functional/subsample/cram/advanced-yaml.t @@ -38,12 +38,12 @@ Keeping it around as a valid test for advanced YAML syntax. Validating schema of 'config.yaml'... [south_america] 9 strains were dropped during filtering [south_america] 6 were filtered out by the query: "region == 'South America'" - [south_america] 2 were added back because they were in include.txt + \[south_america\] \\t2 were added back because they were in .*/include.txt.* (re) [south_america] 5 were dropped because of subsampling criteria [south_america] 3 strains passed all filters [oceania] 10 strains were dropped during filtering [oceania] 11 were filtered out by the query: "region == 'Oceania'" - [oceania] 2 were added back because they were in include.txt + \[oceania\] \\t2 were added back because they were in .*/include.txt.* (re) [oceania] 0 were dropped because of subsampling criteria [oceania] 2 strains passed all filters 9 strains were dropped during filtering diff --git a/tests/functional/subsample/cram/include-file.t b/tests/functional/subsample/cram/include-file.t index 7744f4b34..03dcd1f62 100644 --- a/tests/functional/subsample/cram/include-file.t +++ b/tests/functional/subsample/cram/include-file.t @@ -2,8 +2,7 @@ Setup $ source "$TESTDIR"/_setup.sh -File paths in the config must be relative to the working directory, not the -location of the config file. +File at path relative to current working directory is found $ mkdir -p defaults/ $ cat >defaults/include.txt <<~~ @@ -26,6 +25,91 @@ location of the config file. > --seed 0 Validating schema of 'config/config.yaml'... 9 strains were dropped during filtering - 1 was added back because it was in defaults/include.txt + \\t1 was added back because it was in .*/include.txt.* (re) 10 were dropped because of subsampling criteria 3 strains passed all filters + +File in --search-path is found + + $ mkdir -p custom_dir/ + $ cat >custom_dir/include_custom.txt <<~~ + > EcEs062_16 + > ~~ + + $ cat >config/config.yaml <<~~ + > samples: + > test: + > max_sequences: 2 + > include: + > - include_custom.txt + > ~~ + + $ ${AUGUR} subsample \ + > --metadata "$TESTDIR"/../../filter/data/metadata.tsv \ + > --config config/config.yaml \ + > --search-path custom_dir \ + > --output-metadata output_metadata.tsv \ + > --seed 0 + Validating schema of 'config/config.yaml'... + 9 strains were dropped during filtering + \\t1 was added back because it was in .*/custom_dir/include_custom.txt.* (re) + 10 were dropped because of subsampling criteria + 3 strains passed all filters + +Custom search path is searched before default paths (the file exists in both cwd and search_path_dir) + + $ mkdir -p search_path_dir/ + + $ cat >config/include.txt <<~~ + > EcEs062_16 + > ~~ + + $ cat >search_path_dir/include.txt <<~~ + > SG_018 + > ~~ + + $ cat >config/config.yaml <<~~ + > samples: + > test: + > max_sequences: 2 + > include: + > - include.txt + > ~~ + + $ ${AUGUR} subsample \ + > --metadata "$TESTDIR"/../../filter/data/metadata.tsv \ + > --config config/config.yaml \ + > --search-path search_path_dir \ + > --output-metadata output_metadata.tsv \ + > --seed 0 + Validating schema of 'config/config.yaml'... + 9 strains were dropped during filtering + \\t1 was added back because it was in .*/include.txt.* (re) + 10 were dropped because of subsampling criteria + 3 strains passed all filters + +Verify the file from search_path_dir was used (contains SG_018, not EcEs062_16 from cwd) + + $ grep "SG_018" output_metadata.tsv | cut -f1 + SG_018 + +Error when file not found in any search path + + $ mkdir -p config/ + $ cat >config/config.yaml <<~~ + > samples: + > test: + > max_sequences: 2 + > include: + > - nonexistent.txt + > ~~ + + $ ${AUGUR} subsample \ + > --metadata "$TESTDIR"/../../filter/data/metadata.tsv \ + > --config config/config.yaml \ + > --output-metadata output_metadata.tsv + Validating schema of 'config/config.yaml'... + ERROR: File 'nonexistent.txt' not resolvable from any of the following paths: + + .*/include-file.t (re) + [2] diff --git a/tests/functional/subsample/cram/include-value-types.t b/tests/functional/subsample/cram/include-value-types.t index 6148fe9de..4cb800e1b 100644 --- a/tests/functional/subsample/cram/include-value-types.t +++ b/tests/functional/subsample/cram/include-value-types.t @@ -25,7 +25,7 @@ Create an include file. > --seed 0 Validating schema of 'config.yaml'... 9 strains were dropped during filtering - 1 was added back because it was in include.txt + \\t1 was added back because it was in .*/include.txt.* (re) 10 were dropped because of subsampling criteria 3 strains passed all filters @@ -45,6 +45,6 @@ Create an include file. > --seed 0 Validating schema of 'config.yaml'... 9 strains were dropped during filtering - 1 was added back because it was in include.txt + \\t1 was added back because it was in .*/include.txt.* (re) 10 were dropped because of subsampling criteria 3 strains passed all filters From 17a8a18f873ff1f9621167947018d812e40b650f Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Tue, 21 Oct 2025 16:19:48 -0700 Subject: [PATCH 5/8] Add config's parent directory as first default search path This allows more intuitive relative filepaths from the perspective of the config file. --- augur/subsample.py | 13 ++++++---- .../functional/subsample/cram/include-file.t | 24 ++++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/augur/subsample.py b/augur/subsample.py index b942be84d..77c6ddfe9 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -118,8 +118,9 @@ def register_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.A One or more directories to search for relative filepaths specified in the config file. If a file exists in multiple directories, only the file from the first directory will be used. Specified - directories will be considered before the default (current working - directory)""" + SKIP_AUTO_DEFAULT_IN_HELP)) + directories will be considered before the defaults, which are: + (1) directory containing the config file + (2) current working directory""" + SKIP_AUTO_DEFAULT_IN_HELP)) config_group.add_argument('--nthreads', metavar="N", type=int, default=1, help="Number of CPUs/cores/threads/jobs to utilize at once. For augur subsample, this means the number of samples to run simultaneously. Individual samples are limited to a single thread. The final augur filter call can take advantage of multiple threads.") config_group.add_argument('--seed', metavar="N", type=int, help="random number generator seed for reproducible outputs (with same input data)." + SKIP_AUTO_DEFAULT_IN_HELP) @@ -164,7 +165,7 @@ def run(args: argparse.Namespace) -> None: config = _parse_config(args.config, args.config_section, schema_validator) # Resolve filepaths. - search_paths = _get_search_paths(args.search_paths) + search_paths = _get_search_paths(args.config, args.search_paths) config = _resolve_filepaths(config, search_paths, schema_validator.schema) # Construct argument lists for augur filter. @@ -265,11 +266,15 @@ def string_constructor(loader, node): return config -def _get_search_paths(from_cli: List[str]) -> List[Path]: +def _get_search_paths( + config_file: str, + from_cli: List[str], +) -> List[Path]: """ Returns the paths to search for relative filepaths in config. """ default = [ + Path(config_file).parent, Path.cwd(), ] diff --git a/tests/functional/subsample/cram/include-file.t b/tests/functional/subsample/cram/include-file.t index 03dcd1f62..5bbbcb0f4 100644 --- a/tests/functional/subsample/cram/include-file.t +++ b/tests/functional/subsample/cram/include-file.t @@ -2,7 +2,7 @@ Setup $ source "$TESTDIR"/_setup.sh -File at path relative to current working directory is found +File at path relative to config file location is found $ mkdir -p defaults/ $ cat >defaults/include.txt <<~~ @@ -10,6 +10,27 @@ File at path relative to current working directory is found > ~~ $ mkdir -p config/ + $ cat >config/config.yaml <<~~ + > samples: + > test: + > max_sequences: 2 + > include: + > - ../defaults/include.txt + > ~~ + + $ ${AUGUR} subsample \ + > --metadata "$TESTDIR"/../../filter/data/metadata.tsv \ + > --config config/config.yaml \ + > --output-metadata output_metadata.tsv \ + > --seed 0 + Validating schema of 'config/config.yaml'... + 9 strains were dropped during filtering + \\t1 was added back because it was in .*/defaults/include.txt.* (re) + 10 were dropped because of subsampling criteria + 3 strains passed all filters + +File at path relative to current working directory is found + $ cat >config/config.yaml <<~~ > samples: > test: @@ -111,5 +132,6 @@ Error when file not found in any search path Validating schema of 'config/config.yaml'... ERROR: File 'nonexistent.txt' not resolvable from any of the following paths: + config .*/include-file.t (re) [2] From 3e39a8c53887ca874c1f20b25ee9e692470a9952 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Tue, 21 Oct 2025 16:49:12 -0700 Subject: [PATCH 6/8] Add env var for custom search paths Environment variables can be easier to use than CLI options in some situations. --- augur/subsample.py | 18 ++++++++- docs/usage/envvars.rst | 8 ++++ .../functional/subsample/cram/include-file.t | 40 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/augur/subsample.py b/augur/subsample.py index 77c6ddfe9..35230afe9 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -117,7 +117,8 @@ def register_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.A help=dedent(f"""\ One or more directories to search for relative filepaths specified in the config file. If a file exists in multiple directories, only - the file from the first directory will be used. Specified + the file from the first directory will be used. This can also be set + via the environment variable 'AUGUR_SEARCH_PATHS'. Specified directories will be considered before the defaults, which are: (1) directory containing the config file (2) current working directory""" + SKIP_AUTO_DEFAULT_IN_HELP)) @@ -278,12 +279,25 @@ def _get_search_paths( Path.cwd(), ] + from_env = os.environ.get('AUGUR_SEARCH_PATHS') + if from_cli: + if from_env: + print_err(dedent(f"""\ + WARNING: Both the command line argument --search-paths + and the environment variable AUGUR_SEARCH_PATHS are set. + Only the command line argument will be used.""")) return [ *(Path(p) for p in from_cli), *default, ] - + + if from_env: + return [ + *(Path(p) for p in from_env.split(':')), + *default, + ] + return default diff --git a/docs/usage/envvars.rst b/docs/usage/envvars.rst index 7cb135830..50f42d0a1 100644 --- a/docs/usage/envvars.rst +++ b/docs/usage/envvars.rst @@ -5,6 +5,14 @@ Environment variables Augur's behaviour can be globally modified by the values of some specific environment variables. These can be especially useful in the context of an entire pipeline or workflow which uses Augur, as the environment variables can be set once for all Augur commands at the start of the pipeline. +.. envvar:: AUGUR_SEARCH_PATHS + + Colon-separated directory paths. + If set, ``augur subsample`` will search these directories for relative filepaths specified in the config file. + Multiple directories can be set by separating them with colons (e.g., ``/path/one:/path/two``). + + This takes precedence over the ``--search-paths`` command line argument. + .. envvar:: AUGUR_DEBUG Boolean. diff --git a/tests/functional/subsample/cram/include-file.t b/tests/functional/subsample/cram/include-file.t index 5bbbcb0f4..cbd677603 100644 --- a/tests/functional/subsample/cram/include-file.t +++ b/tests/functional/subsample/cram/include-file.t @@ -135,3 +135,43 @@ Error when file not found in any search path config .*/include-file.t (re) [2] + +--search-paths overrides AUGUR_SEARCH_PATHS + + $ mkdir -p env_dir/ + $ cat >env_dir/include.txt <<~~ + > EcEs062_16 + > ~~ + + $ mkdir -p cli_dir/ + $ cat >cli_dir/include.txt <<~~ + > SG_018 + > ~~ + + $ cat >config/config.yaml <<~~ + > samples: + > test: + > max_sequences: 2 + > include: + > - include.txt + > ~~ + + $ AUGUR_SEARCH_PATHS=env_dir ${AUGUR} subsample \ + > --metadata "$TESTDIR"/../../filter/data/metadata.tsv \ + > --config config/config.yaml \ + > --search-paths cli_dir \ + > --output-metadata output_metadata.tsv \ + > --seed 0 + Validating schema of 'config/config.yaml'... + WARNING: Both the command line argument --search-paths + and the environment variable AUGUR_SEARCH_PATHS are set. + Only the command line argument will be used. + 9 strains were dropped during filtering + \\t1 was added back because it was in .*/cli_dir/include.txt.* (re) + 10 were dropped because of subsampling criteria + 3 strains passed all filters + +Verify the file from cli_dir was used (contains SG_018, not EcEs062_16 from env_dir) + + $ grep "SG_018" output_metadata.tsv | cut -f1 + SG_018 From 4441afafd53a35589940e4a9a4f75052a9cbee33 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Fri, 24 Oct 2025 07:57:08 -0700 Subject: [PATCH 7/8] Add helper function to get resolved filepaths This can be used by Snakemake workflows to declare input files for rules that call augur subsample. --- augur/subsample.py | 62 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/augur/subsample.py b/augur/subsample.py index 35230afe9..e36ee1431 100644 --- a/augur/subsample.py +++ b/augur/subsample.py @@ -14,7 +14,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from textwrap import dedent -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union from augur import filter as augur_filter from augur.argparse_ import ExtendOverwriteDefault, SKIP_AUTO_DEFAULT_IN_HELP from augur.errors import AugurError @@ -167,7 +167,7 @@ def run(args: argparse.Namespace) -> None: # Resolve filepaths. search_paths = _get_search_paths(args.config, args.search_paths) - config = _resolve_filepaths(config, search_paths, schema_validator.schema) + config, _ = _resolve_filepaths(config, search_paths, schema_validator.schema) # Construct argument lists for augur filter. @@ -233,6 +233,49 @@ def run(args: argparse.Namespace) -> None: sample.remove_output_strains() +def get_referenced_files( + config_file: str, + config_section: Optional[List[str]] = None, + search_paths: Optional[List[str]] = None, +) -> Set[str]: + """Get the files referenced in a subsample config file. + + Extracts and resolves all filepath values referenced in the config, + including defaults and individual sample options. + + Parameters + ---------- + config_file + Path to the subsample config file. + + config_section + Optional list of keys to navigate to a specific section of the config file. + + search_paths + Optional list of directories to search for relative filepaths specified + in the config file. If a file exists in multiple directories, only + the file from the first directory will be used. This can also be set + via the environment variable 'AUGUR_SEARCH_PATHS'. Specified + directories will be considered before the defaults, which are: + (1) directory containing the config file + (2) current working directory + + Returns + ------- + set + Resolved filepaths + """ + # Load schema, parse and validate config. + schema_validator = load_json_schema("schema-subsample-config.json") + config = _parse_config(config_file, config_section, schema_validator) + + # Resolve filepaths. + search_path_objs = _get_search_paths(config_file, search_paths) + config, filepaths = _resolve_filepaths(config, search_path_objs, schema_validator.schema) + + return set(filepaths) + + def _parse_config(filename: str, config_section: Optional[List[str]], schema) -> Dict[str, Any]: # Create a custom YAML loader to treat timestamps as strings. class CustomLoader(yaml.SafeLoader): @@ -306,16 +349,18 @@ def _resolve_filepaths( search_paths: List[Path], schema: Dict[str, Any], root_schema: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: +) -> Tuple[Dict[str, Any], List[str]]: """ Resolve filepaths in config. Recursively walks the config alongside the schema to determine which fields - contain filepaths and resolves them. + contain filepaths, resolves them, and collects the resolved filepaths. """ if root_schema is None: root_schema = schema + filepaths = [] + # Get properties schema for current section properties = schema.get("properties", {}) pattern_properties = schema.get("patternProperties", {}) @@ -335,14 +380,19 @@ def _resolve_filepaths( if _is_filepath(prop_schema): if isinstance(value, list): config[key] = [str(_resolve_filepath(Path(v), search_paths)) for v in value] + filepaths.extend(config[key]) elif isinstance(value, str): config[key] = str(_resolve_filepath(Path(value), search_paths)) + filepaths.append(config[key]) # Recurse into config section elif isinstance(value, dict): - config[key] = _resolve_filepaths(value, search_paths, prop_schema, root_schema) + config[key], downstream_filepaths = _resolve_filepaths( + value, search_paths, prop_schema, root_schema + ) + filepaths.extend(downstream_filepaths) - return config + return config, filepaths def _get_referenced_schema( From 1828ddea93be0665b9d3a6ddb7957dc80839bc00 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Fri, 24 Oct 2025 16:37:01 -0700 Subject: [PATCH 8/8] Update changelog --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index b41647f5b..578a85a66 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,14 +5,18 @@ ### Features * augur.io.read_metadata: Added a new parameter `keep_id_as_column` to keep the resolved id column as a column in addition to setting it as the DataFrame index. [#1917][] (@victorlin) +* subsample: Filepaths in the config file can now be relative to the config file's parent directory in addition to the current working directory. Custom directories can also be specified using a new command line option `--search-paths` or environment variable `AUGUR_SEARCH_PATHS`. [#1897][] (@victorlin) +* A helper function – `augur.subsample.get_referenced_files` – has been added to optimize usage of `augur subsample` in Snakemake workflows. This is experimental and not yet part of the public API. [#1918][] (@victorlin) ### Bug fixes * filter: Previously, `--query`, `--exclude-where`, and `--include-where` did not work for the id column (`strain`, `name`, or other from `--metadata-id-columns`). This has been fixed. [#1915][] (@corneliusroemer, @victorlin) * export v2: Support export of URLs for non-string values. [#1926][] (@joverlee521) +[#1897]: https://github.com/nextstrain/augur/issues/1897 [#1915]: https://github.com/nextstrain/augur/issues/1915 [#1917]: https://github.com/nextstrain/augur/pull/1917 +[#1918]: https://github.com/nextstrain/augur/pull/1918 [#1926]: https://github.com/nextstrain/augur/pull/1926 ## 32.0.0 (21 October 2025)