Support nextstrain run (with YAML-configured augur filter)

ingest/Snakefile

@@ -1,8 +1,12 @@
-# The workflow filepaths are written relative to this Snakefile's base directory
-workdir: workflow.current_basedir
+# Utility functions shared across all workflows.
+include: "../shared/vendored/snakemake/config.smk"
 
-# Use default configuration values. Override with Snakemake's --configfile/--config options.
-configfile: "defaults/config.yaml"
+# Use default configuration values. Extend with Snakemake's --configfile/--config options.
+configfile: os.path.join(workflow.basedir, "defaults/config.yaml")
+
+# Use custom configuration from analysis directory (i.e. working dir), if any.
+if os.path.exists("config.yaml"):
+    configfile: "config.yaml"
 
 # This is the default rule that Snakemake will run when there are no specified targets.
 # The default output of the ingest workflow is usually the curated metadata and sequences.
@@ -31,5 +35,10 @@ include: "rules/nextclade.smk"
 # https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#handling-ambiguous-rules
 if "custom_rules" in config:
     for rule_file in config["custom_rules"]:
-
-        include: rule_file
+        # Relative custom rule paths in the config are relative to the analysis
+        # directory (i.e. the current working directory, or workdir, usually
+        # given by --directory), but the "include" directive treats relative
+        # paths as relative to the workflow (e.g. workflow.current_basedir).
+        # Convert to an absolute path based on the analysis/current directory
+        # to avoid this mismatch of expectations.
+        include: os.path.join(os.getcwd(), rule_file)

ingest/build-configs/nextstrain-automation/upload.smk

@@ -29,7 +29,7 @@ rule upload_to_s3:
         cloudfront_domain=config["cloudfront_domain"],
     shell:
         """
-        ./vendored/upload-to-s3 \
+        {workflow.basedir}/../shared/vendored/scripts/upload-to-s3 \
             {params.quiet} \
             {input.file_to_upload:q} \
             {params.s3_dst:q}/{wildcards.remote_file:q} \

nextstrain-pathogen.yaml

@@ -1,5 +1,10 @@
-# This is currently an empty file to indicate the top level pathogen repo.
-# The inclusion of this file allows the Nextstrain CLI to run the
-# `nextstrain build` from any directory regardless of runtime.
+# This file's *existence* marks the top level of a Nextstrain pathogen repo,
+# which allows `nextstrain build` to be run from any subdirectory of the repo
+# regardless of runtime.  For more details, see
+# <https://github.com/nextstrain/cli/releases/tag/8.2.0>.
 #
-# See https://github.com/nextstrain/cli/releases/tag/8.2.0 for more details.
+# This file's *contents* is the "registration metadata" for the pathogen repo,
+# used by `nextstrain setup` and `nextstrain run`.
+---
+compatibility:
+  nextstrain run: true

phylogenetic/Snakefile

@@ -2,29 +2,15 @@
 This is the main phylogenetic Snakefile that orchestrates the full phylogenetic
 workflow and define its default output(s).
 """
-# The workflow filepaths are written relative to this Snakefile's base directory
-workdir: workflow.current_basedir
+# Utility functions shared across all workflows.
+include: "../shared/vendored/snakemake/config.smk"
 
-# Use default configuration values. Override with Snakemake's --configfile/--config options.
-configfile: "defaults/config.yaml"
+configfile: os.path.join(workflow.basedir, "defaults/config.yaml")
 
-# Validate 'builds'
-from textwrap import dedent
+if os.path.exists("config.yaml"):
+    configfile: "config.yaml"
 
-def indented_list(xs, prefix):
-    return f"\n{prefix}".join(xs)
-
-if invalid_builds := set(config["builds"]) - set(config["build_params"]):
-    print(dedent(f"""\
-        ERROR: The following names in 'builds' are not defined in 'build_params':
-
-            {indented_list(invalid_builds, "            ")}
-
-        Available builds are:
-
-            {indented_list(config['build_params'], "            ")}
-        """))
-    exit(1)
+include: "rules/config.smk"
 
 builds = config['builds']
 
@@ -53,7 +39,6 @@ rule all:
 # custom_rules imported below to ensure that the core workflow is not complicated
 # by build specific rules.
 
-include: "rules/write_config.smk"
 include: "rules/merge_additional_inputs.smk"
 include: "rules/prepare_sequences.smk"
 include: "rules/subsample.smk"
@@ -82,4 +67,4 @@ rule clean:
 if "custom_rules" in config:
     for rule_file in config["custom_rules"]:
 
-        include: rule_file
+        include: os.path.join(os.getcwd(), rule_file)

phylogenetic/defaults/config.yaml

@@ -21,15 +21,18 @@ build_params:
 
     # See Nextstrain documentation for an explanation of how subsampling configuration works:
     # <https://docs.nextstrain.org/en/latest/guides/bioinformatics/filtering-and-subsampling.html#generalizing-subsampling-in-a-workflow>
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/all-lineages/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/all-lineages/include.txt
 
     refine:
       treetime_params: --coalescent opt --date-inference marginal --date-confidence --keep-polytomies --clock-rate 0.000755
@@ -57,15 +60,18 @@ build_params:
     reference: "defaults/lineage-1A/reference.gb"
     root: "KX394399"
 
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true' & lineage == '1A'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/lineage-1A/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true' & lineage == '1A'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/lineage-1A/include.txt
 
     # Clock rate from Table 1 of May et al, 2010: https://pmc.ncbi.nlm.nih.gov/articles/PMC3067944/
     refine:
@@ -94,15 +100,18 @@ build_params:
     reference: "defaults/lineage-2/reference.gb"
     root: "best"
 
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true' & lineage == '2'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/lineage-2/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true' & lineage == '2'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/lineage-2/include.txt
 
     # Clock rate from McMullen et al, 2013: https://pmc.ncbi.nlm.nih.gov/articles/PMC3709619/
     refine:

phylogenetic/rules/config.smk

@@ -0,0 +1,108 @@
+"""
+This part of the workflow deals with configuration.
+
+OUTPUTS:
+
+    results/run_configs/{timestamp}.yaml
+"""
+import os
+import sys
+import yaml
+from datetime import datetime
+from textwrap import dedent
+
+
+def main():
+    validate_config()
+    resolve_config_paths()
+    write_config()
+
+
+def validate_config():
+    """
+    Validate the config.
+
+    This could be improved with a schema definition file, but for now it serves
+    to provide useful error messages for common user errors and effects of
+    breaking changes.
+    """
+    # Validate 'builds'
+    if invalid_builds := set(config["builds"]) - set(config["build_params"]):
+        print(dedent(f"""\
+            ERROR: The following names in 'builds' are not defined in 'build_params':
+
+                {indented_list(invalid_builds, "            ")}
+
+            Available builds are:
+
+                {indented_list(config['build_params'], "            ")}
+            """))
+        exit(1)
+
+
+def resolve_config_paths():
+    """
+    Update all file paths in config by passing them through resolve_config_path()
+    """
+    global config
+
+    for build_name, build_config in config["build_params"].items():
+        # config.<build>.reference
+        build_config["reference"] = resolve_config_path(build_config["reference"])({})
+
+        # config.<build>.export
+        for key in ["description", "auspice_config"]:
+            build_config["export"][key] = resolve_config_path(build_config["export"][key])({})
+
+        # config.<build>.subsample
+        subsample_path_keys = ["exclude", "include", "group_by_weights"]
+        for sample_name, sample_config in build_config["subsample"].items():
+            for key in subsample_path_keys:
+                if key in sample_config:
+                    if isinstance(sample_config[key], list):
+                        sample_config[key] = [resolve_config_path(path)({}) for path in sample_config[key]]
+                    else:
+                        sample_config[key] = resolve_config_path(sample_config[key])({})
+
+
+def write_config():
+    """
+    Write Snakemake's 'config' variable to a file.
+
+    This is useful for debugging purposes.
+    """
+    timestamp = datetime.now().astimezone().strftime("%Y-%m-%dT%H%M%S.%f")
+    path = f"results/run_configs/{timestamp}.yaml"
+
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+
+    with open(path, 'w') as f:
+        yaml.dump(config, f, sort_keys=False)
+
+    print(f"Saved current run config to {path!r}.", file=sys.stderr)
+
+
+def indented_list(xs, prefix):
+    return f"\n{prefix}".join(xs)
+
+
+def conditional(option, argument):
+    """Used for config-defined arguments whose presence necessitates a command-line option
+    (e.g. --foo) prepended and whose absence should result in no option/arguments in the CLI command.
+    *argument* can be falsey, in which case an empty string is returned (i.e. "don't pass anything
+    to the CLI"), or a *list* or *string* or *number* in which case a flat list of options/args is returned,
+    or *True* in which case a list of a single element (the option) is returned.
+    Any other argument type is a WorkflowError
+    """
+    if not argument:
+        return ""
+    if argument is True: # must come before `isinstance(argument, int)` as bool is a subclass of int
+        return [option]
+    if isinstance(argument, list):
+        return [option, *argument]
+    if isinstance(argument, int) or isinstance(argument, float) or isinstance(argument, str):
+        return [option, argument]
+    raise WorkflowError(f"Workflow function conditional() received an argument value of unexpected type: {type(argument).__name__}")
+
+
+main()