From d869063d7f9286d42b2116249532f0cc728e4bd7 Mon Sep 17 00:00:00 2001
From: Victor Lin <13424970+victorlin@users.noreply.github.com>
Date: Thu, 18 Sep 2025 10:57:39 -0700
Subject: [PATCH 1/5] Consolidate config code into a single file

This improves readability of the main Snakefile, especially with more
config-related operations in mind.

phylogenetic/rules/config.smk #	deleted:
phylogenetic/rules/write_config.smk #
---
 phylogenetic/Snakefile              | 19 +--------
 phylogenetic/rules/config.smk       | 63 +++++++++++++++++++++++++++++
 phylogenetic/rules/write_config.smk | 21 ----------
 3 files changed, 64 insertions(+), 39 deletions(-)
 create mode 100644 phylogenetic/rules/config.smk
 delete mode 100644 phylogenetic/rules/write_config.smk

diff --git a/phylogenetic/Snakefile b/phylogenetic/Snakefile
index 5ddf4db..ed4ab69 100644
--- a/phylogenetic/Snakefile
+++ b/phylogenetic/Snakefile
@@ -8,23 +8,7 @@ workdir: workflow.current_basedir
 # Use default configuration values. Override with Snakemake's --configfile/--config options.
 configfile: "defaults/config.yaml"
 
-# Validate 'builds'
-from textwrap import dedent
-
-def indented_list(xs, prefix):
-    return f"\n{prefix}".join(xs)
-
-if invalid_builds := set(config["builds"]) - set(config["build_params"]):
-    print(dedent(f"""\
-        ERROR: The following names in 'builds' are not defined in 'build_params':
-
-            {indented_list(invalid_builds, "            ")}
-
-        Available builds are:
-
-            {indented_list(config['build_params'], "            ")}
-        """))
-    exit(1)
+include: "rules/config.smk"
 
 builds = config['builds']
 
@@ -53,7 +37,6 @@ rule all:
 # custom_rules imported below to ensure that the core workflow is not complicated
 # by build specific rules.
 
-include: "rules/write_config.smk"
 include: "rules/merge_additional_inputs.smk"
 include: "rules/prepare_sequences.smk"
 include: "rules/subsample.smk"
diff --git a/phylogenetic/rules/config.smk b/phylogenetic/rules/config.smk
new file mode 100644
index 0000000..8578b14
--- /dev/null
+++ b/phylogenetic/rules/config.smk
@@ -0,0 +1,63 @@
+"""
+This part of the workflow deals with configuration.
+
+OUTPUTS:
+
+    results/run_configs/{timestamp}.yaml
+"""
+import os
+import sys
+import yaml
+from datetime import datetime
+from textwrap import dedent
+
+
+def main():
+    validate_config()
+    write_config()
+
+
+def validate_config():
+    """
+    Validate the config.
+
+    This could be improved with a schema definition file, but for now it serves
+    to provide useful error messages for common user errors and effects of
+    breaking changes.
+    """
+    # Validate 'builds'
+    if invalid_builds := set(config["builds"]) - set(config["build_params"]):
+        print(dedent(f"""\
+            ERROR: The following names in 'builds' are not defined in 'build_params':
+
+                {indented_list(invalid_builds, "            ")}
+
+            Available builds are:
+
+                {indented_list(config['build_params'], "            ")}
+            """))
+        exit(1)
+
+
+def write_config():
+    """
+    Write Snakemake's 'config' variable to a file.
+
+    This is useful for debugging purposes.
+    """
+    timestamp = datetime.now().astimezone().strftime("%Y-%m-%dT%H%M%S.%f")
+    path = f"results/run_configs/{timestamp}.yaml"
+
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+
+    with open(path, 'w') as f:
+        yaml.dump(config, f, sort_keys=False)
+
+    print(f"Saved current run config to {path!r}.", file=sys.stderr)
+
+
+def indented_list(xs, prefix):
+    return f"\n{prefix}".join(xs)
+
+
+main()
diff --git a/phylogenetic/rules/write_config.smk b/phylogenetic/rules/write_config.smk
deleted file mode 100644
index 0a070be..0000000
--- a/phylogenetic/rules/write_config.smk
+++ /dev/null
@@ -1,21 +0,0 @@
-"""
-This part of the workflow writes run time configuration to a YAML file.
-
-OUTPUTS:
-
-    results/run_configs/{timestamp}.yaml
-"""
-import os
-import sys
-import yaml
-from datetime import datetime
-
-timestamp = datetime.now().astimezone().strftime("%Y-%m-%dT%H%M%S.%f")
-path = f"results/run_configs/{timestamp}.yaml"
-
-os.makedirs(os.path.dirname(path), exist_ok=True)
-
-with open(path, 'w') as f:
-    yaml.dump(config, f, sort_keys=False)
-
-print(f"Saved current run config to {path!r}.", file=sys.stderr)

From 89ae09468e71385e480a37ef225a7f62e0819b22 Mon Sep 17 00:00:00 2001
From: Victor Lin <13424970+victorlin@users.noreply.github.com>
Date: Thu, 18 Sep 2025 16:17:17 -0700
Subject: [PATCH 2/5] Support YAML-based config for augur filter

This is an alternative to using augur subsample.
---
 phylogenetic/defaults/config.yaml | 63 ++++++++++++++++++-------------
 phylogenetic/rules/config.smk     | 19 ++++++++++
 phylogenetic/rules/subsample.smk  | 42 +++++++++++++++++++--
 3 files changed, 94 insertions(+), 30 deletions(-)

diff --git a/phylogenetic/defaults/config.yaml b/phylogenetic/defaults/config.yaml
index e22cbff..8009650 100644
--- a/phylogenetic/defaults/config.yaml
+++ b/phylogenetic/defaults/config.yaml
@@ -21,15 +21,18 @@ build_params:
 
     # See Nextstrain documentation for an explanation of how subsampling configuration works:
     # <https://docs.nextstrain.org/en/latest/guides/bioinformatics/filtering-and-subsampling.html#generalizing-subsampling-in-a-workflow>
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/all-lineages/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/all-lineages/include.txt
 
     refine:
       treetime_params: --coalescent opt --date-inference marginal --date-confidence --keep-polytomies --clock-rate 0.000755
@@ -57,15 +60,18 @@ build_params:
     reference: "defaults/lineage-1A/reference.gb"
     root: "KX394399"
 
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true' & lineage == '1A'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/lineage-1A/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true' & lineage == '1A'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/lineage-1A/include.txt
 
     # Clock rate from Table 1 of May et al, 2010: https://pmc.ncbi.nlm.nih.gov/articles/PMC3067944/
     refine:
@@ -94,15 +100,18 @@ build_params:
     reference: "defaults/lineage-2/reference.gb"
     root: "best"
 
-    subsampling:
-      region: >-
-        --query "is_lab_host != 'true' & lineage == '2'"
-        --query-columns is_lab_host:str
-        --min-length '8200'
-        --group-by region year
-        --subsample-max-sequences 3000
-        --exclude defaults/exclude.txt
-        --include defaults/lineage-2/include.txt
+    subsample:
+      region:
+        query: is_lab_host != 'true' & lineage == '2'
+        query_columns:
+          - is_lab_host:str
+        min_length: 8200
+        group_by:
+          - region
+          - year
+        subsample_max_sequences: 3000
+        exclude: defaults/exclude.txt
+        include: defaults/lineage-2/include.txt
 
     # Clock rate from McMullen et al, 2013: https://pmc.ncbi.nlm.nih.gov/articles/PMC3709619/
     refine:
diff --git a/phylogenetic/rules/config.smk b/phylogenetic/rules/config.smk
index 8578b14..d3f3d16 100644
--- a/phylogenetic/rules/config.smk
+++ b/phylogenetic/rules/config.smk
@@ -60,4 +60,23 @@ def indented_list(xs, prefix):
     return f"\n{prefix}".join(xs)
 
 
+def conditional(option, argument):
+    """Used for config-defined arguments whose presence necessitates a command-line option
+    (e.g. --foo) prepended and whose absence should result in no option/arguments in the CLI command.
+    *argument* can be falsey, in which case an empty string is returned (i.e. "don't pass anything
+    to the CLI"), or a *list* or *string* or *number* in which case a flat list of options/args is returned,
+    or *True* in which case a list of a single element (the option) is returned.
+    Any other argument type is a WorkflowError
+    """
+    if not argument:
+        return ""
+    if argument is True: # must come before `isinstance(argument, int)` as bool is a subclass of int
+        return [option]
+    if isinstance(argument, list):
+        return [option, *argument]
+    if isinstance(argument, int) or isinstance(argument, float) or isinstance(argument, str):
+        return [option, argument]
+    raise WorkflowError(f"Workflow function conditional() received an argument value of unexpected type: {type(argument).__name__}")
+
+
 main()
diff --git a/phylogenetic/rules/subsample.smk b/phylogenetic/rules/subsample.smk
index 48e889f..b723124 100644
--- a/phylogenetic/rules/subsample.smk
+++ b/phylogenetic/rules/subsample.smk
@@ -29,7 +29,26 @@ rule subsample:
     benchmark:
         "benchmarks/{build}/{subsample}/subsampled_strains.txt",
     params:
-        filters = lambda w: config["build_params"][w.build]["subsampling"][w.subsample],
+        exclude = lambda w: conditional("--exclude", config["build_params"][w.build]["subsample"][w.subsample].get("exclude")),
+        exclude_all = lambda w: conditional("--exclude-all", config["build_params"][w.build]["subsample"][w.subsample].get("exclude_all")),
+        exclude_ambiguous_dates_by = lambda w: conditional("--exclude-ambiguous-dates-by", config["build_params"][w.build]["subsample"][w.subsample].get("exclude_ambiguous_dates_by")),
+        exclude_where = lambda w: conditional("--exclude-where", config["build_params"][w.build]["subsample"][w.subsample].get("exclude_where")),
+        group_by = lambda w: conditional("--group-by", config["build_params"][w.build]["subsample"][w.subsample].get("group_by")),
+        group_by_weights = lambda w: conditional("--group-by-weights", config["build_params"][w.build]["subsample"][w.subsample].get("group_by_weights")),
+        include = lambda w: conditional("--include", config["build_params"][w.build]["subsample"][w.subsample].get("include")),
+        include_where = lambda w: conditional("--include-where", config["build_params"][w.build]["subsample"][w.subsample].get("include_where")),
+        max_date = lambda w: conditional("--max-date", config["build_params"][w.build]["subsample"][w.subsample].get("max_date")),
+        max_length = lambda w: conditional("--max-length", config["build_params"][w.build]["subsample"][w.subsample].get("max_length")),
+        min_date = lambda w: conditional("--min-date", config["build_params"][w.build]["subsample"][w.subsample].get("min_date")),
+        min_length = lambda w: conditional("--min-length", config["build_params"][w.build]["subsample"][w.subsample].get("min_length")),
+        non_nucleotide = lambda w: conditional("--non-nucleotide", config["build_params"][w.build]["subsample"][w.subsample].get("non_nucleotide")),
+        probabilistic_sampling = lambda w: conditional("--probabilistic-sampling", config["build_params"][w.build]["subsample"][w.subsample].get("probabilistic_sampling")),
+        query = lambda w: conditional("--query", config["build_params"][w.build]["subsample"][w.subsample].get("query")),
+        query_columns = lambda w: conditional("--query-columns", config["build_params"][w.build]["subsample"][w.subsample].get("query_columns")),
+        # FIXME: --no-probabilistic-sampling?
+        # FIXME: --priority?
+        sequences_per_group = lambda w: conditional("--sequences-per-group", config["build_params"][w.build]["subsample"][w.subsample].get("sequences_per_group")),
+        subsample_max_sequences = lambda w: conditional("--subsample-max-sequences", config["build_params"][w.build]["subsample"][w.subsample].get("subsample_max_sequences")),
         id_column = config["strain_id_field"],
     shell:
         """
@@ -37,7 +56,24 @@ rule subsample:
             --sequences {input.sequences} \
             --metadata {input.metadata} \
             --metadata-id-columns {params.id_column} \
-            {params.filters} \
+            {params.exclude_all:q} \
+            {params.exclude_ambiguous_dates_by:q} \
+            {params.exclude_where:q} \
+            {params.exclude:q} \
+            {params.group_by_weights:q} \
+            {params.group_by:q} \
+            {params.include_where:q} \
+            {params.include:q} \
+            {params.max_date:q} \
+            {params.max_length:q} \
+            {params.min_date:q} \
+            {params.min_length:q} \
+            {params.non_nucleotide:q} \
+            {params.probabilistic_sampling:q} \
+            {params.query_columns:q} \
+            {params.query:q} \
+            {params.sequences_per_group:q} \
+            {params.subsample_max_sequences:q} \
             --output-strains {output.subsampled_strains} 2>&1 | tee {log}
         """
 
@@ -45,7 +81,7 @@ rule extract_subsampled_sequences_and_metadata:
     input:
         sequences = input_sequences,
         metadata = input_metadata,
-        subsampled_strains = lambda w: expand("results/{build}/subsampled_strains_{subsample}.txt", build=w.build, subsample=list(config["build_params"][w.build]["subsampling"].keys()))
+        subsampled_strains = lambda w: expand("results/{build}/subsampled_strains_{subsample}.txt", build=w.build, subsample=list(config["build_params"][w.build]["subsample"].keys()))
     output:
         sequences = "results/{build}/sequences_filtered.fasta",
         metadata = "results/{build}/metadata_filtered.tsv",

From 7c38893fd1cb587a5ad25882c505e376300b0611 Mon Sep 17 00:00:00 2001
From: Victor Lin <vlin@fredhutch.org>
Date: Mon, 8 Sep 2025 16:05:41 -0700
Subject: [PATCH 3/5] git subrepo clone (merge)
 https://github.com/nextstrain/shared shared/vendored

subrepo:
  subdir:   "shared/vendored"
  merged:   "2d063cf"
upstream:
  origin:   "https://github.com/nextstrain/shared"
  branch:   "main"
  commit:   "2d063cf"
git-subrepo:
  version:  "0.4.9"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "4f60dd7"
---
 shared/vendored/.github/dependabot.yml        |  17 ++
 .../vendored/.github/pull_request_template.md |  16 ++
 shared/vendored/.github/workflows/ci.yaml     |  15 ++
 .../.github/workflows/pre-commit.yaml         |  14 ++
 shared/vendored/.gitrepo                      |  12 ++
 shared/vendored/.pre-commit-config.yaml       |  40 +++++
 shared/vendored/.shellcheckrc                 |   6 +
 shared/vendored/README.md                     | 158 +++++++++++++++++
 shared/vendored/scripts/assign-colors         |  96 +++++++++++
 shared/vendored/scripts/cloudfront-invalidate |  42 +++++
 shared/vendored/scripts/download-from-s3      |  48 ++++++
 .../vendored/scripts/fetch-from-ncbi-entrez   |  70 ++++++++
 shared/vendored/scripts/notify-on-diff        |  35 ++++
 shared/vendored/scripts/notify-on-job-fail    |  23 +++
 shared/vendored/scripts/notify-on-job-start   |  27 +++
 .../vendored/scripts/notify-on-record-change  |  53 ++++++
 shared/vendored/scripts/notify-slack          |  93 ++++++++++
 shared/vendored/scripts/s3-object-exists      |   8 +
 shared/vendored/scripts/sha256sum             |  15 ++
 shared/vendored/scripts/trigger               |  56 ++++++
 shared/vendored/scripts/trigger-on-new-data   |  32 ++++
 shared/vendored/scripts/upload-to-s3          |  78 +++++++++
 shared/vendored/snakemake/config.smk          |  77 +++++++++
 shared/vendored/snakemake/remote_files.smk    | 159 ++++++++++++++++++
 24 files changed, 1190 insertions(+)
 create mode 100644 shared/vendored/.github/dependabot.yml
 create mode 100644 shared/vendored/.github/pull_request_template.md
 create mode 100644 shared/vendored/.github/workflows/ci.yaml
 create mode 100644 shared/vendored/.github/workflows/pre-commit.yaml
 create mode 100644 shared/vendored/.gitrepo
 create mode 100644 shared/vendored/.pre-commit-config.yaml
 create mode 100644 shared/vendored/.shellcheckrc
 create mode 100644 shared/vendored/README.md
 create mode 100755 shared/vendored/scripts/assign-colors
 create mode 100755 shared/vendored/scripts/cloudfront-invalidate
 create mode 100755 shared/vendored/scripts/download-from-s3
 create mode 100755 shared/vendored/scripts/fetch-from-ncbi-entrez
 create mode 100755 shared/vendored/scripts/notify-on-diff
 create mode 100755 shared/vendored/scripts/notify-on-job-fail
 create mode 100755 shared/vendored/scripts/notify-on-job-start
 create mode 100755 shared/vendored/scripts/notify-on-record-change
 create mode 100755 shared/vendored/scripts/notify-slack
 create mode 100755 shared/vendored/scripts/s3-object-exists
 create mode 100755 shared/vendored/scripts/sha256sum
 create mode 100755 shared/vendored/scripts/trigger
 create mode 100755 shared/vendored/scripts/trigger-on-new-data
 create mode 100755 shared/vendored/scripts/upload-to-s3
 create mode 100644 shared/vendored/snakemake/config.smk
 create mode 100644 shared/vendored/snakemake/remote_files.smk

diff --git a/shared/vendored/.github/dependabot.yml b/shared/vendored/.github/dependabot.yml
new file mode 100644
index 0000000..0a50ee1
--- /dev/null
+++ b/shared/vendored/.github/dependabot.yml
@@ -0,0 +1,17 @@
+# Dependabot configuration file
+# <https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file>
+#
+# Each ecosystem is checked on a scheduled interval defined below.  To trigger
+# a check manually, go to
+#
+#   https://github.com/nextstrain/shared/network/updates
+#
+# and look for a "Check for updates" button.  You may need to click around a
+# bit first.
+---
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
diff --git a/shared/vendored/.github/pull_request_template.md b/shared/vendored/.github/pull_request_template.md
new file mode 100644
index 0000000..ed4a5b2
--- /dev/null
+++ b/shared/vendored/.github/pull_request_template.md
@@ -0,0 +1,16 @@
+### Description of proposed changes
+
+<!-- What is the goal of this pull request? What does this pull request change? -->
+
+### Related issue(s)
+
+<!-- Link any related issues here. -->
+
+### Checklist
+
+<!-- Make sure checks are successful at the bottom of the PR. -->
+
+- [ ] Checks pass
+- [ ] If adding a script, add an entry for it in the README.
+
+<!-- 🙌 Thank you for contributing to Nextstrain! ✨ -->
diff --git a/shared/vendored/.github/workflows/ci.yaml b/shared/vendored/.github/workflows/ci.yaml
new file mode 100644
index 0000000..94d3054
--- /dev/null
+++ b/shared/vendored/.github/workflows/ci.yaml
@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: nextstrain/.github/actions/shellcheck@master
diff --git a/shared/vendored/.github/workflows/pre-commit.yaml b/shared/vendored/.github/workflows/pre-commit.yaml
new file mode 100644
index 0000000..a418753
--- /dev/null
+++ b/shared/vendored/.github/workflows/pre-commit.yaml
@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  - push
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: pre-commit/action@v3.0.1
diff --git a/shared/vendored/.gitrepo b/shared/vendored/.gitrepo
new file mode 100644
index 0000000..e174fc0
--- /dev/null
+++ b/shared/vendored/.gitrepo
@@ -0,0 +1,12 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme
+;
+[subrepo]
+	remote = https://github.com/nextstrain/shared
+	branch = main
+	commit = 2d063cf2bae0cfc91d70fda2c36f1451656e5757
+	parent = 6f9a6bb9a7625aa1f8146b4349e4466103d0a53f
+	method = merge
+	cmdver = 0.4.9
diff --git a/shared/vendored/.pre-commit-config.yaml b/shared/vendored/.pre-commit-config.yaml
new file mode 100644
index 0000000..2cdf88b
--- /dev/null
+++ b/shared/vendored/.pre-commit-config.yaml
@@ -0,0 +1,40 @@
+default_language_version:
+  python: python3
+repos:
+  - repo: https://github.com/pre-commit/sync-pre-commit-deps
+    rev: v0.0.1
+    hooks:
+      - id: sync-pre-commit-deps
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.6.27
+    hooks:
+      - id: actionlint
+        entry: env SHELLCHECK_OPTS='--exclude=SC2027' actionlint
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-json
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: destroyed-symlinks
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.6
+    hooks:
+      # Run the linter.
+      - id: ruff
diff --git a/shared/vendored/.shellcheckrc b/shared/vendored/.shellcheckrc
new file mode 100644
index 0000000..ebed438
--- /dev/null
+++ b/shared/vendored/.shellcheckrc
@@ -0,0 +1,6 @@
+# Use of this file requires Shellcheck v0.7.0 or newer.
+#
+# SC2064 - We intentionally want variables to expand immediately within traps
+#          so the trap can not fail due to variable interpolation later.
+#
+disable=SC2064
diff --git a/shared/vendored/README.md b/shared/vendored/README.md
new file mode 100644
index 0000000..eb2e3ba
--- /dev/null
+++ b/shared/vendored/README.md
@@ -0,0 +1,158 @@
+# shared
+
+Shared internal tooling for pathogen workflows.  Used by our individual
+pathogen repos which produce Nextstrain builds.  Expected to be vendored by
+each pathogen repo using `git subrepo`.
+
+Some tools may only live here temporarily before finding a permanent home in
+`augur curate` or Nextstrain CLI.  Others may happily live out their days here.
+
+## Vendoring
+
+Nextstrain maintained pathogen repos will use [`git subrepo`](https://github.com/ingydotnet/git-subrepo) to vendor shared scripts.
+(See discussion on this decision in https://github.com/nextstrain/shared/issues/3)
+
+For a list of Nextstrain repos that are currently using this method, use [this
+GitHub code search](https://github.com/search?type=code&q=org%3Anextstrain+subrepo+%22remote+%3D+https%3A%2F%2Fgithub.com%2Fnextstrain%2Fingest%22).
+
+If you don't already have `git subrepo` installed, follow the [git subrepo installation instructions](https://github.com/ingydotnet/git-subrepo#installation).
+Then add the latest shared scripts to the pathogen repo by running:
+
+```
+git subrepo clone https://github.com/nextstrain/shared shared/vendored
+```
+
+Any future updates of sahred scripts can be pulled in with:
+
+```
+git subrepo pull shared/vendored
+```
+
+If you run into merge conflicts and would like to pull in a fresh copy of the
+latest shared scripts, pull with the `--force` flag:
+
+```
+git subrepo pull shared/vendored --force
+```
+
+> **Warning**
+> Beware of rebasing/dropping the parent commit of a `git subrepo` update
+
+`git subrepo` relies on metadata in the `shared/vendored/.gitrepo` file,
+which includes the hash for the parent commit in the pathogen repos.
+If this hash no longer exists in the commit history, there will be errors when
+running future `git subrepo pull` commands.
+
+If you run into an error similar to the following:
+```
+$ git subrepo pull shared/vendored
+git-subrepo: Command failed: 'git branch subrepo/shared/vendored '.
+fatal: not a valid object name: ''
+```
+Check the parent commit hash in the `shared/vendored/.gitrepo` file and make
+sure the commit exists in the commit history. Update to the appropriate parent
+commit hash if needed.
+
+## History
+
+Much of this tooling originated in
+[ncov-ingest](https://github.com/nextstrain/ncov-ingest) and was passaged thru
+[mpox's ingest/](https://github.com/nextstrain/mpox/tree/@/ingest/). It
+subsequently proliferated from [mpox][] to other pathogen repos ([rsv][],
+[zika][], [dengue][], [hepatitisB][], [forecasts-ncov][]) primarily thru
+copying.  To [counter that
+proliferation](https://bedfordlab.slack.com/archives/C7SDVPBLZ/p1688577879947079),
+this repo was made.
+
+[mpox]: https://github.com/nextstrain/mpox
+[rsv]: https://github.com/nextstrain/rsv
+[zika]: https://github.com/nextstrain/zika/pull/24
+[dengue]: https://github.com/nextstrain/dengue/pull/10
+[hepatitisB]: https://github.com/nextstrain/hepatitisB
+[forecasts-ncov]: https://github.com/nextstrain/forecasts-ncov
+
+## Elsewhere
+
+The creation of this repo, in both the abstract and concrete, and the general
+approach to "ingest" has been discussed in various internal places, including:
+
+- https://github.com/nextstrain/private/issues/59
+- @joverlee521's [workflows document](https://docs.google.com/document/d/1rLWPvEuj0Ayc8MR0O1lfRJZfj9av53xU38f20g8nU_E/edit#heading=h.4g0d3mjvb89i)
+- [5 July 2023 Slack thread](https://bedfordlab.slack.com/archives/C7SDVPBLZ/p1688577879947079)
+- [6 July 2023 team meeting](https://docs.google.com/document/d/1FPfx-ON5RdqL2wyvODhkrCcjgOVX3nlXgBwCPhIEsco/edit)
+- _…many others_
+
+## Scripts
+
+Scripts for supporting workflow automation that don’t really belong in any of our existing tools.
+
+- [assign-colors](scripts/assign-colors) - Generate colors.tsv for augur export based on ordering, color schemes, and what exists in the metadata. Used in the phylogenetic or nextclade workflows.
+- [notify-on-diff](scripts/notify-on-diff) - Send Slack message with diff of a local file and an S3 object
+- [notify-on-job-fail](scripts/notify-on-job-fail) - Send Slack message with details about failed workflow job on GitHub Actions and/or AWS Batch
+- [notify-on-job-start](scripts/notify-on-job-start) - Send Slack message with details about workflow job on GitHub Actions and/or AWS Batch
+- [notify-on-record-change](scripts/notify-on-recod-change) - Send Slack message with details about line count changes for a file compared to an S3 object's metadata `recordcount`.
+  If the S3 object's metadata does not have `recordcount`, then will attempt to download S3 object to count lines locally, which only supports `xz` compressed S3 objects.
+- [notify-slack](scripts/notify-slack) - Send message or file to Slack
+- [s3-object-exists](scripts/s3-object-exists) - Used to prevent 404 errors during S3 file comparisons in the notify-* scripts
+- [trigger](scripts/trigger) - Triggers downstream GitHub Actions via the GitHub API using repository_dispatch events.
+- [trigger-on-new-data](scripts/trigger-on-new-data) - Triggers downstream GitHub Actions if the provided `upload-to-s3` outputs do not contain the `identical_file_message`
+  A hacky way to ensure that we only trigger downstream phylogenetic builds if the S3 objects have been updated.
+
+
+NCBI interaction scripts that are useful for fetching public metadata and sequences.
+
+- [fetch-from-ncbi-entrez](scripts/fetch-from-ncbi-entrez) - Fetch metadata and nucleotide sequences from [NCBI Entrez](https://www.ncbi.nlm.nih.gov/books/NBK25501/) and output to a GenBank file.
+  Useful for pathogens with metadata and annotations in custom fields that are not part of the standard [NCBI Datasets](https://www.ncbi.nlm.nih.gov/datasets/) outputs.
+
+Historically, some pathogen repos used the undocumented NCBI Virus API through [fetch-from-ncbi-virus](https://github.com/nextstrain/shared/blob/c97df238518171c2b1574bec0349a55855d1e7a7/fetch-from-ncbi-virus) to fetch data. However we've opted to drop the NCBI Virus scripts due to https://github.com/nextstrain/shared/issues/18.
+
+Potential Nextstrain CLI scripts
+
+- [sha256sum](scripts/sha256sum) - Used to check if files are identical in upload-to-s3 and download-from-s3 scripts.
+- [cloudfront-invalidate](scripts/cloudfront-invalidate) - CloudFront invalidation is already supported in the [nextstrain remote command for S3 files](https://github.com/nextstrain/cli/blob/a5dda9c0579ece7acbd8e2c32a4bbe95df7c0bce/nextstrain/cli/remote/s3.py#L104).
+  This exists as a separate script to support CloudFront invalidation when using the upload-to-s3 script.
+- [upload-to-s3](scripts/upload-to-s3) - Upload file to AWS S3 bucket with compression based on file extension in S3 URL.
+  Skips upload if the local file's hash is identical to the S3 object's metadata `sha256sum`.
+  Adds the following user defined metadata to uploaded S3 object:
+    - `sha256sum` - hash of the file generated by [sha256sum](sha256sum)
+    - `recordcount` - the line count of the file
+- [download-from-s3](scripts/download-from-s3) - Download file from AWS S3 bucket with decompression based on file extension in S3 URL.
+  Skips download if the local file already exists and has a hash identical to the S3 object's metadata `sha256sum`.
+
+## Snakemake
+
+Snakemake workflow functions that are shared across many pathogen workflows that don’t really belong in any of our existing tools.
+
+- [config.smk](snakemake/config.smk) - Shared functions for parsing workflow configs.
+- [remote_files.smk](snakemake/remote_files.smk) - Exposes the `path_or_url` function which will use Snakemake's storage plugins to download/upload files to remote providers as needed.
+
+
+## Software requirements
+
+Some scripts may require Bash ≥4. If you are running these scripts on macOS, the builtin Bash (`/bin/bash`) does not meet this requirement. You can install [Homebrew's Bash](https://formulae.brew.sh/formula/bash) which is more up to date.
+
+## Testing
+
+Most scripts are untested within this repo, relying on "testing in production". That is the only practical testing option for some scripts such as the ones interacting with S3 and Slack.
+
+## Working on this repo
+
+This repo is configured to use [pre-commit](https://pre-commit.com),
+to help automatically catch common coding errors and syntax issues
+with changes before they are committed to the repo.
+
+If you will be writing new code or otherwise working within this repo,
+please do the following to get started:
+
+1. [install `pre-commit`](https://pre-commit.com/#install) by running
+   either `python -m pip install pre-commit` or `brew install
+   pre-commit`, depending on your preferred package management
+   solution
+2. install the local git hooks by running `pre-commit install` from
+   the root of the repo
+3. when problems are detected, correct them in your local working tree
+   before committing them.
+
+Note that these pre-commit checks are also run in a GitHub Action when
+changes are pushed to GitHub, so correcting issues locally will
+prevent extra cycles of correction.
diff --git a/shared/vendored/scripts/assign-colors b/shared/vendored/scripts/assign-colors
new file mode 100755
index 0000000..e42a44d
--- /dev/null
+++ b/shared/vendored/scripts/assign-colors
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Generate colors.tsv for augur export based on ordering, color schemes, and
+traits that exists in the metadata.
+"""
+import argparse
+import pandas as pd
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Assign colors based on defined ordering of traits.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument('--ordering', type=str, required=True,
+        help="""Input TSV file defining the color ordering where the first
+        column is the field and the second column is the trait in that field.
+        Blank lines are ignored. Lines starting with '#' will be ignored as comments.""")
+    parser.add_argument('--color-schemes', type=str, required=True,
+        help="Input color schemes where each line is a different color scheme separated by tabs.")
+    parser.add_argument('--metadata', type=str,
+        help="""If provided, restrict colors to only those traits found in
+        metadata. If the metadata includes a 'focal' column that only contains
+        boolean values, then restrict colors to traits for rows where 'focal'
+        is set to True.""")
+    parser.add_argument('--output', type=str, required=True,
+        help="Output colors TSV file to be passed to augur export.")
+    args = parser.parse_args()
+
+    assignment = {}
+    with open(args.ordering) as f:
+        for line in f.readlines():
+            array = line.strip().split("\t")
+            # Ignore empty lines or commented lines
+            if not array or not array[0] or array[0].startswith('#'):
+                continue
+            # Throw a warning if encountering a line not matching the expected number of columns, ignore line
+            elif len(array)!=2:
+                print(f"WARNING: Could not decode color ordering line: {line}")
+                continue
+            # Otherwise, process color ordering where we expect 2 columns: name, traits
+            else:
+                name = array[0]
+                trait = array[1]
+                if name not in assignment:
+                    assignment[name] = [trait]
+                else:
+                    assignment[name].append(trait)
+
+    # if metadata supplied, go through and
+    # 1. remove assignments that don't exist in metadata
+    # 2. remove assignments that have 'focal' set to 'False' in metadata
+    if args.metadata:
+        metadata = pd.read_csv(args.metadata, delimiter='\t')
+        for name, trait in assignment.items():
+            if name in metadata:
+                if 'focal' in metadata and metadata['focal'].dtype == 'bool':
+                    focal_list = metadata.loc[metadata['focal'], name].unique()
+                    subset_focal = [x for x in assignment[name] if x in focal_list]
+                    assignment[name] = subset_focal
+                else: # no 'focal' present
+                    subset_present = [x for x in assignment[name] if x in metadata[name].unique()]
+                    assignment[name] = subset_present
+
+
+    schemes = {}
+    counter = 0
+    with open(args.color_schemes) as f:
+        for line in f.readlines():
+            counter += 1
+            array = line.lstrip().rstrip().split("\t")
+            schemes[counter] = array
+
+    with open(args.output, 'w') as f:
+        for trait_name, trait_array in assignment.items():
+            if len(trait_array)==0:
+                print(f"No traits found for {trait_name}")
+                continue
+            if len(schemes)<len(trait_array):
+              print(f"WARNING: insufficient colours available for trait {trait_name} - reusing colours!")
+              remain = len(trait_array)
+              color_array = []
+              while(remain>0):
+                if (remain>len(schemes)):
+                  color_array = [*color_array, *schemes[len(schemes)]]
+                  remain -= len(schemes)
+                else:
+                  color_array = [*color_array, *schemes[remain]]
+                  remain = 0
+            else:
+              color_array = schemes[len(trait_array)]
+
+            zipped = list(zip(trait_array, color_array))
+            for trait_value, color in zipped:
+                f.write(trait_name + "\t" + trait_value + "\t" + color + "\n")
+            f.write("\n")
diff --git a/shared/vendored/scripts/cloudfront-invalidate b/shared/vendored/scripts/cloudfront-invalidate
new file mode 100755
index 0000000..dbea398
--- /dev/null
+++ b/shared/vendored/scripts/cloudfront-invalidate
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Originally from @tsibley's gist: https://gist.github.com/tsibley/a66262d341dedbea39b02f27e2837ea8
+set -euo pipefail
+
+main() {
+    local domain="$1"
+    shift
+    local paths=("$@")
+    local distribution invalidation
+
+    echo "-> Finding CloudFront distribution"
+    distribution=$(
+        aws cloudfront list-distributions \
+            --query "DistributionList.Items[?contains(Aliases.Items, \`$domain\`)] | [0].Id" \
+            --output text
+    )
+
+    if [[ -z $distribution || $distribution == None ]]; then
+        exec >&2
+        echo "Unable to find CloudFront distribution id for $domain"
+        echo
+        echo "Are your AWS CLI credentials for the right account?"
+        exit 1
+    fi
+
+    echo "-> Creating CloudFront invalidation for distribution $distribution"
+    invalidation=$(
+        aws cloudfront create-invalidation \
+            --distribution-id "$distribution" \
+            --paths "${paths[@]}" \
+            --query Invalidation.Id \
+            --output text
+    )
+
+    echo "-> Waiting for CloudFront invalidation $invalidation to complete"
+    echo "   Ctrl-C to stop waiting."
+    aws cloudfront wait invalidation-completed \
+        --distribution-id "$distribution" \
+        --id "$invalidation"
+}
+
+main "$@"
diff --git a/shared/vendored/scripts/download-from-s3 b/shared/vendored/scripts/download-from-s3
new file mode 100755
index 0000000..4981186
--- /dev/null
+++ b/shared/vendored/scripts/download-from-s3
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+bin="$(dirname "$0")"
+
+main() {
+    local src="${1:?A source s3:// URL is required as the first argument.}"
+    local dst="${2:?A destination file path is required as the second argument.}"
+    # How many lines to subsample to. 0 means no subsampling. Optional.
+    # It is not advised to use this for actual subsampling! This is intended to be
+    # used for debugging workflows with large datasets such as ncov-ingest as
+    # described in https://github.com/nextstrain/ncov-ingest/pull/367
+
+    # Uses `tsv-sample` to subsample, so it will not work as expected with files
+    # that have a single record split across multiple lines (i.e. FASTA sequences)
+    local n="${3:-0}"
+
+    local s3path="${src#s3://}"
+    local bucket="${s3path%%/*}"
+    local key="${s3path#*/}"
+
+    local src_hash dst_hash no_hash=0000000000000000000000000000000000000000000000000000000000000000
+    dst_hash="$("$bin/sha256sum" < "$dst" || true)"
+    src_hash="$(aws s3api head-object --bucket "$bucket" --key "$key" --query Metadata.sha256sum --output text 2>/dev/null || echo "$no_hash")"
+
+    echo "[ INFO] Downloading $src → $dst"
+    if [[ $src_hash != "$dst_hash" ]]; then
+        aws s3 cp --no-progress "$src" - |
+        if [[ "$src" == *.gz ]]; then
+            gunzip -cfq
+        elif  [[ "$src" == *.xz ]]; then
+            xz -T0 -dcq
+        elif [[ "$src" == *.zst ]]; then
+            zstd -T0 -dcq
+        else
+            cat
+        fi |
+        if [[ "$n" -gt 0 ]]; then
+            tsv-sample -H -i -n "$n"
+        else
+            cat
+        fi >"$dst"
+    else
+        echo "[ INFO] Files are identical, skipping download"
+    fi
+}
+
+main "$@"
diff --git a/shared/vendored/scripts/fetch-from-ncbi-entrez b/shared/vendored/scripts/fetch-from-ncbi-entrez
new file mode 100755
index 0000000..194a0c8
--- /dev/null
+++ b/shared/vendored/scripts/fetch-from-ncbi-entrez
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Fetch metadata and nucleotide sequences from NCBI Entrez and output to a GenBank file.
+"""
+import json
+import argparse
+from Bio import SeqIO, Entrez
+
+# To use the efetch API, the docs indicate only around 10,000 records should be fetched per request
+# https://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch
+# However, in my testing with HepB, the max records returned was 9,999
+#   - Jover, 16 August 2023
+BATCH_SIZE = 9999
+
+Entrez.email = "hello@nextstrain.org"
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--term', required=True, type=str,
+        help='Genbank search term. Replace spaces with "+", e.g. "Hepatitis+B+virus[All+Fields]complete+genome[All+Fields]"')
+    parser.add_argument('--output', required=True, type=str, help='Output file (Genbank)')
+    return parser.parse_args()
+
+
+def get_esearch_history(term):
+    """
+    Search for the provided *term* via ESearch and store the results using the
+    Entrez history server.¹
+
+    Returns the total count of returned records, query key, and web env needed
+    to access the records from the server.
+
+    ¹ https://www.ncbi.nlm.nih.gov/books/NBK25497/#chapter2.Using_the_Entrez_History_Server
+    """
+    handle = Entrez.esearch(db="nucleotide", term=term, retmode="json", usehistory="y", retmax=0)
+    esearch_result = json.loads(handle.read())['esearchresult']
+    print(f"Search term {term!r} returned {esearch_result['count']} IDs.")
+    return {
+        "count": int(esearch_result["count"]),
+        "query_key": esearch_result["querykey"],
+        "web_env": esearch_result["webenv"]
+    }
+
+
+def fetch_from_esearch_history(count, query_key, web_env):
+    """
+    Fetch records in batches from Entrez history server using the provided
+    *query_key* and *web_env* and yields them as a BioPython SeqRecord iterator.
+    """
+    print(f"Fetching GenBank records in batches of n={BATCH_SIZE}")
+
+    for start in range(0, count, BATCH_SIZE):
+        handle = Entrez.efetch(
+            db="nucleotide",
+            query_key=query_key,
+            webenv=web_env,
+            retstart=start,
+            retmax=BATCH_SIZE,
+            rettype="gb",
+            retmode="text")
+
+        yield SeqIO.parse(handle, "genbank")
+
+
+if __name__=="__main__":
+    args = parse_args()
+
+    with open(args.output, "w") as output_handle:
+        for batch_results in fetch_from_esearch_history(**get_esearch_history(args.term)):
+            SeqIO.write(batch_results, output_handle, "genbank")
diff --git a/shared/vendored/scripts/notify-on-diff b/shared/vendored/scripts/notify-on-diff
new file mode 100755
index 0000000..ddbe7da
--- /dev/null
+++ b/shared/vendored/scripts/notify-on-diff
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
+: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
+
+bin="$(dirname "$0")"
+
+src="${1:?A source file is required as the first argument.}"
+dst="${2:?A destination s3:// URL is required as the second argument.}"
+
+dst_local="$(mktemp -t s3-file-XXXXXX)"
+diff="$(mktemp -t diff-XXXXXX)"
+
+trap "rm -f '$dst_local' '$diff'" EXIT
+
+# if the file is not already present, just exit
+"$bin"/s3-object-exists "$dst" || exit 0
+
+"$bin"/download-from-s3 "$dst" "$dst_local"
+
+# diff's exit code is 0 for no differences, 1 for differences found, and >1 for errors
+diff_exit_code=0
+diff "$dst_local" "$src" > "$diff" || diff_exit_code=$?
+
+if [[ "$diff_exit_code" -eq 1 ]]; then
+    echo "Notifying Slack about diff."
+    "$bin"/notify-slack --upload "$src.diff" < "$diff"
+elif [[ "$diff_exit_code" -gt 1 ]]; then
+    echo "Notifying Slack about diff failure"
+    "$bin"/notify-slack "Diff failed for $src"
+else
+    echo "No change in $src."
+fi
diff --git a/shared/vendored/scripts/notify-on-job-fail b/shared/vendored/scripts/notify-on-job-fail
new file mode 100755
index 0000000..7dd2409
--- /dev/null
+++ b/shared/vendored/scripts/notify-on-job-fail
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
+: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
+
+: "${AWS_BATCH_JOB_ID:=}"
+: "${GITHUB_RUN_ID:=}"
+
+bin="$(dirname "$0")"
+job_name="${1:?A job name is required as the first argument}"
+github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}"
+
+echo "Notifying Slack about failed ${job_name} job."
+message="❌ ${job_name} job has FAILED 😞 "
+
+if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then
+    message+="See AWS Batch job \`${AWS_BATCH_JOB_ID}\` (<https://console.aws.amazon.com/batch/v2/home?region=us-east-1#jobs/detail/${AWS_BATCH_JOB_ID}|link>) for error details. "
+elif [[ -n "${GITHUB_RUN_ID}" ]]; then
+    message+="See GitHub Action <https://github.com/${github_repo}/actions/runs/${GITHUB_RUN_ID}?check_suite_focus=true|${GITHUB_RUN_ID}> for error details. "
+fi
+
+"$bin"/notify-slack "$message"
diff --git a/shared/vendored/scripts/notify-on-job-start b/shared/vendored/scripts/notify-on-job-start
new file mode 100755
index 0000000..1c8ce7d
--- /dev/null
+++ b/shared/vendored/scripts/notify-on-job-start
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
+: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
+
+: "${AWS_BATCH_JOB_ID:=}"
+: "${GITHUB_RUN_ID:=}"
+
+bin="$(dirname "$0")"
+job_name="${1:?A job name is required as the first argument}"
+github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}"
+build_dir="${3:-ingest}"
+
+echo "Notifying Slack about started ${job_name} job."
+message="${job_name} job has started."
+
+if [[ -n "${GITHUB_RUN_ID}" ]]; then
+  message+=" The job was submitted by GitHub Action <https://github.com/${github_repo}/actions/runs/${GITHUB_RUN_ID}?check_suite_focus=true|${GITHUB_RUN_ID}>."
+fi
+
+if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then
+  message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` (<https://console.aws.amazon.com/batch/v2/home?region=us-east-1#jobs/detail/${AWS_BATCH_JOB_ID}|link>)."
+  message+=" Follow along in your local clone of ${github_repo} with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ${build_dir}"'```'
+fi
+
+"$bin"/notify-slack "$message"
diff --git a/shared/vendored/scripts/notify-on-record-change b/shared/vendored/scripts/notify-on-record-change
new file mode 100755
index 0000000..f424252
--- /dev/null
+++ b/shared/vendored/scripts/notify-on-record-change
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
+: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
+
+bin="$(dirname "$0")"
+
+src="${1:?A source ndjson file is required as the first argument.}"
+dst="${2:?A destination ndjson s3:// URL is required as the second argument.}"
+source_name=${3:?A record source name is required as the third argument.}
+
+# if the file is not already present, just exit
+"$bin"/s3-object-exists "$dst" || exit 0
+
+s3path="${dst#s3://}"
+bucket="${s3path%%/*}"
+key="${s3path#*/}"
+
+src_record_count="$(wc -l < "$src")"
+
+# Try getting record count from S3 object metadata
+dst_record_count="$(aws s3api head-object --bucket "$bucket" --key "$key" --query "Metadata.recordcount || ''" --output text 2>/dev/null || true)"
+if [[ -z "$dst_record_count" ]]; then
+  # This object doesn't have the record count stored as metadata
+  # We have to download it and count the lines locally
+  dst_record_count="$(wc -l < <(aws s3 cp --no-progress "$dst" - | xz -T0 -dcfq))"
+fi
+
+added_records="$(( src_record_count - dst_record_count ))"
+
+printf "%'4d %s\n" "$src_record_count" "$src"
+printf "%'4d %s\n" "$dst_record_count" "$dst"
+printf "%'4d added records\n" "$added_records"
+
+slack_message=""
+
+if [[ $added_records -gt 0 ]]; then
+    echo "Notifying Slack about added records (n=$added_records)"
+    slack_message="📈 New records (n=$added_records) found on $source_name."
+
+elif [[ $added_records -lt 0 ]]; then
+    echo "Notifying Slack about fewer records (n=$added_records)"
+    slack_message="📉 Fewer records (n=$added_records) found on $source_name."
+
+else
+    echo "Notifying Slack about same number of records"
+    slack_message="⛔ No new records found on $source_name."
+fi
+
+slack_message+=" (Total record count: $src_record_count)"
+
+"$bin"/notify-slack "$slack_message"
diff --git a/shared/vendored/scripts/notify-slack b/shared/vendored/scripts/notify-slack
new file mode 100755
index 0000000..c6f1a87
--- /dev/null
+++ b/shared/vendored/scripts/notify-slack
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
+: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
+
+upload=0
+output=/dev/null
+thread_ts=""
+broadcast=0
+fail_on_error=0
+args=()
+
+for arg; do
+    case "$arg" in
+        --upload)
+            upload=1;;
+        --output=*)
+            output="${arg#*=}";;
+        --thread-ts=*)
+            thread_ts="${arg#*=}";;
+        --broadcast)
+            broadcast=1;;
+        --fail-on-error)
+            fail_on_error=1;;
+        *)
+            args+=("$arg");;
+    esac
+done
+
+set -- "${args[@]}"
+
+text="${1:?Some message text is required.}"
+
+send_slack_message() {
+    if [[ "$upload" == 1 ]]; then
+        echo "Uploading data to Slack with the message: $text"
+
+        upload_file="$(mktemp -t upload-file-XXXXXX)"
+        trap "rm -f '$upload_file'" EXIT
+
+        cat /dev/stdin > "$upload_file"
+        # printf used to strip whitespace from output of macOS/BSD wc
+        # See <https://github.com/nextstrain/shared/pull/47#discussion_r1974802967>
+        length=$(printf '%d' "$(<"$upload_file" wc -c)")
+
+        upload_info=$(curl https://slack.com/api/files.getUploadURLExternal \
+            --header "Authorization: Bearer $SLACK_TOKEN" \
+            --form-string filename="$text" \
+            --form-string length="$length" \
+            --fail --silent --show-error \
+            --http1.1 )
+
+        upload_url="$(jq -r .upload_url <<< "$upload_info")"
+        curl "$upload_url" \
+            --form-string filename="$text" \
+            --form file="@$upload_file" \
+            --fail --silent --show-error \
+            --http1.1 > /dev/null
+
+        files_uploaded="$(jq -r "[{id: .file_id}]" <<< "$upload_info")"
+        curl -X POST https://slack.com/api/files.completeUploadExternal \
+            --header "Authorization: Bearer $SLACK_TOKEN" \
+            --form-string channel_id="$SLACK_CHANNELS" \
+            --form-string thread_ts="$thread_ts" \
+            --form-string files="$files_uploaded" \
+            --fail --silent --show-error \
+            --http1.1 \
+            --output "$output"
+
+    else
+        echo "Posting Slack message: $text"
+        curl https://slack.com/api/chat.postMessage \
+            --header "Authorization: Bearer $SLACK_TOKEN" \
+            --form-string channel="$SLACK_CHANNELS" \
+            --form-string text="$text" \
+            --form-string thread_ts="$thread_ts" \
+            --form-string reply_broadcast="$broadcast" \
+            --fail --silent --show-error \
+            --http1.1 \
+            --output "$output"
+    fi
+}
+
+if ! send_slack_message; then
+    if [[ "$fail_on_error" == 1 ]]; then
+        echo "Sending Slack message failed"
+        exit 1
+    else
+        echo "Sending Slack message failed, but exiting with success anyway."
+        exit 0
+    fi
+fi
diff --git a/shared/vendored/scripts/s3-object-exists b/shared/vendored/scripts/s3-object-exists
new file mode 100755
index 0000000..679c20a
--- /dev/null
+++ b/shared/vendored/scripts/s3-object-exists
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+url="${1#s3://}"
+bucket="${url%%/*}"
+key="${url#*/}"
+
+aws s3api head-object --bucket "$bucket" --key "$key" &>/dev/null
diff --git a/shared/vendored/scripts/sha256sum b/shared/vendored/scripts/sha256sum
new file mode 100755
index 0000000..32d7ef8
--- /dev/null
+++ b/shared/vendored/scripts/sha256sum
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+Portable sha256sum utility.
+"""
+from hashlib import sha256
+from sys import stdin
+
+chunk_size = 5 * 1024**2 # 5 MiB
+
+h = sha256()
+
+for chunk in iter(lambda: stdin.buffer.read(chunk_size), b""):
+    h.update(chunk)
+
+print(h.hexdigest())
diff --git a/shared/vendored/scripts/trigger b/shared/vendored/scripts/trigger
new file mode 100755
index 0000000..586f9cc
--- /dev/null
+++ b/shared/vendored/scripts/trigger
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${PAT_GITHUB_DISPATCH:=}"
+
+github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
+event_type="${2:?An event type is required as the second argument.}"
+shift 2
+
+if [[ $# -eq 0 && -z $PAT_GITHUB_DISPATCH ]]; then
+    cat >&2 <<.
+You must specify options to curl for your GitHub credentials.  For example, you
+can specify your GitHub username, and will be prompted for your password:
+
+  $0 $github_repo $event_type --user <your-github-username>
+
+Be sure to enter a personal access token¹ as your password since GitHub has
+discontinued password authentication to the API starting on November 13, 2020².
+
+You can also store your credentials or a personal access token in a netrc
+file³:
+
+  machine api.github.com
+  login <your-username>
+  password <your-token>
+
+and then tell curl to use it:
+
+  $0 $github_repo $event_type --netrc
+
+which will then not require you to type your password every time.
+
+¹ https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
+² https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password
+³ https://ec.haxx.se/usingcurl/usingcurl-netrc
+.
+    exit 1
+fi
+
+auth=':'
+if [[ -n $PAT_GITHUB_DISPATCH ]]; then
+  auth="Authorization: Bearer ${PAT_GITHUB_DISPATCH}"
+fi
+
+if curl -fsS "https://api.github.com/repos/${github_repo}/dispatches" \
+    -H 'Accept: application/vnd.github.v3+json' \
+    -H 'Content-Type: application/json' \
+    -H "$auth" \
+    -d '{"event_type":"'"$event_type"'"}' \
+    "$@"
+then
+    echo "Successfully triggered $event_type"
+else
+    echo "Request failed" >&2
+    exit 1
+fi
diff --git a/shared/vendored/scripts/trigger-on-new-data b/shared/vendored/scripts/trigger-on-new-data
new file mode 100755
index 0000000..470d2f4
--- /dev/null
+++ b/shared/vendored/scripts/trigger-on-new-data
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${PAT_GITHUB_DISPATCH:?The PAT_GITHUB_DISPATCH environment variable is required.}"
+
+bin="$(dirname "$0")"
+
+github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
+event_type="${2:?An event type is required as the second argument.}"
+metadata="${3:?A metadata upload output file is required as the third argument.}"
+sequences="${4:?An sequence FASTA upload output file is required as the fourth argument.}"
+identical_file_message="${5:-files are identical}"
+
+new_metadata=$(grep "$identical_file_message" "$metadata" >/dev/null; echo $?)
+new_sequences=$(grep "$identical_file_message" "$sequences" >/dev/null; echo $?)
+
+slack_message=""
+
+# grep exit status 0 for found match, 1 for no match, 2 if an error occurred
+if [[ $new_metadata -eq 1 || $new_sequences -eq 1 ]]; then
+    slack_message="Triggering new builds due to updated metadata and/or sequences"
+    "$bin"/trigger "$github_repo" "$event_type"
+elif [[ $new_metadata -eq 0 && $new_sequences -eq 0 ]]; then
+    slack_message="Skipping trigger of rebuild: Both metadata TSV and sequences FASTA are identical to S3 files."
+else
+    slack_message="Skipping trigger of rebuild: Unable to determine if data has been updated."
+fi
+
+
+if ! "$bin"/notify-slack "$slack_message"; then
+    echo "Notifying Slack failed, but exiting with success anyway."
+fi
diff --git a/shared/vendored/scripts/upload-to-s3 b/shared/vendored/scripts/upload-to-s3
new file mode 100755
index 0000000..36d171c
--- /dev/null
+++ b/shared/vendored/scripts/upload-to-s3
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+bin="$(dirname "$0")"
+
+main() {
+    local quiet=0
+
+    for arg; do
+        case "$arg" in
+            --quiet)
+                quiet=1
+                shift;;
+            *)
+                break;;
+        esac
+    done
+
+    local src="${1:?A source file is required as the first argument.}"
+    local dst="${2:?A destination s3:// URL is required as the second argument.}"
+    local cloudfront_domain="${3:-}"
+
+    local s3path="${dst#s3://}"
+    local bucket="${s3path%%/*}"
+    local key="${s3path#*/}"
+
+    local src_hash dst_hash no_hash=0000000000000000000000000000000000000000000000000000000000000000
+    src_hash="$("$bin/sha256sum" < "$src")"
+    dst_hash="$(aws s3api head-object --bucket "$bucket" --key "$key" --query Metadata.sha256sum --output text 2>/dev/null || echo "$no_hash")"
+
+    if [[ $src_hash != "$dst_hash" ]]; then
+        # The record count may have changed
+        src_record_count="$(wc -l < "$src")"
+
+        echo "Uploading $src → $dst"
+        if [[ "$dst" == *.gz ]]; then
+            gzip -c "$src"
+        elif  [[ "$dst" == *.xz ]]; then
+            xz -2 -T0 -c "$src"
+        elif [[ "$dst" == *.zst ]]; then
+            zstd -T0 -c "$src"
+        else
+            cat "$src"
+        fi | aws s3 cp --no-progress - "$dst" --metadata sha256sum="$src_hash",recordcount="$src_record_count" "$(content-type "$dst")"
+
+        if [[ -n $cloudfront_domain ]]; then
+            echo "Creating CloudFront invalidation for $cloudfront_domain/$key"
+            if ! "$bin"/cloudfront-invalidate "$cloudfront_domain" "/$key"; then
+                echo "CloudFront invalidation failed, but exiting with success anyway."
+            fi
+        fi
+
+        if [[ $quiet == 1 ]]; then
+            echo "Quiet mode. No Slack notification sent."
+            exit 0
+        fi
+
+        if ! "$bin"/notify-slack "Updated $dst available."; then
+            echo "Notifying Slack failed, but exiting with success anyway."
+        fi
+    else
+        echo "Uploading $src → $dst: files are identical, skipping upload"
+    fi
+}
+
+content-type() {
+    case "$1" in
+        *.tsv)      echo --content-type=text/tab-separated-values;;
+        *.csv)      echo --content-type=text/comma-separated-values;;
+        *.ndjson)   echo --content-type=application/x-ndjson;;
+        *.gz)       echo --content-type=application/gzip;;
+        *.xz)       echo --content-type=application/x-xz;;
+        *.zst)      echo --content-type=application/zstd;;
+        *)          echo --content-type=text/plain;;
+    esac
+}
+
+main "$@"
diff --git a/shared/vendored/snakemake/config.smk b/shared/vendored/snakemake/config.smk
new file mode 100644
index 0000000..c6217b5
--- /dev/null
+++ b/shared/vendored/snakemake/config.smk
@@ -0,0 +1,77 @@
+"""
+Shared functions to be used within a Snakemake workflow for parsing
+workflow configs.
+"""
+import os.path
+from collections.abc import Callable
+from snakemake.io import Wildcards
+from typing import Optional
+from textwrap import dedent, indent
+
+
+class InvalidConfigError(Exception):
+    pass
+
+
+def resolve_config_path(path: str, defaults_dir: Optional[str] = None) -> Callable[[Wildcards], str]:
+    """
+    Resolve a relative *path* given in a configuration value. Will always try to
+    resolve *path* after expanding wildcards with Snakemake's `expand` functionality.
+
+    Returns the path for the first existing file, checked in the following order:
+    1. relative to the analysis directory or workdir, usually given by ``--directory`` (``-d``)
+    2. relative to *defaults_dir* if it's provided
+    3. relative to the workflow's ``defaults/`` directory if *defaults_dir* is _not_ provided
+
+    This behaviour allows a default configuration value to point to a default
+    auxiliary file while also letting the file used be overridden either by
+    setting an alternate file path in the configuration or by creating a file
+    with the conventional name in the workflow's analysis directory.
+    """
+    global workflow
+
+    def _resolve_config_path(wildcards):
+        try:
+            expanded_path = expand(path, **wildcards)[0]
+        except snakemake.exceptions.WildcardError as e:
+            available_wildcards = "\n".join(f"  - {wildcard}" for wildcard in wildcards)
+            raise snakemake.exceptions.WildcardError(indent(dedent(f"""\
+                {str(e)}
+
+                However, resolve_config_path({{path}}) requires the wildcard.
+
+                Wildcards available for this path are:
+
+                {{available_wildcards}}
+
+                Hint: Check that the config path value does not misspell the wildcard name
+                and that the rule actually uses the wildcard name.
+                """.lstrip("\n").rstrip()).format(path=repr(path), available_wildcards=available_wildcards), " " * 4))
+
+        if os.path.exists(expanded_path):
+            return expanded_path
+
+        if defaults_dir:
+            defaults_path = os.path.join(defaults_dir, expanded_path)
+        else:
+            # Special-case defaults/… for backwards compatibility with older
+            # configs.  We could achieve the same behaviour with a symlink
+            # (defaults/defaults → .) but that seems less clear.
+            if path.startswith("defaults/"):
+                defaults_path = os.path.join(workflow.basedir, expanded_path)
+            else:
+                defaults_path = os.path.join(workflow.basedir, "defaults", expanded_path)
+
+        if os.path.exists(defaults_path):
+            return defaults_path
+
+        raise InvalidConfigError(indent(dedent(f"""\
+            Unable to resolve the config-provided path {path!r},
+            expanded to {expanded_path!r} after filling in wildcards.
+            The workflow does not include the default file {defaults_path!r}.
+
+            Hint: Check that the file {expanded_path!r} exists in your analysis
+            directory or remove the config param to use the workflow defaults.
+            """), " " * 4))
+
+    return _resolve_config_path
diff --git a/shared/vendored/snakemake/remote_files.smk b/shared/vendored/snakemake/remote_files.smk
new file mode 100644
index 0000000..844f80e
--- /dev/null
+++ b/shared/vendored/snakemake/remote_files.smk
@@ -0,0 +1,159 @@
+"""
+Helper functions to set-up storage plugins for remote inputs/outputs. See the
+docstring of `path_or_url` for usage instructions.
+
+The errors raised by storage plugins are often confusing. For instance, a HTTP
+404 error will result in a `MissingInputException` with little hint as to the
+underlying issue. S3 credentials errors are similarly confusing and we attempt
+to check these ourselves to improve UX here.
+"""
+
+from urllib.parse import urlparse
+
+# Keep a list of known public buckets, which we'll allow uncredentialled (unsigned) access to
+# We could make this config-definable in the future
+PUBLIC_BUCKETS = set(['nextstrain-data'])
+
+# Keep track of registered storage plugins to enable reuse
+_storage_registry = {}
+
+class RemoteFilesMissingCredentials(Exception):
+    pass
+
+def _storage_s3(*, bucket, keep_local, retries) -> snakemake.storage.StorageProviderProxy:
+    """
+    Registers and returns an instance of snakemake-storage-plugin-s3. Typically AWS
+    credentials are required for _any_ request however we allow requests to known
+    public buckets (see `PUBLIC_BUCKETS`) to be unsigned which allows for a nice user
+    experience in the common case of downloading inputs from s3://nextstrain-data.
+
+    The intended behaviour for various (S3) URIs supplied to `path_or_url` is:
+
+    |          | S3 buckets                 | credentials present | credentials missing |
+    |----------|----------------------------|---------------------|---------------------|
+    | download | private / private + public | signed              | Credentials Error   |
+    |          | public                     | signed              | unsigned            |
+    | upload   | private / private + public | signed              | Credentials Error   |
+    |          | public                     | signed              | AccessDenied Error  |
+    """
+    # If the bucket is public then we may use an unsigned request which has the nice UX
+    # of not needing credentials to be present. If we've made other signed requests _or_
+    # credentials are present then we just sign everything. This has implications for upload:
+    # if you attempt to upload to a public bucket without credentials then we allow that here
+    # and you'll get a subsequent `AccessDenied` error when the upload is attempted.
+    if bucket in PUBLIC_BUCKETS and \
+        "s3_signed" not in _storage_registry and \
+        ("s3_unsigned" in _storage_registry or not _aws_credentials_present()):
+
+        if provider:=_storage_registry.get('s3_unsigned', None):
+            return provider
+
+        from botocore import UNSIGNED # dependency of snakemake-storage-plugin-s3
+        storage s3_unsigned:
+            provider="s3",
+            signature_version=UNSIGNED,
+            retries=retries,
+            keep_local=keep_local,
+
+        _storage_registry['s3_unsigned'] = storage.s3_unsigned
+        return _storage_registry['s3_unsigned']
+
+    # Resource fetched/uploaded via a signed request, which will require AWS credentials
+    if provider:=_storage_registry.get('s3_signed', None):
+        return provider
+
+    # Enforce the presence of credentials to paper over <https://github.com/snakemake/snakemake/issues/3663>
+    if not _aws_credentials_present():
+        raise RemoteFilesMissingCredentials()
+
+    # the tag appears in the local file path, so reference 'signed' to give a hint about credential errors
+    storage s3_signed:
+        provider="s3",
+        retries=retries,
+        keep_local=keep_local,
+
+    _storage_registry['s3_signed'] = storage.s3_signed
+    return _storage_registry['s3_signed']
+
+def _aws_credentials_present() -> bool:
+    import boto3 # dependency of snakemake-storage-plugin-s3
+    session = boto3.Session()
+    creds = session.get_credentials()
+    return creds is not None
+
+def _storage_http(*, keep_local, retries) -> snakemake.storage.StorageProviderProxy:
+    """
+    Registers and returns an instance of snakemake-storage-plugin-http
+    """
+    if provider:=_storage_registry.get('http', None):
+        return provider
+
+    storage:
+        provider="http",
+        allow_redirects=True,
+        supports_head=True,
+        keep_local=keep_local,
+        retries=retries,
+
+    _storage_registry['http'] = storage.http
+    return _storage_registry['http']
+
+
+def path_or_url(uri, *, keep_local=True, retries=2) -> str:
+    """
+    Intended for use in Snakemake inputs / outputs to transparently use remote
+    resources. Returns the URI wrapped by an applicable storage plugin. Local
+    filepaths will be returned unchanged.
+
+    For example, the following rule will download inputs from HTTPs and upload
+    the output to S3:
+
+        rule filter:
+            input:
+                sequences = path_or_url("https://data.nextstrain.org/..."),
+                metadata = path_or_url("https://data.nextstrain.org/..."),
+            output:
+                sequences = path_or_url("s3://...")
+            shell:
+                r'''
+                augur filter \
+                    --sequences {input.sequences:q} \
+                    --metadata {input.metadata:q} \
+                    --metadata-id-columns accession \
+                    --output-sequences {output.sequences:q}
+                '''
+
+    If *keep_local* is True (the default) then downloaded/uploaded files will
+    remain in `.snakemake/storage/`. The presence of a previously downloaded
+    file (via `keep_local=True`) does not guarantee that the file will not be
+    re-downloaded if the storage plugin decides the local file is out of date.
+
+    Depending on the *uri* authentication may be required. See the specific
+    helper functions (such as `_storage_s3`) for more details.
+
+    See <https://snakemake.readthedocs.io/en/stable/snakefiles/storage.html> for
+    more information on Snakemake storage plugins. Note: various snakemake
+    plugins will be required depending on the URIs provided.
+    """
+    info = urlparse(uri)
+
+    if info.scheme=='': # local
+        return uri      # no storage wrapper
+
+    if info.scheme=='s3':
+        try:
+            return _storage_s3(bucket=info.netloc, keep_local=keep_local, retries=retries)(uri)
+        except RemoteFilesMissingCredentials as e:
+            raise Exception(f"AWS credentials are required to access {uri!r}") from e
+
+    if info.scheme=='https':
+        return _storage_http(keep_local=keep_local, retries=retries)(uri)
+    elif info.scheme=='http':
+        raise Exception(f"HTTP remote file support is not implemented in nextstrain workflows (attempting to access {uri!r}).\n"
+            "Please use an HTTPS address instead.")
+
+    if info.scheme in ['gs', 'gcs']:
+        raise Exception(f"Google Storage is not yet implemented for nextstrain workflows (attempting to access {uri!r}).\n"
+            "Please get in touch if you require this functionality and we can add it to our workflows")
+
+    raise Exception(f"Input address {uri!r} (scheme={info.scheme!r}) is from a non-supported remote")

From 9c652362a868422d23d5be0bb7cf11be07f93901 Mon Sep 17 00:00:00 2001
From: Victor Lin <vlin@fredhutch.org>
Date: Mon, 8 Sep 2025 16:08:20 -0700
Subject: [PATCH 4/5] Switch from ingest/vendored to shared/vendored

---
 .../nextstrain-automation/upload.smk          |   2 +-
 ingest/vendored/.github/dependabot.yml        |  17 --
 .../vendored/.github/pull_request_template.md |  16 --
 ingest/vendored/.github/workflows/ci.yaml     |  15 --
 .../.github/workflows/pre-commit.yaml         |  14 --
 ingest/vendored/.gitrepo                      |  12 --
 ingest/vendored/.pre-commit-config.yaml       |  40 -----
 ingest/vendored/.shellcheckrc                 |   6 -
 ingest/vendored/README.md                     | 148 ------------------
 ingest/vendored/cloudfront-invalidate         |  42 -----
 ingest/vendored/download-from-s3              |  48 ------
 ingest/vendored/fetch-from-ncbi-entrez        |  70 ---------
 ingest/vendored/notify-on-diff                |  35 -----
 ingest/vendored/notify-on-job-fail            |  23 ---
 ingest/vendored/notify-on-job-start           |  27 ----
 ingest/vendored/notify-on-record-change       |  53 -------
 ingest/vendored/notify-slack                  |  56 -------
 ingest/vendored/s3-object-exists              |   8 -
 ingest/vendored/sha256sum                     |  15 --
 ingest/vendored/trigger                       |  56 -------
 ingest/vendored/trigger-on-new-data           |  32 ----
 ingest/vendored/upload-to-s3                  |  78 ---------
 22 files changed, 1 insertion(+), 812 deletions(-)
 delete mode 100644 ingest/vendored/.github/dependabot.yml
 delete mode 100644 ingest/vendored/.github/pull_request_template.md
 delete mode 100644 ingest/vendored/.github/workflows/ci.yaml
 delete mode 100644 ingest/vendored/.github/workflows/pre-commit.yaml
 delete mode 100644 ingest/vendored/.gitrepo
 delete mode 100644 ingest/vendored/.pre-commit-config.yaml
 delete mode 100644 ingest/vendored/.shellcheckrc
 delete mode 100644 ingest/vendored/README.md
 delete mode 100755 ingest/vendored/cloudfront-invalidate
 delete mode 100755 ingest/vendored/download-from-s3
 delete mode 100755 ingest/vendored/fetch-from-ncbi-entrez
 delete mode 100755 ingest/vendored/notify-on-diff
 delete mode 100755 ingest/vendored/notify-on-job-fail
 delete mode 100755 ingest/vendored/notify-on-job-start
 delete mode 100755 ingest/vendored/notify-on-record-change
 delete mode 100755 ingest/vendored/notify-slack
 delete mode 100755 ingest/vendored/s3-object-exists
 delete mode 100755 ingest/vendored/sha256sum
 delete mode 100755 ingest/vendored/trigger
 delete mode 100755 ingest/vendored/trigger-on-new-data
 delete mode 100755 ingest/vendored/upload-to-s3

diff --git a/ingest/build-configs/nextstrain-automation/upload.smk b/ingest/build-configs/nextstrain-automation/upload.smk
index b5549eb..a38801e 100644
--- a/ingest/build-configs/nextstrain-automation/upload.smk
+++ b/ingest/build-configs/nextstrain-automation/upload.smk
@@ -29,7 +29,7 @@ rule upload_to_s3:
         cloudfront_domain=config["cloudfront_domain"],
     shell:
         """
-        ./vendored/upload-to-s3 \
+        {workflow.basedir}/../shared/vendored/scripts/upload-to-s3 \
             {params.quiet} \
             {input.file_to_upload:q} \
             {params.s3_dst:q}/{wildcards.remote_file:q} \
diff --git a/ingest/vendored/.github/dependabot.yml b/ingest/vendored/.github/dependabot.yml
deleted file mode 100644
index 89bd084..0000000
--- a/ingest/vendored/.github/dependabot.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-# Dependabot configuration file
-# <https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file>
-#
-# Each ecosystem is checked on a scheduled interval defined below.  To trigger
-# a check manually, go to
-#
-#   https://github.com/nextstrain/ingest/network/updates
-#
-# and look for a "Check for updates" button.  You may need to click around a
-# bit first.
----
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"
diff --git a/ingest/vendored/.github/pull_request_template.md b/ingest/vendored/.github/pull_request_template.md
deleted file mode 100644
index ed4a5b2..0000000
--- a/ingest/vendored/.github/pull_request_template.md
+++ /dev/null
@@ -1,16 +0,0 @@
-### Description of proposed changes
-
-<!-- What is the goal of this pull request? What does this pull request change? -->
-
-### Related issue(s)
-
-<!-- Link any related issues here. -->
-
-### Checklist
-
-<!-- Make sure checks are successful at the bottom of the PR. -->
-
-- [ ] Checks pass
-- [ ] If adding a script, add an entry for it in the README.
-
-<!-- 🙌 Thank you for contributing to Nextstrain! ✨ -->
diff --git a/ingest/vendored/.github/workflows/ci.yaml b/ingest/vendored/.github/workflows/ci.yaml
deleted file mode 100644
index c716277..0000000
--- a/ingest/vendored/.github/workflows/ci.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  shellcheck:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: nextstrain/.github/actions/shellcheck@master
diff --git a/ingest/vendored/.github/workflows/pre-commit.yaml b/ingest/vendored/.github/workflows/pre-commit.yaml
deleted file mode 100644
index 70da533..0000000
--- a/ingest/vendored/.github/workflows/pre-commit.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: pre-commit
-
-on:
-  - push
-
-jobs:
-  pre-commit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - uses: pre-commit/action@v3.0.1
diff --git a/ingest/vendored/.gitrepo b/ingest/vendored/.gitrepo
deleted file mode 100644
index 3f0f3da..0000000
--- a/ingest/vendored/.gitrepo
+++ /dev/null
@@ -1,12 +0,0 @@
-; DO NOT EDIT (unless you know what you are doing)
-;
-; This subdirectory is a git "subrepo", and this file is maintained by the
-; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme
-;
-[subrepo]
-	remote = https://github.com/nextstrain/ingest
-	branch = main
-	commit = 258ab8ce898a88089bc88caee336f8d683a0e79a
-	parent = 6ee5db50d2506315b264a8ffba657039eceb379e
-	method = merge
-	cmdver = 0.4.6
diff --git a/ingest/vendored/.pre-commit-config.yaml b/ingest/vendored/.pre-commit-config.yaml
deleted file mode 100644
index 2cdf88b..0000000
--- a/ingest/vendored/.pre-commit-config.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-default_language_version:
-  python: python3
-repos:
-  - repo: https://github.com/pre-commit/sync-pre-commit-deps
-    rev: v0.0.1
-    hooks:
-      - id: sync-pre-commit-deps
-  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
-    hooks:
-      - id: shellcheck
-  - repo: https://github.com/rhysd/actionlint
-    rev: v1.6.27
-    hooks:
-      - id: actionlint
-        entry: env SHELLCHECK_OPTS='--exclude=SC2027' actionlint
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-      - id: trailing-whitespace
-      - id: check-ast
-      - id: check-case-conflict
-      - id: check-docstring-first
-      - id: check-json
-      - id: check-executables-have-shebangs
-      - id: check-merge-conflict
-      - id: check-shebang-scripts-are-executable
-      - id: check-symlinks
-      - id: check-toml
-      - id: check-yaml
-      - id: destroyed-symlinks
-      - id: detect-private-key
-      - id: end-of-file-fixer
-      - id: fix-byte-order-marker
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.4.6
-    hooks:
-      # Run the linter.
-      - id: ruff
diff --git a/ingest/vendored/.shellcheckrc b/ingest/vendored/.shellcheckrc
deleted file mode 100644
index ebed438..0000000
--- a/ingest/vendored/.shellcheckrc
+++ /dev/null
@@ -1,6 +0,0 @@
-# Use of this file requires Shellcheck v0.7.0 or newer.
-#
-# SC2064 - We intentionally want variables to expand immediately within traps
-#          so the trap can not fail due to variable interpolation later.
-#
-disable=SC2064
diff --git a/ingest/vendored/README.md b/ingest/vendored/README.md
deleted file mode 100644
index a2b54cb..0000000
--- a/ingest/vendored/README.md
+++ /dev/null
@@ -1,148 +0,0 @@
-# ingest
-
-Shared internal tooling for pathogen data ingest.  Used by our individual
-pathogen repos which produce Nextstrain builds.  Expected to be vendored by
-each pathogen repo using `git subrepo`.
-
-Some tools may only live here temporarily before finding a permanent home in
-`augur curate` or Nextstrain CLI.  Others may happily live out their days here.
-
-## Vendoring
-
-Nextstrain maintained pathogen repos will use [`git subrepo`](https://github.com/ingydotnet/git-subrepo) to vendor ingest scripts.
-(See discussion on this decision in https://github.com/nextstrain/ingest/issues/3)
-
-For a list of Nextstrain repos that are currently using this method, use [this
-GitHub code search](https://github.com/search?type=code&q=org%3Anextstrain+subrepo+%22remote+%3D+https%3A%2F%2Fgithub.com%2Fnextstrain%2Fingest%22).
-
-If you don't already have `git subrepo` installed, follow the [git subrepo installation instructions](https://github.com/ingydotnet/git-subrepo#installation).
-Then add the latest ingest scripts to the pathogen repo by running:
-
-```
-git subrepo clone https://github.com/nextstrain/ingest ingest/vendored
-```
-
-Any future updates of ingest scripts can be pulled in with:
-
-```
-git subrepo pull ingest/vendored
-```
-
-If you run into merge conflicts and would like to pull in a fresh copy of the
-latest ingest scripts, pull with the `--force` flag:
-
-```
-git subrepo pull ingest/vendored --force
-```
-
-> **Warning**
-> Beware of rebasing/dropping the parent commit of a `git subrepo` update
-
-`git subrepo` relies on metadata in the `ingest/vendored/.gitrepo` file,
-which includes the hash for the parent commit in the pathogen repos.
-If this hash no longer exists in the commit history, there will be errors when
-running future `git subrepo pull` commands.
-
-If you run into an error similar to the following:
-```
-$ git subrepo pull ingest/vendored
-git-subrepo: Command failed: 'git branch subrepo/ingest/vendored '.
-fatal: not a valid object name: ''
-```
-Check the parent commit hash in the `ingest/vendored/.gitrepo` file and make
-sure the commit exists in the commit history. Update to the appropriate parent
-commit hash if needed.
-
-## History
-
-Much of this tooling originated in
-[ncov-ingest](https://github.com/nextstrain/ncov-ingest) and was passaged thru
-[mpox's ingest/](https://github.com/nextstrain/mpox/tree/@/ingest/). It
-subsequently proliferated from [mpox][] to other pathogen repos ([rsv][],
-[zika][], [dengue][], [hepatitisB][], [forecasts-ncov][]) primarily thru
-copying.  To [counter that
-proliferation](https://bedfordlab.slack.com/archives/C7SDVPBLZ/p1688577879947079),
-this repo was made.
-
-[mpox]: https://github.com/nextstrain/mpox
-[rsv]: https://github.com/nextstrain/rsv
-[zika]: https://github.com/nextstrain/zika/pull/24
-[dengue]: https://github.com/nextstrain/dengue/pull/10
-[hepatitisB]: https://github.com/nextstrain/hepatitisB
-[forecasts-ncov]: https://github.com/nextstrain/forecasts-ncov
-
-## Elsewhere
-
-The creation of this repo, in both the abstract and concrete, and the general
-approach to "ingest" has been discussed in various internal places, including:
-
-- https://github.com/nextstrain/private/issues/59
-- @joverlee521's [workflows document](https://docs.google.com/document/d/1rLWPvEuj0Ayc8MR0O1lfRJZfj9av53xU38f20g8nU_E/edit#heading=h.4g0d3mjvb89i)
-- [5 July 2023 Slack thread](https://bedfordlab.slack.com/archives/C7SDVPBLZ/p1688577879947079)
-- [6 July 2023 team meeting](https://docs.google.com/document/d/1FPfx-ON5RdqL2wyvODhkrCcjgOVX3nlXgBwCPhIEsco/edit)
-- _…many others_
-
-## Scripts
-
-Scripts for supporting ingest workflow automation that don’t really belong in any of our existing tools.
-
-- [notify-on-diff](notify-on-diff) - Send Slack message with diff of a local file and an S3 object
-- [notify-on-job-fail](notify-on-job-fail) - Send Slack message with details about failed workflow job on GitHub Actions and/or AWS Batch
-- [notify-on-job-start](notify-on-job-start) - Send Slack message with details about workflow job on GitHub Actions and/or AWS Batch
-- [notify-on-record-change](notify-on-recod-change) - Send Slack message with details about line count changes for a file compared to an S3 object's metadata `recordcount`.
-  If the S3 object's metadata does not have `recordcount`, then will attempt to download S3 object to count lines locally, which only supports `xz` compressed S3 objects.
-- [notify-slack](notify-slack) - Send message or file to Slack
-- [s3-object-exists](s3-object-exists) - Used to prevent 404 errors during S3 file comparisons in the notify-* scripts
-- [trigger](trigger) - Triggers downstream GitHub Actions via the GitHub API using repository_dispatch events.
-- [trigger-on-new-data](trigger-on-new-data) - Triggers downstream GitHub Actions if the provided `upload-to-s3` outputs do not contain the `identical_file_message`
-  A hacky way to ensure that we only trigger downstream phylogenetic builds if the S3 objects have been updated.
-
-NCBI interaction scripts that are useful for fetching public metadata and sequences.
-
-- [fetch-from-ncbi-entrez](fetch-from-ncbi-entrez) - Fetch metadata and nucleotide sequences from [NCBI Entrez](https://www.ncbi.nlm.nih.gov/books/NBK25501/) and output to a GenBank file.
-  Useful for pathogens with metadata and annotations in custom fields that are not part of the standard [NCBI Datasets](https://www.ncbi.nlm.nih.gov/datasets/) outputs.
-
-Historically, some pathogen repos used the undocumented NCBI Virus API through [fetch-from-ncbi-virus](https://github.com/nextstrain/ingest/blob/c97df238518171c2b1574bec0349a55855d1e7a7/fetch-from-ncbi-virus) to fetch data. However we've opted to drop the NCBI Virus scripts due to https://github.com/nextstrain/ingest/issues/18.
-
-Potential Nextstrain CLI scripts
-
-- [sha256sum](sha256sum) - Used to check if files are identical in upload-to-s3 and download-from-s3 scripts.
-- [cloudfront-invalidate](cloudfront-invalidate) - CloudFront invalidation is already supported in the [nextstrain remote command for S3 files](https://github.com/nextstrain/cli/blob/a5dda9c0579ece7acbd8e2c32a4bbe95df7c0bce/nextstrain/cli/remote/s3.py#L104).
-  This exists as a separate script to support CloudFront invalidation when using the upload-to-s3 script.
-- [upload-to-s3](upload-to-s3) - Upload file to AWS S3 bucket with compression based on file extension in S3 URL.
-  Skips upload if the local file's hash is identical to the S3 object's metadata `sha256sum`.
-  Adds the following user defined metadata to uploaded S3 object:
-    - `sha256sum` - hash of the file generated by [sha256sum](sha256sum)
-    - `recordcount` - the line count of the file
-- [download-from-s3](download-from-s3) - Download file from AWS S3 bucket with decompression based on file extension in S3 URL.
-  Skips download if the local file already exists and has a hash identical to the S3 object's metadata `sha256sum`.
-
-## Software requirements
-
-Some scripts may require Bash ≥4. If you are running these scripts on macOS, the builtin Bash (`/bin/bash`) does not meet this requirement. You can install [Homebrew's Bash](https://formulae.brew.sh/formula/bash) which is more up to date.
-
-## Testing
-
-Most scripts are untested within this repo, relying on "testing in production". That is the only practical testing option for some scripts such as the ones interacting with S3 and Slack.
-
-## Working on this repo
-
-This repo is configured to use [pre-commit](https://pre-commit.com),
-to help automatically catch common coding errors and syntax issues
-with changes before they are committed to the repo.
-
-If you will be writing new code or otherwise working within this repo,
-please do the following to get started:
-
-1. [install `pre-commit`](https://pre-commit.com/#install) by running
-   either `python -m pip install pre-commit` or `brew install
-   pre-commit`, depending on your preferred package management
-   solution
-2. install the local git hooks by running `pre-commit install` from
-   the root of the repo
-3. when problems are detected, correct them in your local working tree
-   before committing them.
-
-Note that these pre-commit checks are also run in a GitHub Action when
-changes are pushed to GitHub, so correcting issues locally will
-prevent extra cycles of correction.
diff --git a/ingest/vendored/cloudfront-invalidate b/ingest/vendored/cloudfront-invalidate
deleted file mode 100755
index dbea398..0000000
--- a/ingest/vendored/cloudfront-invalidate
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Originally from @tsibley's gist: https://gist.github.com/tsibley/a66262d341dedbea39b02f27e2837ea8
-set -euo pipefail
-
-main() {
-    local domain="$1"
-    shift
-    local paths=("$@")
-    local distribution invalidation
-
-    echo "-> Finding CloudFront distribution"
-    distribution=$(
-        aws cloudfront list-distributions \
-            --query "DistributionList.Items[?contains(Aliases.Items, \`$domain\`)] | [0].Id" \
-            --output text
-    )
-
-    if [[ -z $distribution || $distribution == None ]]; then
-        exec >&2
-        echo "Unable to find CloudFront distribution id for $domain"
-        echo
-        echo "Are your AWS CLI credentials for the right account?"
-        exit 1
-    fi
-
-    echo "-> Creating CloudFront invalidation for distribution $distribution"
-    invalidation=$(
-        aws cloudfront create-invalidation \
-            --distribution-id "$distribution" \
-            --paths "${paths[@]}" \
-            --query Invalidation.Id \
-            --output text
-    )
-
-    echo "-> Waiting for CloudFront invalidation $invalidation to complete"
-    echo "   Ctrl-C to stop waiting."
-    aws cloudfront wait invalidation-completed \
-        --distribution-id "$distribution" \
-        --id "$invalidation"
-}
-
-main "$@"
diff --git a/ingest/vendored/download-from-s3 b/ingest/vendored/download-from-s3
deleted file mode 100755
index 4981186..0000000
--- a/ingest/vendored/download-from-s3
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-bin="$(dirname "$0")"
-
-main() {
-    local src="${1:?A source s3:// URL is required as the first argument.}"
-    local dst="${2:?A destination file path is required as the second argument.}"
-    # How many lines to subsample to. 0 means no subsampling. Optional.
-    # It is not advised to use this for actual subsampling! This is intended to be
-    # used for debugging workflows with large datasets such as ncov-ingest as
-    # described in https://github.com/nextstrain/ncov-ingest/pull/367
-
-    # Uses `tsv-sample` to subsample, so it will not work as expected with files
-    # that have a single record split across multiple lines (i.e. FASTA sequences)
-    local n="${3:-0}"
-
-    local s3path="${src#s3://}"
-    local bucket="${s3path%%/*}"
-    local key="${s3path#*/}"
-
-    local src_hash dst_hash no_hash=0000000000000000000000000000000000000000000000000000000000000000
-    dst_hash="$("$bin/sha256sum" < "$dst" || true)"
-    src_hash="$(aws s3api head-object --bucket "$bucket" --key "$key" --query Metadata.sha256sum --output text 2>/dev/null || echo "$no_hash")"
-
-    echo "[ INFO] Downloading $src → $dst"
-    if [[ $src_hash != "$dst_hash" ]]; then
-        aws s3 cp --no-progress "$src" - |
-        if [[ "$src" == *.gz ]]; then
-            gunzip -cfq
-        elif  [[ "$src" == *.xz ]]; then
-            xz -T0 -dcq
-        elif [[ "$src" == *.zst ]]; then
-            zstd -T0 -dcq
-        else
-            cat
-        fi |
-        if [[ "$n" -gt 0 ]]; then
-            tsv-sample -H -i -n "$n"
-        else
-            cat
-        fi >"$dst"
-    else
-        echo "[ INFO] Files are identical, skipping download"
-    fi
-}
-
-main "$@"
diff --git a/ingest/vendored/fetch-from-ncbi-entrez b/ingest/vendored/fetch-from-ncbi-entrez
deleted file mode 100755
index 194a0c8..0000000
--- a/ingest/vendored/fetch-from-ncbi-entrez
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-"""
-Fetch metadata and nucleotide sequences from NCBI Entrez and output to a GenBank file.
-"""
-import json
-import argparse
-from Bio import SeqIO, Entrez
-
-# To use the efetch API, the docs indicate only around 10,000 records should be fetched per request
-# https://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch
-# However, in my testing with HepB, the max records returned was 9,999
-#   - Jover, 16 August 2023
-BATCH_SIZE = 9999
-
-Entrez.email = "hello@nextstrain.org"
-
-def parse_args():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--term', required=True, type=str,
-        help='Genbank search term. Replace spaces with "+", e.g. "Hepatitis+B+virus[All+Fields]complete+genome[All+Fields]"')
-    parser.add_argument('--output', required=True, type=str, help='Output file (Genbank)')
-    return parser.parse_args()
-
-
-def get_esearch_history(term):
-    """
-    Search for the provided *term* via ESearch and store the results using the
-    Entrez history server.¹
-
-    Returns the total count of returned records, query key, and web env needed
-    to access the records from the server.
-
-    ¹ https://www.ncbi.nlm.nih.gov/books/NBK25497/#chapter2.Using_the_Entrez_History_Server
-    """
-    handle = Entrez.esearch(db="nucleotide", term=term, retmode="json", usehistory="y", retmax=0)
-    esearch_result = json.loads(handle.read())['esearchresult']
-    print(f"Search term {term!r} returned {esearch_result['count']} IDs.")
-    return {
-        "count": int(esearch_result["count"]),
-        "query_key": esearch_result["querykey"],
-        "web_env": esearch_result["webenv"]
-    }
-
-
-def fetch_from_esearch_history(count, query_key, web_env):
-    """
-    Fetch records in batches from Entrez history server using the provided
-    *query_key* and *web_env* and yields them as a BioPython SeqRecord iterator.
-    """
-    print(f"Fetching GenBank records in batches of n={BATCH_SIZE}")
-
-    for start in range(0, count, BATCH_SIZE):
-        handle = Entrez.efetch(
-            db="nucleotide",
-            query_key=query_key,
-            webenv=web_env,
-            retstart=start,
-            retmax=BATCH_SIZE,
-            rettype="gb",
-            retmode="text")
-
-        yield SeqIO.parse(handle, "genbank")
-
-
-if __name__=="__main__":
-    args = parse_args()
-
-    with open(args.output, "w") as output_handle:
-        for batch_results in fetch_from_esearch_history(**get_esearch_history(args.term)):
-            SeqIO.write(batch_results, output_handle, "genbank")
diff --git a/ingest/vendored/notify-on-diff b/ingest/vendored/notify-on-diff
deleted file mode 100755
index ddbe7da..0000000
--- a/ingest/vendored/notify-on-diff
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
-: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
-
-bin="$(dirname "$0")"
-
-src="${1:?A source file is required as the first argument.}"
-dst="${2:?A destination s3:// URL is required as the second argument.}"
-
-dst_local="$(mktemp -t s3-file-XXXXXX)"
-diff="$(mktemp -t diff-XXXXXX)"
-
-trap "rm -f '$dst_local' '$diff'" EXIT
-
-# if the file is not already present, just exit
-"$bin"/s3-object-exists "$dst" || exit 0
-
-"$bin"/download-from-s3 "$dst" "$dst_local"
-
-# diff's exit code is 0 for no differences, 1 for differences found, and >1 for errors
-diff_exit_code=0
-diff "$dst_local" "$src" > "$diff" || diff_exit_code=$?
-
-if [[ "$diff_exit_code" -eq 1 ]]; then
-    echo "Notifying Slack about diff."
-    "$bin"/notify-slack --upload "$src.diff" < "$diff"
-elif [[ "$diff_exit_code" -gt 1 ]]; then
-    echo "Notifying Slack about diff failure"
-    "$bin"/notify-slack "Diff failed for $src"
-else
-    echo "No change in $src."
-fi
diff --git a/ingest/vendored/notify-on-job-fail b/ingest/vendored/notify-on-job-fail
deleted file mode 100755
index 7dd2409..0000000
--- a/ingest/vendored/notify-on-job-fail
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
-: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
-
-: "${AWS_BATCH_JOB_ID:=}"
-: "${GITHUB_RUN_ID:=}"
-
-bin="$(dirname "$0")"
-job_name="${1:?A job name is required as the first argument}"
-github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}"
-
-echo "Notifying Slack about failed ${job_name} job."
-message="❌ ${job_name} job has FAILED 😞 "
-
-if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then
-    message+="See AWS Batch job \`${AWS_BATCH_JOB_ID}\` (<https://console.aws.amazon.com/batch/v2/home?region=us-east-1#jobs/detail/${AWS_BATCH_JOB_ID}|link>) for error details. "
-elif [[ -n "${GITHUB_RUN_ID}" ]]; then
-    message+="See GitHub Action <https://github.com/${github_repo}/actions/runs/${GITHUB_RUN_ID}?check_suite_focus=true|${GITHUB_RUN_ID}> for error details. "
-fi
-
-"$bin"/notify-slack "$message"
diff --git a/ingest/vendored/notify-on-job-start b/ingest/vendored/notify-on-job-start
deleted file mode 100755
index 1c8ce7d..0000000
--- a/ingest/vendored/notify-on-job-start
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
-: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
-
-: "${AWS_BATCH_JOB_ID:=}"
-: "${GITHUB_RUN_ID:=}"
-
-bin="$(dirname "$0")"
-job_name="${1:?A job name is required as the first argument}"
-github_repo="${2:?A GitHub repository with owner and repository name is required as the second argument}"
-build_dir="${3:-ingest}"
-
-echo "Notifying Slack about started ${job_name} job."
-message="${job_name} job has started."
-
-if [[ -n "${GITHUB_RUN_ID}" ]]; then
-  message+=" The job was submitted by GitHub Action <https://github.com/${github_repo}/actions/runs/${GITHUB_RUN_ID}?check_suite_focus=true|${GITHUB_RUN_ID}>."
-fi
-
-if [[ -n "${AWS_BATCH_JOB_ID}" ]]; then
-  message+=" The job was launched as AWS Batch job \`${AWS_BATCH_JOB_ID}\` (<https://console.aws.amazon.com/batch/v2/home?region=us-east-1#jobs/detail/${AWS_BATCH_JOB_ID}|link>)."
-  message+=" Follow along in your local clone of ${github_repo} with: "'```'"nextstrain build --aws-batch --no-download --attach ${AWS_BATCH_JOB_ID} ${build_dir}"'```'
-fi
-
-"$bin"/notify-slack "$message"
diff --git a/ingest/vendored/notify-on-record-change b/ingest/vendored/notify-on-record-change
deleted file mode 100755
index f424252..0000000
--- a/ingest/vendored/notify-on-record-change
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
-: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
-
-bin="$(dirname "$0")"
-
-src="${1:?A source ndjson file is required as the first argument.}"
-dst="${2:?A destination ndjson s3:// URL is required as the second argument.}"
-source_name=${3:?A record source name is required as the third argument.}
-
-# if the file is not already present, just exit
-"$bin"/s3-object-exists "$dst" || exit 0
-
-s3path="${dst#s3://}"
-bucket="${s3path%%/*}"
-key="${s3path#*/}"
-
-src_record_count="$(wc -l < "$src")"
-
-# Try getting record count from S3 object metadata
-dst_record_count="$(aws s3api head-object --bucket "$bucket" --key "$key" --query "Metadata.recordcount || ''" --output text 2>/dev/null || true)"
-if [[ -z "$dst_record_count" ]]; then
-  # This object doesn't have the record count stored as metadata
-  # We have to download it and count the lines locally
-  dst_record_count="$(wc -l < <(aws s3 cp --no-progress "$dst" - | xz -T0 -dcfq))"
-fi
-
-added_records="$(( src_record_count - dst_record_count ))"
-
-printf "%'4d %s\n" "$src_record_count" "$src"
-printf "%'4d %s\n" "$dst_record_count" "$dst"
-printf "%'4d added records\n" "$added_records"
-
-slack_message=""
-
-if [[ $added_records -gt 0 ]]; then
-    echo "Notifying Slack about added records (n=$added_records)"
-    slack_message="📈 New records (n=$added_records) found on $source_name."
-
-elif [[ $added_records -lt 0 ]]; then
-    echo "Notifying Slack about fewer records (n=$added_records)"
-    slack_message="📉 Fewer records (n=$added_records) found on $source_name."
-
-else
-    echo "Notifying Slack about same number of records"
-    slack_message="⛔ No new records found on $source_name."
-fi
-
-slack_message+=" (Total record count: $src_record_count)"
-
-"$bin"/notify-slack "$slack_message"
diff --git a/ingest/vendored/notify-slack b/ingest/vendored/notify-slack
deleted file mode 100755
index a343435..0000000
--- a/ingest/vendored/notify-slack
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${SLACK_TOKEN:?The SLACK_TOKEN environment variable is required.}"
-: "${SLACK_CHANNELS:?The SLACK_CHANNELS environment variable is required.}"
-
-upload=0
-output=/dev/null
-thread_ts=""
-broadcast=0
-args=()
-
-for arg; do
-    case "$arg" in
-        --upload)
-            upload=1;;
-        --output=*)
-            output="${arg#*=}";;
-        --thread-ts=*)
-            thread_ts="${arg#*=}";;
-        --broadcast)
-            broadcast=1;;
-        *)
-            args+=("$arg");;
-    esac
-done
-
-set -- "${args[@]}"
-
-text="${1:?Some message text is required.}"
-
-if [[ "$upload" == 1 ]]; then
-    echo "Uploading data to Slack with the message: $text"
-    curl https://slack.com/api/files.upload \
-        --header "Authorization: Bearer $SLACK_TOKEN" \
-        --form-string channels="$SLACK_CHANNELS" \
-        --form-string title="$text" \
-        --form-string filename="$text" \
-        --form-string thread_ts="$thread_ts" \
-        --form file=@/dev/stdin \
-        --form filetype=text \
-        --fail --silent --show-error \
-        --http1.1 \
-        --output "$output"
-else
-    echo "Posting Slack message: $text"
-    curl https://slack.com/api/chat.postMessage \
-        --header "Authorization: Bearer $SLACK_TOKEN" \
-        --form-string channel="$SLACK_CHANNELS" \
-        --form-string text="$text" \
-        --form-string thread_ts="$thread_ts" \
-        --form-string reply_broadcast="$broadcast" \
-        --fail --silent --show-error \
-        --http1.1 \
-        --output "$output"
-fi
diff --git a/ingest/vendored/s3-object-exists b/ingest/vendored/s3-object-exists
deleted file mode 100755
index 679c20a..0000000
--- a/ingest/vendored/s3-object-exists
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-url="${1#s3://}"
-bucket="${url%%/*}"
-key="${url#*/}"
-
-aws s3api head-object --bucket "$bucket" --key "$key" &>/dev/null
diff --git a/ingest/vendored/sha256sum b/ingest/vendored/sha256sum
deleted file mode 100755
index 32d7ef8..0000000
--- a/ingest/vendored/sha256sum
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-"""
-Portable sha256sum utility.
-"""
-from hashlib import sha256
-from sys import stdin
-
-chunk_size = 5 * 1024**2 # 5 MiB
-
-h = sha256()
-
-for chunk in iter(lambda: stdin.buffer.read(chunk_size), b""):
-    h.update(chunk)
-
-print(h.hexdigest())
diff --git a/ingest/vendored/trigger b/ingest/vendored/trigger
deleted file mode 100755
index 586f9cc..0000000
--- a/ingest/vendored/trigger
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${PAT_GITHUB_DISPATCH:=}"
-
-github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
-event_type="${2:?An event type is required as the second argument.}"
-shift 2
-
-if [[ $# -eq 0 && -z $PAT_GITHUB_DISPATCH ]]; then
-    cat >&2 <<.
-You must specify options to curl for your GitHub credentials.  For example, you
-can specify your GitHub username, and will be prompted for your password:
-
-  $0 $github_repo $event_type --user <your-github-username>
-
-Be sure to enter a personal access token¹ as your password since GitHub has
-discontinued password authentication to the API starting on November 13, 2020².
-
-You can also store your credentials or a personal access token in a netrc
-file³:
-
-  machine api.github.com
-  login <your-username>
-  password <your-token>
-
-and then tell curl to use it:
-
-  $0 $github_repo $event_type --netrc
-
-which will then not require you to type your password every time.
-
-¹ https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
-² https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password
-³ https://ec.haxx.se/usingcurl/usingcurl-netrc
-.
-    exit 1
-fi
-
-auth=':'
-if [[ -n $PAT_GITHUB_DISPATCH ]]; then
-  auth="Authorization: Bearer ${PAT_GITHUB_DISPATCH}"
-fi
-
-if curl -fsS "https://api.github.com/repos/${github_repo}/dispatches" \
-    -H 'Accept: application/vnd.github.v3+json' \
-    -H 'Content-Type: application/json' \
-    -H "$auth" \
-    -d '{"event_type":"'"$event_type"'"}' \
-    "$@"
-then
-    echo "Successfully triggered $event_type"
-else
-    echo "Request failed" >&2
-    exit 1
-fi
diff --git a/ingest/vendored/trigger-on-new-data b/ingest/vendored/trigger-on-new-data
deleted file mode 100755
index 470d2f4..0000000
--- a/ingest/vendored/trigger-on-new-data
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-: "${PAT_GITHUB_DISPATCH:?The PAT_GITHUB_DISPATCH environment variable is required.}"
-
-bin="$(dirname "$0")"
-
-github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
-event_type="${2:?An event type is required as the second argument.}"
-metadata="${3:?A metadata upload output file is required as the third argument.}"
-sequences="${4:?An sequence FASTA upload output file is required as the fourth argument.}"
-identical_file_message="${5:-files are identical}"
-
-new_metadata=$(grep "$identical_file_message" "$metadata" >/dev/null; echo $?)
-new_sequences=$(grep "$identical_file_message" "$sequences" >/dev/null; echo $?)
-
-slack_message=""
-
-# grep exit status 0 for found match, 1 for no match, 2 if an error occurred
-if [[ $new_metadata -eq 1 || $new_sequences -eq 1 ]]; then
-    slack_message="Triggering new builds due to updated metadata and/or sequences"
-    "$bin"/trigger "$github_repo" "$event_type"
-elif [[ $new_metadata -eq 0 && $new_sequences -eq 0 ]]; then
-    slack_message="Skipping trigger of rebuild: Both metadata TSV and sequences FASTA are identical to S3 files."
-else
-    slack_message="Skipping trigger of rebuild: Unable to determine if data has been updated."
-fi
-
-
-if ! "$bin"/notify-slack "$slack_message"; then
-    echo "Notifying Slack failed, but exiting with success anyway."
-fi
diff --git a/ingest/vendored/upload-to-s3 b/ingest/vendored/upload-to-s3
deleted file mode 100755
index 36d171c..0000000
--- a/ingest/vendored/upload-to-s3
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-bin="$(dirname "$0")"
-
-main() {
-    local quiet=0
-
-    for arg; do
-        case "$arg" in
-            --quiet)
-                quiet=1
-                shift;;
-            *)
-                break;;
-        esac
-    done
-
-    local src="${1:?A source file is required as the first argument.}"
-    local dst="${2:?A destination s3:// URL is required as the second argument.}"
-    local cloudfront_domain="${3:-}"
-
-    local s3path="${dst#s3://}"
-    local bucket="${s3path%%/*}"
-    local key="${s3path#*/}"
-
-    local src_hash dst_hash no_hash=0000000000000000000000000000000000000000000000000000000000000000
-    src_hash="$("$bin/sha256sum" < "$src")"
-    dst_hash="$(aws s3api head-object --bucket "$bucket" --key "$key" --query Metadata.sha256sum --output text 2>/dev/null || echo "$no_hash")"
-
-    if [[ $src_hash != "$dst_hash" ]]; then
-        # The record count may have changed
-        src_record_count="$(wc -l < "$src")"
-
-        echo "Uploading $src → $dst"
-        if [[ "$dst" == *.gz ]]; then
-            gzip -c "$src"
-        elif  [[ "$dst" == *.xz ]]; then
-            xz -2 -T0 -c "$src"
-        elif [[ "$dst" == *.zst ]]; then
-            zstd -T0 -c "$src"
-        else
-            cat "$src"
-        fi | aws s3 cp --no-progress - "$dst" --metadata sha256sum="$src_hash",recordcount="$src_record_count" "$(content-type "$dst")"
-
-        if [[ -n $cloudfront_domain ]]; then
-            echo "Creating CloudFront invalidation for $cloudfront_domain/$key"
-            if ! "$bin"/cloudfront-invalidate "$cloudfront_domain" "/$key"; then
-                echo "CloudFront invalidation failed, but exiting with success anyway."
-            fi
-        fi
-
-        if [[ $quiet == 1 ]]; then
-            echo "Quiet mode. No Slack notification sent."
-            exit 0
-        fi
-
-        if ! "$bin"/notify-slack "Updated $dst available."; then
-            echo "Notifying Slack failed, but exiting with success anyway."
-        fi
-    else
-        echo "Uploading $src → $dst: files are identical, skipping upload"
-    fi
-}
-
-content-type() {
-    case "$1" in
-        *.tsv)      echo --content-type=text/tab-separated-values;;
-        *.csv)      echo --content-type=text/comma-separated-values;;
-        *.ndjson)   echo --content-type=application/x-ndjson;;
-        *.gz)       echo --content-type=application/gzip;;
-        *.xz)       echo --content-type=application/x-xz;;
-        *.zst)      echo --content-type=application/zstd;;
-        *)          echo --content-type=text/plain;;
-    esac
-}
-
-main "$@"

From d2d19587c882ef196ccb629ef9b92caa7d514b77 Mon Sep 17 00:00:00 2001
From: Victor Lin <vlin@fredhutch.org>
Date: Tue, 9 Sep 2025 13:13:49 -0700
Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=9A=A7=20Support=20nextstrain=20run?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Update all workflows to support nextstrain run. Instead of applying
resolve_config_path() in individual rules, resolve them all at once at
the start of the workflow. This is not strictly necessary, but
consolidates code changes to a single function instead of all rules that
take file paths from config.

🚧: update the Nextclade workflow?

Testing commands:

    mkdir -p ~/.nextstrain/pathogens/wnv
    ln -sv ~/repos/nextstrain/wnv ~/.nextstrain/pathogens/wnv/local=NRXWGYLM
    nextstrain run wnv@local phylogenetic ./tmp

References:
- nextstrain/mumps@6343dd8...0866a88
- nextstrain/zika@5041a36...e4f084d
---
 ingest/Snakefile              | 21 +++++++++++++++------
 nextstrain-pathogen.yaml      | 13 +++++++++----
 phylogenetic/Snakefile        | 12 +++++++-----
 phylogenetic/rules/config.smk | 26 ++++++++++++++++++++++++++
 4 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/ingest/Snakefile b/ingest/Snakefile
index dd74e5b..493fc73 100644
--- a/ingest/Snakefile
+++ b/ingest/Snakefile
@@ -1,8 +1,12 @@
-# The workflow filepaths are written relative to this Snakefile's base directory
-workdir: workflow.current_basedir
+# Utility functions shared across all workflows.
+include: "../shared/vendored/snakemake/config.smk"
 
-# Use default configuration values. Override with Snakemake's --configfile/--config options.
-configfile: "defaults/config.yaml"
+# Use default configuration values. Extend with Snakemake's --configfile/--config options.
+configfile: os.path.join(workflow.basedir, "defaults/config.yaml")
+
+# Use custom configuration from analysis directory (i.e. working dir), if any.
+if os.path.exists("config.yaml"):
+    configfile: "config.yaml"
 
 # This is the default rule that Snakemake will run when there are no specified targets.
 # The default output of the ingest workflow is usually the curated metadata and sequences.
@@ -31,5 +35,10 @@ include: "rules/nextclade.smk"
 # https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#handling-ambiguous-rules
 if "custom_rules" in config:
     for rule_file in config["custom_rules"]:
-
-        include: rule_file
\ No newline at end of file
+        # Relative custom rule paths in the config are relative to the analysis
+        # directory (i.e. the current working directory, or workdir, usually
+        # given by --directory), but the "include" directive treats relative
+        # paths as relative to the workflow (e.g. workflow.current_basedir).
+        # Convert to an absolute path based on the analysis/current directory
+        # to avoid this mismatch of expectations.
+        include: os.path.join(os.getcwd(), rule_file)
diff --git a/nextstrain-pathogen.yaml b/nextstrain-pathogen.yaml
index b74c50d..9a723f0 100644
--- a/nextstrain-pathogen.yaml
+++ b/nextstrain-pathogen.yaml
@@ -1,5 +1,10 @@
-# This is currently an empty file to indicate the top level pathogen repo.
-# The inclusion of this file allows the Nextstrain CLI to run the
-# `nextstrain build` from any directory regardless of runtime.
+# This file's *existence* marks the top level of a Nextstrain pathogen repo,
+# which allows `nextstrain build` to be run from any subdirectory of the repo
+# regardless of runtime.  For more details, see
+# <https://github.com/nextstrain/cli/releases/tag/8.2.0>.
 #
-# See https://github.com/nextstrain/cli/releases/tag/8.2.0 for more details.
+# This file's *contents* is the "registration metadata" for the pathogen repo,
+# used by `nextstrain setup` and `nextstrain run`.
+---
+compatibility:
+  nextstrain run: true
diff --git a/phylogenetic/Snakefile b/phylogenetic/Snakefile
index ed4ab69..3c1f95f 100644
--- a/phylogenetic/Snakefile
+++ b/phylogenetic/Snakefile
@@ -2,11 +2,13 @@
 This is the main phylogenetic Snakefile that orchestrates the full phylogenetic
 workflow and define its default output(s).
 """
-# The workflow filepaths are written relative to this Snakefile's base directory
-workdir: workflow.current_basedir
+# Utility functions shared across all workflows.
+include: "../shared/vendored/snakemake/config.smk"
 
-# Use default configuration values. Override with Snakemake's --configfile/--config options.
-configfile: "defaults/config.yaml"
+configfile: os.path.join(workflow.basedir, "defaults/config.yaml")
+
+if os.path.exists("config.yaml"):
+    configfile: "config.yaml"
 
 include: "rules/config.smk"
 
@@ -65,4 +67,4 @@ rule clean:
 if "custom_rules" in config:
     for rule_file in config["custom_rules"]:
 
-        include: rule_file
+        include: os.path.join(os.getcwd(), rule_file)
diff --git a/phylogenetic/rules/config.smk b/phylogenetic/rules/config.smk
index d3f3d16..5d28829 100644
--- a/phylogenetic/rules/config.smk
+++ b/phylogenetic/rules/config.smk
@@ -14,6 +14,7 @@ from textwrap import dedent
 
 def main():
     validate_config()
+    resolve_config_paths()
     write_config()
 
 
@@ -39,6 +40,31 @@ def validate_config():
         exit(1)
 
 
+def resolve_config_paths():
+    """
+    Update all file paths in config by passing them through resolve_config_path()
+    """
+    global config
+
+    for build_name, build_config in config["build_params"].items():
+        # config.<build>.reference
+        build_config["reference"] = resolve_config_path(build_config["reference"])({})
+
+        # config.<build>.export
+        for key in ["description", "auspice_config"]:
+            build_config["export"][key] = resolve_config_path(build_config["export"][key])({})
+
+        # config.<build>.subsample
+        subsample_path_keys = ["exclude", "include", "group_by_weights"]
+        for sample_name, sample_config in build_config["subsample"].items():
+            for key in subsample_path_keys:
+                if key in sample_config:
+                    if isinstance(sample_config[key], list):
+                        sample_config[key] = [resolve_config_path(path)({}) for path in sample_config[key]]
+                    else:
+                        sample_config[key] = resolve_config_path(sample_config[key])({})
+
+
 def write_config():
     """
     Write Snakemake's 'config' variable to a file.