nextstrain · j23414 · Nov 30, 2022 · Nov 28, 2022 · Dec 2, 2022 · Dec 16, 2022
diff --git a/README.md b/README.md
@@ -16,6 +16,41 @@ housed in this repository.
 
 ## Usage
 
+### Provision input data
+
+Input sequences and metadata can be retrieved from data.nextstrain.org
+
+* [sequences.fasta.xz](https://data.nextstrain.org/files/workflows/ebola/test/sequences.fasta.zst)
+* [metadata.tsv.zst](https://data.nextstrain.org/files/workflows/ebola/test/metadata.tsv.zst)
+
+Note that these data are generously shared by many labs around the world.
+If you analyze and plan to publish using these data, please contact these labs first.
+
+Within the analysis pipeline, these data are fetched from data.nextstrain.org and written to `data/` with:
+
+```bash
+nextstrain build . data/sequences.fasta data/metadata.tsv
+```
+
+### Run analysis pipeline
+
+Run pipeline to produce "overview" tree for `/ebola` with:
+
+```bash
+nextstrain build . 
+```
+
+### Visualize results
+
+View results with:
+
+```bash
+nextstrain view auspice/
+```
+
+
+<!--
+
 If you're unfamiliar with Nextstrain builds, you may want to follow our
 [quickstart guide][] first and then come back here.
 
@@ -36,6 +71,7 @@ Once you've run the build, you can view the results in auspice:
 
     nextstrain view auspice/
 
+-->
 
 ## Configuration
 

diff --git a/Snakefile b/Snakefile
@@ -1,3 +1,6 @@
+if not config:
+    configfile: "config/config_ebola.yaml"
+
 rule all:
     input:
         auspice_json = "auspice/ebola.json"
@@ -16,40 +19,62 @@ rule files:
 files = rules.files.params
 
 rule download:
-    """Downloading sequences from fauna"""
+    """Downloading sequences and metadata from data.nextstrain.org"""
     output:
-        sequences = "data/ebola.fasta"
+        sequences = "data/sequences.fasta.zst",
+        metadata = "data/metadata.tsv.zst",
     params:
-        fasta_fields = "strain virus accession collection_date region country division location source locus authors url title journal puburl"
+        sequences_url = "https://data.nextstrain.org/files/workflows/ebola/test/sequences.fasta.zst",
+        metadata_url = "https://data.nextstrain.org/files/workflows/ebola/test/metadata.tsv.zst",
     shell:
         """
-        python3 ../fauna/vdb/download.py \
-            --database vdb \
-            --virus ebola \
-            --fasta_fields {params.fasta_fields} \
-            --resolve_method choose_genbank \
-            --path $(dirname {output.sequences}) \
-            --fstem $(basename {output.sequences} .fasta)
+        curl -fsSL --compressed {params.sequences_url:q} --output {output.sequences}
+        curl -fsSL --compressed {params.metadata_url:q} --output {output.metadata}
         """
 
-rule parse:
-    """Parsing fasta into sequences and metadata"""
+rule decompress:
+    message: "Decompressing sequences and metadata"
     input:
-        sequences = files.input_fasta
+        sequences = "data/sequences.fasta.zst",
+        metadata = "data/metadata.tsv.zst"
     output:
-        sequences = "results/sequences.fasta",
-        metadata = "results/metadata.tsv"
+        sequences = "data/sequences.fasta",
+        metadata = "data/metadata.tsv",
+    shell:
+        """
+        zstd -d -c {input.sequences} > {output.sequences}
+        zstd -d -c {input.metadata} > {output.metadata}
+        """
+
+rule wrangle_metadata:
+    input:
+        metadata="data/metadata.tsv",
+    output:
+        metadata="results/wrangled_metadata.tsv",
     params:
-        fasta_fields = "strain virus accession date region country division city db segment authors url title journal paper_url",
-        prettify_fields = "region country division city"
+        strain_id=lambda w: config.get("strain_id_field", "strain"),
+        wrangle_metadata_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/scripts/wrangle_metadata.py",
     shell:
         """
-        augur parse \
-            --sequences {input.sequences} \
-            --output-sequences {output.sequences} \
-            --output-metadata {output.metadata} \
-            --fields {params.fasta_fields} \
-            --prettify-fields {params.prettify_fields}
+        # (1) Pick curl or wget based on availability    
+        if which curl > /dev/null; then
+            download_cmd="curl -fsSL --output"
+        elif which wget > /dev/null; then
+            download_cmd="wget -O"
+        else
+            echo "ERROR: Neither curl nor wget found. Please install one of them."
+            exit 1
+        fi
+
+        # (2) Download the required scripts if not already present
+        [[ -d bin ]] || mkdir bin
+        [[ -f bin/wrangle_metadata.py ]] || $download_cmd bin/wrangle_metadata.py {params.wrangle_metadata_url}
+        chmod +x bin/*
+
+        # (3) Run the script
+        python3 ./bin/wrangle_metadata.py --metadata {input.metadata} \
+            --strain-id {params.strain_id} \
+            --output {output.metadata}
         """
 
 rule filter:
@@ -60,8 +85,8 @@ rule filter:
       - excluding strains in {input.exclude}
     """
     input:
-        sequences = "results/sequences.fasta",
-        metadata = "results/metadata.tsv",
+        sequences = "data/sequences.fasta",
+        metadata = "results/wrangled_metadata.tsv",
         include = files.forced_strains,
         exclude = files.dropped_strains
     output:
@@ -129,7 +154,7 @@ rule refine:
     input:
         tree = "results/tree_raw.nwk",
         alignment = "results/aligned.fasta",
-        metadata = "results/metadata.tsv"
+        metadata = "results/wrangled_metadata.tsv"
     output:
         tree = "results/tree.nwk",
         node_data = "results/branch_lengths.json"
@@ -154,7 +179,7 @@ rule ancestral:
     """Reconstructing ancestral sequences and mutations"""
     input:
         tree = "results/tree.nwk",
-        alignment = "results/aligned.fasta"
+        alignment = "results/aligned.fasta",
     output:
         node_data = "results/nt_muts.json"
     params:
@@ -189,7 +214,7 @@ rule traits:
     """Inferring ancestral traits for {params.columns!s}"""
     input:
         tree = "results/tree.nwk",
-        metadata = "results/metadata.tsv"
+        metadata = "results/wrangled_metadata.tsv"
     output:
         node_data = "results/traits.json",
     params:
@@ -208,7 +233,7 @@ rule export:
     """Exporting data files for for auspice"""
     input:
         tree = "results/tree.nwk",
-        metadata = "results/metadata.tsv",
+        metadata = "results/wrangled_metadata.tsv",
         branch_lengths = "results/branch_lengths.json",
         traits = "results/traits.json",
         nt_muts = "results/nt_muts.json",
@@ -218,7 +243,8 @@ rule export:
         auspice_config = files.auspice_config,
         description = files.description
     output:
-        auspice_json = rules.all.input.auspice_json
+        auspice_json = "results/raw_ebola.json",
+        root_sequence="results/raw_ebola_root-sequence.json",
     shell:
         """
         augur export v2 \
@@ -233,6 +259,44 @@ rule export:
             --output {output.auspice_json}
         """
 
+rule final_strain_name:
+    input:
+        auspice_json="results/raw_ebola.json",
+        metadata="results/wrangled_metadata.tsv",
+        root_sequence="results/raw_ebola_root-sequence.json",
+    output:
+        auspice_json="auspice/ebola.json",
+        root_sequence="auspice/ebola_root-sequence.json",
+    params:
+        display_strain_field=lambda w: config.get("display_strain_field", "strain"),
+        set_final_strain_name_url="https://raw.githubusercontent.com/nextstrain/monkeypox/644d07ebe3fa5ded64d27d0964064fb722797c5d/scripts/set_final_strain_name.py",
+    shell:
+        """
+         # (1) Pick curl or wget based on availability
+        if which curl > /dev/null; then
+            download_cmd="curl -fsSL --output"
+        elif which wget > /dev/null; then
+            download_cmd="wget -O"
+        else
+            echo "ERROR: Neither curl nor wget found. Please install one of them."
+            exit 1
+        fi
+
+        # (2) Download the required scripts if not already present
+        [[ -d bin ]] || mkdir bin
+        [[ -f bin/set_final_strain_name.py ]] || $download_cmd bin/set_final_strain_name.py {params.set_final_strain_name_url}
+        chmod +x bin/*
+
+        # (3) Run the script
+        python3 bin/set_final_strain_name.py \
+            --metadata {input.metadata} \
+            --input-auspice-json {input.auspice_json} \
+            --display-strain-name {params.display_strain_field} \
+            --output {output.auspice_json}
+
+        cp {input.root_sequence} {output.root_sequence}
+        """
+
 rule clean:
     """Removing directories: {params}"""
     params:

diff --git a/config/config_ebola.yaml b/config/config_ebola.yaml
@@ -0,0 +1,2 @@
+strain_id_field: "accession"
+display_strain_field: "strain_original"
diff --git a/example_data/metadata.tsv b/example_data/metadata.tsv
@@ -0,0 +1,31 @@
+strain	accession	date	region	country	division	city	authors	url	title
+G5016.1	KR105293	2014-08-16	subsaharan_africa	sierra_leone	kenema	kenema	Goba et al	https://www.ncbi.nlm.nih.gov/nuccore/KR105293	"Ebola virus epidemiology, transmission, and viral evolution from four months of sequencing in Sierra Leone"
+EM_080193	KR817239	2014-06-29	subsaharan_africa	liberia	liberia	liberia	Carroll et al	https://www.ncbi.nlm.nih.gov/nuccore/KR817239	Temporal and spatial analysis of the 2014-2015 Ebola virus outbreak in West Africa
+EM_074351	KR817110	2014-07-22	subsaharan_africa	liberia	lofa	lofa	Carroll et al	https://www.ncbi.nlm.nih.gov/nuccore/KR817110	Temporal and spatial analysis of the 2014-2015 Ebola virus outbreak in West Africa
+LIBR10253	KT725282	2014-09-16	subsaharan_africa	liberia	margibi	margibi	Ladner et al	https://www.ncbi.nlm.nih.gov/nuccore/KT725282	"Evolution and spread of Ebola virus in Liberia, 2014-2015"
+G3848	KM233110	2014-06-18	subsaharan_africa	sierra_leone	kailahun	kailahun	Gire et al	https://www.ncbi.nlm.nih.gov/nuccore/KM233110	Genomic surveillance elucidates Ebola virus origin and transmission during the 2014 outbreak
+EM_000027	KR817068	2014-09-01	subsaharan_africa	guinea	guinea	guinea	Carroll et al	https://www.ncbi.nlm.nih.gov/nuccore/KR817068	Temporal and spatial analysis of the 2014-2015 Ebola virus outbreak in West Africa
+LIBR10047	KT725345	2014-10-01	subsaharan_africa	liberia	margibi	margibi	Ladner et al	https://www.ncbi.nlm.nih.gov/nuccore/KT725345	"Evolution and spread of Ebola virus in Liberia, 2014-2015"
+G4350.1	KR105242	2014-07-14	subsaharan_africa	sierra_leone	kenema	kenema	Goba et al	https://www.ncbi.nlm.nih.gov/nuccore/KR105242	"Ebola virus epidemiology, transmission, and viral evolution from four months of sequencing in Sierra Leone"
+J0165	KP759703	2014-11-08	subsaharan_africa	sierra_leone	western_urban	western_urban	Tong et al	https://www.ncbi.nlm.nih.gov/nuccore/KP759703	Genetic diversity and evolutionary dynamics of Ebola virus in Sierra Leone
+G3926.2	KR105208	2014-06-22	subsaharan_africa	sierra_leone	kailahun	kailahun	Goba et al	https://www.ncbi.nlm.nih.gov/nuccore/KR105208	"Ebola virus epidemiology, transmission, and viral evolution from four months of sequencing in Sierra Leone"
+EM_080223	KR817241	2014-07-04	subsaharan_africa	liberia	liberia	liberia	Carroll et al	https://www.ncbi.nlm.nih.gov/nuccore/KR817241	Temporal and spatial analysis of the 2014-2015 Ebola virus outbreak in West Africa
+G4345.1	KR105239	2014-07-18	subsaharan_africa	sierra_leone	kenema	kenema	Goba et al	https://www.ncbi.nlm.nih.gov/nuccore/KR105239	"Ebola virus epidemiology, transmission, and viral evolution from four months of sequencing in Sierra Leone"
+LIBR10218	KT725284	2014-09-29	subsaharan_africa	liberia	liberia	liberia	Ladner et al	https://www.ncbi.nlm.nih.gov/nuccore/KT725284	"Evolution and spread of Ebola virus in Liberia, 2014-2015"
+G3807	KM233084	2014-06-15	subsaharan_africa	sierra_leone	kailahun	kailahun	Gire et al	https://www.ncbi.nlm.nih.gov/nuccore/KM233084	Genomic surveillance elucidates Ebola virus origin and transmission during the 2014 outbreak
+LIBR10224	KT725302	2014-09-26	subsaharan_africa	liberia	montserrado	montserrado	Ladner et al	https://www.ncbi.nlm.nih.gov/nuccore/KT725302	"Evolution and spread of Ebola virus in Liberia, 2014-2015"
+13031_EMLH	KU296665	2015-05-18	subsaharan_africa	sierra_leone	western_urban	western_urban	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296665	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+G5996.1	KR105337	2014-09-25	subsaharan_africa	sierra_leone	sierra_leone	sierra_leone	Goba et al	https://www.ncbi.nlm.nih.gov/nuccore/KR105337	"Ebola virus epidemiology, transmission, and viral evolution from four months of sequencing in Sierra Leone"
+DML24708	KT357845	2015-01-28	subsaharan_africa	sierra_leone	kono	kono	Smits et al	https://www.ncbi.nlm.nih.gov/nuccore/KT357845	"Hypermutated Ebola virus circulating in Magazine Wharf area, Freetown, Sierra Leone"
+15686_EMLK	KU296555	2015-06-30	subsaharan_africa	sierra_leone	kambia	kambia	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296555	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+EM_COY_2015_016617	EM_COY_2015_016617	2015-05-16	subsaharan_africa	guinea	dubreka	dubreka	Quick et al	https://github.com/nickloman/ebov/	?
+Makona-UK3	KR025228	2015-03-12	subsaharan_africa	sierra_leone	western_area	western_area	Lewandowski et al	https://www.ncbi.nlm.nih.gov/nuccore/KR025228	Direct Submission
+EM_000968	KR817085	2014-10-01	subsaharan_africa	guinea	macenta	macenta	Carroll et al	https://www.ncbi.nlm.nih.gov/nuccore/KR817085	Temporal and spatial analysis of the 2014-2015 Ebola virus outbreak in West Africa
+490_EMLH	KU296746	2015-01-30	subsaharan_africa	sierra_leone	sierra_leone	sierra_leone	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296746	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+PL4696	KU296513	2015-03-13	subsaharan_africa	sierra_leone	port_loko	port_loko	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296513	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+MK2788	KU296573	2015-03-07	subsaharan_africa	sierra_leone	bombali	bombali	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296573	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+LIBR10110	KT725387	2014-08-20	subsaharan_africa	liberia	liberia	liberia	Ladner et al	https://www.ncbi.nlm.nih.gov/nuccore/KT725387	"Evolution and spread of Ebola virus in Liberia, 2014-2015"
+J0102	KP759657	2014-10-27	subsaharan_africa	sierra_leone	port_loko	port_loko	Tong et al	https://www.ncbi.nlm.nih.gov/nuccore/KP759657	Genetic diversity and evolutionary dynamics of Ebola virus in Sierra Leone
+EM_COY_2015_017574	EM_COY_2015_017574	2015-06-10	subsaharan_africa	guinea	boke	boke	Quick et al	https://github.com/nickloman/ebov/	?
+18636_EMLH	KU296460	2015-07-08	subsaharan_africa	sierra_leone	sierra_leone	sierra_leone	Arias et al	https://www.ncbi.nlm.nih.gov/nuccore/KU296460	Rapid outbreak sequencing of Ebola virus in Sierra Leone identifies transmission chains linked to sporadic cases
+V768	KR534588	2014-08-27	subsaharan_africa	guinea	conakry	conakry	Simon-Loriere et al	https://www.ncbi.nlm.nih.gov/nuccore/KR534588	Distinct lineages of Ebola virus in Guinea during the 2014 West African epidemic
diff --git a/example_data/sequences.fasta b/example_data/sequences.fasta
diff --git a/ingest/README.md b/ingest/README.md
@@ -0,0 +1,89 @@
+# nextstrain.org/ebola/ingest
+
+This is the ingest pipeline for Ebola virus sequences.
+
+## Usage
+
+> NOTE: All command examples assume you are within the `ingest` directory.
+> If running commands from the outer `ebola` directory, please replace the `.` with `ingest`
+
+Fetch sequences with
+
+```sh
+nextstrain build --cpus 1 . data/sequences.ndjson
+```
+
+Run the complete ingest pipeline with
+
+```sh
+nextstrain build --cpus 1 .
+```
+
+This will produce two files (within the `ingest` directory):
+
+- data/metadata.tsv
+- data/sequences.fasta
+
+Run the complete ingest pipeline and upload results to AWS S3 with
+
+```sh
+nextstrain build . --configfiles config/config.yaml config/optional.yaml
+```
+
+### Adding new sequences not from GenBank
+
+#### Static Files
+
+Do the following to include sequences from static FASTA files.
+
+1. Convert the FASTA files to NDJSON files with:
+
+    ```sh
+    wget https://raw.githubusercontent.com/nextstrain/monkeypox/master/ingest/bin/fasta-to-ndjson -O bin/fasta-to-ndjson
+    chmod 755 bin/fasta-to-ndjson
+    ./bin/fasta-to-ndjson \
+        --fasta {path-to-fasta-file} \
+        --fields {fasta-header-field-names} \
+        --separator {field-separator-in-header} \
+        --exclude {fields-to-exclude-in-output} \
+        > ingest/data/{file-name}.ndjson
+    ```
+
+2. Add the following to the `.gitignore` to allow the file to be included in the repo:
+
+    ```gitignore
+    !ingest/data/{file-name}.ndjson
+    ```
+
+3. Add the `file-name` (without the `.ndjson` extension) as a source to `ingest/config/config.yaml`. This will tell the ingest pipeline to concatenate the records to the GenBank sequences and run them through the same transform pipeline.
+
+## Configuration
+
+Configuration takes place in `config/config.yaml` by default.
+Optional configs for uploading files and Slack notifications are in `config/optional.yaml`.
+
+### Environment Variables
+
+The complete ingest pipeline with AWS S3 uploads and Slack notifications uses the following environment variables:
+
+#### Required
+
+- `AWS_DEFAULT_REGION`
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `SLACK_TOKEN`
+- `SLACK_CHANNELS`
+
+#### Optional
+
+These are optional environment variables used in our automated pipeline for providing detailed Slack notifications.
+
+- `GITHUB_RUN_ID` - provided via [`github.run_id` in a GitHub Action workflow](https://docs.github.com/en/actions/learn-github-actions/contexts#github-context)
+- `AWS_BATCH_JOB_ID` - provided via [AWS Batch Job environment variables](https://docs.aws.amazon.com/batch/latest/userguide/job_env_vars.html)
+
+## Input data
+
+### GenBank data
+
+GenBank sequences and metadata are fetched via NCBI Virus.
+The exact URL used to fetch data is constructed in `bin/genbank-url`.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		strain_id_field: "accession"
		display_strain_field: "strain_original"