From d9721e10511325c1861470a5e5b1d3541bbe69c2 Mon Sep 17 00:00:00 2001
From: Jennifer Chang <jennifer.chang.bioinform@gmail.com>
Date: Fri, 27 Oct 2023 10:02:15 -0700
Subject: [PATCH] Make s3_dst config to top level config

Nested configs for s3 URLs are challenging to override by the Snakemake
`--config` option, as discussed in the following comment:

https://github.com/nextstrain/dengue/pull/13#discussion_r1361131210

This moves the url to the top level, and propagate those changes to the relevant Snakefiles.
---
 ingest/Snakefile                           | 2 +-
 ingest/config/optional.yaml                | 4 ++--
 ingest/workflow/snakemake_rules/upload.smk | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ingest/Snakefile b/ingest/Snakefile
index 4e829ea4..168dd2a7 100644
--- a/ingest/Snakefile
+++ b/ingest/Snakefile
@@ -32,7 +32,7 @@ def _get_all_targets(wildcards):
             print(
                 f"Skipping file upload for {target!r} because there are duplicate remote file names."
             )
-        elif not params.get("dst"):
+        elif not params.get("s3_dst"):
             print(
                 f"Skipping file upload for {target!r} because the destintion was not defined."
             )
diff --git a/ingest/config/optional.yaml b/ingest/config/optional.yaml
index b361327a..5a3cb028 100644
--- a/ingest/config/optional.yaml
+++ b/ingest/config/optional.yaml
@@ -1,10 +1,10 @@
 # Optional configs used by Nextstrain team
 # Params for uploads
+# AWS S3 Bucket with prefix
+s3_dst: 's3://nextstrain-data/files/workflows/monkeypox'
 upload:
   # Upload params for AWS S3
   s3:
-    # AWS S3 Bucket with prefix
-    dst: 's3://nextstrain-data/files/workflows/monkeypox'
     # Files to upload to S3 that are in the `data` directory
     files_to_upload: [
       'genbank.ndjson',
diff --git a/ingest/workflow/snakemake_rules/upload.smk b/ingest/workflow/snakemake_rules/upload.smk
index fd4d3305..94afc3f3 100644
--- a/ingest/workflow/snakemake_rules/upload.smk
+++ b/ingest/workflow/snakemake_rules/upload.smk
@@ -52,7 +52,7 @@ rule upload_to_s3:
         "data/upload/s3/{file_to_upload}-to-{remote_file_name}.done",
     params:
         quiet="" if send_notifications else "--quiet",
-        s3_dst=config["upload"].get("s3", {}).get("dst", ""),
+        s3_dst=config.get("s3_dst", ""),
         cloudfront_domain=config["upload"].get("s3", {}).get("cloudfront_domain", ""),
     shell:
         """