Argo.design

argo/assignment_workflows/assignment_template.yaml

@@ -0,0 +1,271 @@
+---
+# Default workflow for assigning metadata to dates.
+# The data is loaded to a shared volume.
+# The assignment module processes whatever is in the volume.
+# The final output is written to a specified path in the specified output bucket.
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: assignment-template
+spec:
+  entrypoint: main
+  # This block for granting setting the group permissions on emptyDir volumes
+  securityContext:
+    fsGroup: 1001
+    fsGroupChangePolicy: OnRootMismatch
+  templates:
+  - name: main
+    inputs:
+      parameters:
+      - name: source_type
+      - name: r_args
+      # Define constant paths for use across container mounted volumes.
+      - name: in_path
+        value: /inputs
+      - name: out_path
+        value: /outputs
+      - name: tmp_path
+        value: /tmp
+      - name: error_path
+        value: /errors
+      - name: download_data_dir
+        value: download_data
+      - name: data_year_dir
+        value: data_year
+      # ConfigMap constants.
+      - name: log_level
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: log_level
+      # Git
+      - name: git_secret_name
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_secret_name
+      - name: git_secret_file_name
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_secret_file_name
+      - name: git_repo_url
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_repo_url
+      - name: git_branch
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_branch
+      - name: git_repo_name
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_repo_name
+      - name: git_repo_dir
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: git_repo_dir
+      # Output bucket
+      - name: output_bucket
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: output_bucket
+      - name: output_bucket_prefix
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: output_bucket_prefix
+      # Data year
+      - name: data_year_file
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: data_year_file
+      - name: data_year_bucket
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: data_year_bucket
+      - name: data_year_bucket_prefix
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: data_year_bucket_prefix
+      # Manifests
+      - name: manifest_file_name
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: manifest_file_name
+      - name: change_manifest_file_name
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: change_manifest_file_name
+      # R Code
+      - name: parallelization_internal
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: parallelization_internal
+      - name: relative_path_index
+        valueFrom:
+          configMapKeyRef:
+            name: "{{workflow.parameters.env_config}}"
+            key: relative_path_index
+      artifacts: 
+      - name: data-year
+        path: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/{{inputs.parameters.data_year_file}}"
+        gcs:
+          bucket: "{{inputs.parameters.data_year_bucket}}"
+          key: "{{inputs.parameters.data_year_bucket_prefix}}"
+    volumes:
+    - name: in-vol
+      emptyDir: { }
+    - name: out-vol
+      emptyDir: { }
+    - name: tmp-vol
+      emptyDir: { }
+    - name: err-vol
+      emptyDir: { }
+    - name: git-secret-vol
+      secret:
+        secretName: "{{inputs.parameters.git_secret_name}}"
+    containerSet:
+      # Must mount volumes to make them accessible to all containers 
+      # (otherwise they are only accessible to the "main" container).
+      volumeMounts:
+      - name: in-vol
+        mountPath: "{{inputs.parameters.in_path}}"
+      - name: out-vol
+        mountPath: "{{inputs.parameters.out_path}}"
+      - name: tmp-vol
+        mountPath: "{{inputs.parameters.tmp_path}}"  # Required to run R code that creates a temp directory
+      - name: err-vol
+        mountPath: "{{inputs.parameters.error_path}}"
+      containers:
+      # Read all the metadata files from Git that have changed in the latest commit.
+      - name: load
+        image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-argo-workflows-data-change-manifest:sha-461c14b
+        securityContext:
+          runAsUser: 1001 
+          runAsGroup: 1001
+        volumeMounts:
+        - name: git-secret-vol
+          mountPath: /key
+        resources:
+          requests:
+            memory: 1.5Gi
+            cpu: 1
+          limits:
+            memory: 2.5Gi
+            cpu: 1.25
+        env:
+        - name: DOWNLOAD_PATH
+          value: "{{inputs.parameters.tmp_path}}/{{inputs.parameters.git_repo_name}}"
+        - name: GIT_BRANCH_NAME
+          value: "{{inputs.parameters.git_branch}}"
+        - name: GIT_REPO_DIR
+          value: "{{inputs.parameters.git_repo_dir}}"
+        - name: GIT_REPO_NAME
+          value: "{{inputs.parameters.git_repo_name}}"
+        - name: GIT_REPO_URL
+          value: "{{inputs.parameters.git_repo_url}}"
+        - name: KEY_PATH
+          value: "/key/{{inputs.parameters.git_secret_file_name}}"
+        - name: LOGURU_LEVEL
+          value: "{{inputs.parameters.log_level}}"
+        - name: OUTPUT_PATH
+          value: "{{inputs.parameters.in_path}}/{{inputs.parameters.download_data_dir}}"
+        command:
+        - sh
+        - -c
+        - python3 /app/clone_changed_files.py
+      # Assign the assets and locations
+      - name: assign
+        dependencies:
+        - load
+        image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:v1.3.4
+        # Need to run as user 1001 because the gcs input is loaded as 
+        # this user and group, as controlled by the helm chart.
+        # Note that the image sets the user and group as 9999, but this 
+        # doesn't seem to matter because we don't need write priveleges in the container.
+        securityContext:
+          runAsUser: 1001 
+          runAsGroup: 1001
+        # NOTE: The individual resources requested for each container don't really matter; 
+        # it's the sum total that is actually available to each container, 
+        # so long as they run in sequence. Any containers running in parallel must have 
+        # the sum of their resource needs available.
+        resources:
+          requests:
+            memory: 3Gi
+            cpu: 1
+          limits:
+            memory: 4Gi
+            cpu: 1.25
+        env:
+        - name: DIR_IN
+          value: "{{inputs.parameters.in_path}}/{{inputs.parameters.download_data_dir}}"
+        - name: DIR_OUT
+          value: "{{inputs.parameters.out_path}}"
+        - name: ERR_PATH
+          value: "{{inputs.parameters.error_path}}"
+        - name: FILE_YEAR
+          value: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/{{inputs.parameters.data_year_file}}"
+        - name: LOG_LEVEL
+          value: "{{inputs.parameters.log_level}}"
+        - name: RELATIVE_PATH_INDEX
+          value: "{{inputs.parameters.relative_path_index}}"
+        - name: R_ARGS
+          value: "{{inputs.parameters.r_args}}"
+        - name: PARALLELIZATION_INTERNAL
+          value: "{{inputs.parameters.parallelization_internal}}"
+        command:
+        - bash
+        - -c
+        - |
+          # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/
+          set -euo pipefail
+          IFS=$'\n\t'
+          eval "Rscript ./flow.loc.grp.asgn.R $R_ARGS"
+      # Generated the change manifest and write it to a bucket.
+      - name: main
+        dependencies:
+        - assign
+        image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-argo-workflows-data-change-manifest:sha-461c14b
+        resources:
+          requests:
+            memory: 300Mi
+            cpu: 100m
+          limits:
+            memory: 500Mi
+            cpu: 1
+        env:
+        - name: CHANGE_MANIFEST_FILE_NAME
+          value: "{{inputs.parameters.change_manifest_file_name}}"
+        - name: LOGURU_LEVEL
+          value: "{{inputs.parameters.log_level}}"
+        - name: MANIFEST_BUCKET
+          value: "{{inputs.parameters.output_bucket}}"
+        - name: MANIFEST_BUCKET_PREFIX
+          value: "{{inputs.parameters.output_bucket_prefix}}"
+        - name: MANIFEST_FILE_NAME
+          value: "{{inputs.parameters.manifest_file_name}}"
+        - name: OUTPUT_PATH
+          value: "{{inputs.parameters.tmp_path}}"
+        - name: RUN_DATE
+          value: "{{workflow.parameters.run_date}}"
+        - name: SOURCE_PATH
+          value: "{{inputs.parameters.out_path}}/{{inputs.parameters.source_type}}"
+        - name: SOURCE_TYPE
+          value: "{{inputs.parameters.source_type}}"
+        command:
+        - sh 
+        - -c
+        - python3 /app/main.py