diff --git a/argo/assignment_workflows/assignment_template.yaml b/argo/assignment_workflows/assignment_template.yaml new file mode 100644 index 0000000000..baff2faf6a --- /dev/null +++ b/argo/assignment_workflows/assignment_template.yaml @@ -0,0 +1,271 @@ +--- +# Default workflow for assigning metadata to dates. +# The data is loaded to a shared volume. +# The assignment module processes whatever is in the volume. +# The final output is written to a specified path in the specified output bucket. +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: assignment-template +spec: + entrypoint: main + # This block for granting setting the group permissions on emptyDir volumes + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + templates: + - name: main + inputs: + parameters: + - name: source_type + - name: r_args + # Define constant paths for use across container mounted volumes. + - name: in_path + value: /inputs + - name: out_path + value: /outputs + - name: tmp_path + value: /tmp + - name: error_path + value: /errors + - name: download_data_dir + value: download_data + - name: data_year_dir + value: data_year + # ConfigMap constants. + - name: log_level + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: log_level + # Git + - name: git_secret_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_secret_name + - name: git_secret_file_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_secret_file_name + - name: git_repo_url + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_repo_url + - name: git_branch + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_branch + - name: git_repo_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_repo_name + - name: git_repo_dir + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: git_repo_dir + # Output bucket + - name: output_bucket + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: output_bucket + - name: output_bucket_prefix + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: output_bucket_prefix + # Data year + - name: data_year_file + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_file + - name: data_year_bucket + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_bucket + - name: data_year_bucket_prefix + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_bucket_prefix + # Manifests + - name: manifest_file_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: manifest_file_name + - name: change_manifest_file_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: change_manifest_file_name + # R Code + - name: parallelization_internal + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: parallelization_internal + - name: relative_path_index + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: relative_path_index + artifacts: + - name: data-year + path: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/{{inputs.parameters.data_year_file}}" + gcs: + bucket: "{{inputs.parameters.data_year_bucket}}" + key: "{{inputs.parameters.data_year_bucket_prefix}}" + volumes: + - name: in-vol + emptyDir: { } + - name: out-vol + emptyDir: { } + - name: tmp-vol + emptyDir: { } + - name: err-vol + emptyDir: { } + - name: git-secret-vol + secret: + secretName: "{{inputs.parameters.git_secret_name}}" + containerSet: + # Must mount volumes to make them accessible to all containers + # (otherwise they are only accessible to the "main" container). + volumeMounts: + - name: in-vol + mountPath: "{{inputs.parameters.in_path}}" + - name: out-vol + mountPath: "{{inputs.parameters.out_path}}" + - name: tmp-vol + mountPath: "{{inputs.parameters.tmp_path}}" # Required to run R code that creates a temp directory + - name: err-vol + mountPath: "{{inputs.parameters.error_path}}" + containers: + # Read all the metadata files from Git that have changed in the latest commit. + - name: load + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-argo-workflows-data-change-manifest:sha-461c14b + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + volumeMounts: + - name: git-secret-vol + mountPath: /key + resources: + requests: + memory: 1.5Gi + cpu: 1 + limits: + memory: 2.5Gi + cpu: 1.25 + env: + - name: DOWNLOAD_PATH + value: "{{inputs.parameters.tmp_path}}/{{inputs.parameters.git_repo_name}}" + - name: GIT_BRANCH_NAME + value: "{{inputs.parameters.git_branch}}" + - name: GIT_REPO_DIR + value: "{{inputs.parameters.git_repo_dir}}" + - name: GIT_REPO_NAME + value: "{{inputs.parameters.git_repo_name}}" + - name: GIT_REPO_URL + value: "{{inputs.parameters.git_repo_url}}" + - name: KEY_PATH + value: "/key/{{inputs.parameters.git_secret_file_name}}" + - name: LOGURU_LEVEL + value: "{{inputs.parameters.log_level}}" + - name: OUTPUT_PATH + value: "{{inputs.parameters.in_path}}/{{inputs.parameters.download_data_dir}}" + command: + - sh + - -c + - python3 /app/clone_changed_files.py + # Assign the assets and locations + - name: assign + dependencies: + - load + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:v1.3.4 + # Need to run as user 1001 because the gcs input is loaded as + # this user and group, as controlled by the helm chart. + # Note that the image sets the user and group as 9999, but this + # doesn't seem to matter because we don't need write priveleges in the container. + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + # NOTE: The individual resources requested for each container don't really matter; + # it's the sum total that is actually available to each container, + # so long as they run in sequence. Any containers running in parallel must have + # the sum of their resource needs available. + resources: + requests: + memory: 3Gi + cpu: 1 + limits: + memory: 4Gi + cpu: 1.25 + env: + - name: DIR_IN + value: "{{inputs.parameters.in_path}}/{{inputs.parameters.download_data_dir}}" + - name: DIR_OUT + value: "{{inputs.parameters.out_path}}" + - name: ERR_PATH + value: "{{inputs.parameters.error_path}}" + - name: FILE_YEAR + value: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/{{inputs.parameters.data_year_file}}" + - name: LOG_LEVEL + value: "{{inputs.parameters.log_level}}" + - name: RELATIVE_PATH_INDEX + value: "{{inputs.parameters.relative_path_index}}" + - name: R_ARGS + value: "{{inputs.parameters.r_args}}" + - name: PARALLELIZATION_INTERNAL + value: "{{inputs.parameters.parallelization_internal}}" + command: + - bash + - -c + - | + # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/ + set -euo pipefail + IFS=$'\n\t' + eval "Rscript ./flow.loc.grp.asgn.R $R_ARGS" + # Generated the change manifest and write it to a bucket. + - name: main + dependencies: + - assign + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-argo-workflows-data-change-manifest:sha-461c14b + resources: + requests: + memory: 300Mi + cpu: 100m + limits: + memory: 500Mi + cpu: 1 + env: + - name: CHANGE_MANIFEST_FILE_NAME + value: "{{inputs.parameters.change_manifest_file_name}}" + - name: LOGURU_LEVEL + value: "{{inputs.parameters.log_level}}" + - name: MANIFEST_BUCKET + value: "{{inputs.parameters.output_bucket}}" + - name: MANIFEST_BUCKET_PREFIX + value: "{{inputs.parameters.output_bucket_prefix}}" + - name: MANIFEST_FILE_NAME + value: "{{inputs.parameters.manifest_file_name}}" + - name: OUTPUT_PATH + value: "{{inputs.parameters.tmp_path}}" + - name: RUN_DATE + value: "{{workflow.parameters.run_date}}" + - name: SOURCE_PATH + value: "{{inputs.parameters.out_path}}/{{inputs.parameters.source_type}}" + - name: SOURCE_TYPE + value: "{{inputs.parameters.source_type}}" + command: + - sh + - -c + - python3 /app/main.py \ No newline at end of file diff --git a/argo/assignment_workflows/calibration_assignment/calibration_assignment.yaml b/argo/assignment_workflows/calibration_assignment/calibration_assignment.yaml new file mode 100644 index 0000000000..ef71a42567 --- /dev/null +++ b/argo/assignment_workflows/calibration_assignment/calibration_assignment.yaml @@ -0,0 +1,299 @@ +--- +# Default workflow for assigning calibrations to data days. +# The calibration data is loaded to a local shared volume. +# The calibration assignment module processes whatever is in the volume. +# The final output is written to a specified prefix in the output bucket. +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: calibration-assignment +spec: + schedules: + - "* * 31 2 *" + concurrencyPolicy: Forbid + workflowSpec: + entrypoint: main + # This block for granting setting the group permissions on emptyDir volumes + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + arguments: + parameters: + - name: run_date + value: "{{=sprig.date('2006/01/02', workflow.creationTimestamp)}}" + - name: env_config + value: calibration-assignment-env + - name: assignment_config + valueFrom: + configMapKeyRef: + name: calibration-assignment-config + key: config + + templates: + # Loop over source types and r code config. + - name: main + steps: + - - name: run-step + template: run + arguments: + parameters: + - name: source_type + value: "{{item.key}}" + - name: r_args + value: "{{item.value}}" + withParam: "{{workflow.parameters.assignment_config}}" + + # Process each source type. + - name: run + inputs: + parameters: + + # Required input parameters. + - name: source_type + - name: r_args + + # Define constant paths for use across container mounted volumes. + - name: in_path + value: /inputs + - name: out_path + value: /outputs + - name: tmp_path + value: /tmp + - name: error_path + value: /errors + - name: calibration_dir + value: calibrations + - name: data_year_dir + value: data_year + + # ConfigMap constants. + - name: log_level + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: log_level + + # Calibration source bucket + - name: calibration_bucket + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: calibration_bucket + - name: calibration_bucket_prefix + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: calibration_bucket_prefix + + # Output bucket + - name: output_bucket + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: output_bucket + - name: output_bucket_prefix + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: output_bucket_prefix + + # Data year + - name: data_year_file + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_file + - name: data_year_bucket + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_bucket + - name: data_year_bucket_prefix + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: data_year_bucket_prefix + + # Manifests + - name: manifest_file_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: manifest_file_name + - name: change_manifest_file_name + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: change_manifest_file_name + + # R Code + - name: parallelization_internal + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: parallelization_internal + - name: relative_path_index + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.env_config}}" + key: relative_path_index + + artifacts: + - name: data-year + path: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/{{inputs.parameters.data_year_file}}" + gcs: + bucket: "{{inputs.parameters.data_year_bucket}}" + key: "{{inputs.parameters.data_year_bucket_prefix}}" + + volumes: + - name: in-vol + emptyDir: { } + - name: out-vol + emptyDir: { } + - name: tmp-vol + emptyDir: { } + - name: err-vol + emptyDir: { } + # A writable volume to run gcloud command, which tries to write to HOME/config + - name: gcloud-volume + emptyDir: {} + + containerSet: + # Must mount volumes to make them accessible to all containers + # (otherwise they are only accessible to the "main" container). + + volumeMounts: + - name: in-vol + mountPath: "{{inputs.parameters.in_path}}" + - name: out-vol + mountPath: "{{inputs.parameters.out_path}}" + - name: tmp-vol + mountPath: "{{inputs.parameters.tmp_path}}" # Required to run R code that creates a temp directory + - name: err-vol + mountPath: "{{inputs.parameters.error_path}}" + - name: gcloud-volume + mountPath: /mnt/gcloud + + containers: + # Download the calibration files + - name: download + image: gcr.io/google.com/cloudsdktool/google-cloud-cli:slim + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + resources: + requests: + memory: 2Gi + cpu: 1 + limits: + memory: 3Gi + cpu: 1.25 + command: + - sh + - -c + - | + export HOME=/mnt/gcloud + + bucket={{inputs.parameters.calibration_bucket}} + prefix={{inputs.parameters.calibration_bucket_prefix}} + source_type={{inputs.parameters.source_type}} + in_dir={{inputs.parameters.in_path}} + calibration_dir={{inputs.parameters.calibration_dir}} + + url="gs://${bucket}/${prefix}/${source_type}/" + + download_dir=${in_dir}/${calibration_dir} + + mkdir -p ${download_dir} + cd ${download_dir} + echo "Copying files from GCS bucket..." + if gcloud storage cp --recursive $url ${download_dir}/; then + echo "Copy succeeded." + else + echo "No matching files for $url (or another error occurred). Continuing..." + fi + + # Assign the calibrations + - name: assign + dependencies: + - download + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-cal-asgn:v2.0.7 + # Need to run as user 1001 because the gcs input is loaded as + # this user and group, as controlled by the helm chart. + # Note that the image sets the user and group as 9999, but this + # doesn't seem to matter because we don't need write priveleges in the container. + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + # NOTE: The individual resources requested for each container don't really matter; + # it's the sum total that is actually available to each container, + # so long as they run in sequence. Any containers running in parallel must have + # the sum of their resource needs available. + resources: + requests: + memory: 2Gi + cpu: 1 + limits: + memory: 3Gi + cpu: 1.25 + env: + - name: DIR_IN + value: "{{inputs.parameters.in_path}}/{{inputs.parameters.calibration_dir}}" + - name: DIR_OUT + value: "{{inputs.parameters.out_path}}" + - name: ERR_PATH + value: "{{inputs.parameters.error_path}}" + - name: RELATIVE_PATH_INDEX + value: "{{inputs.parameters.relative_path_index}}" + - name: LOG_LEVEL + value: "{{inputs.parameters.log_level}}" + - name: R_ARGS + value: "{{inputs.parameters.r_args}}" + - name: FILE_YEAR + value: "{{inputs.parameters.in_path}}/{{inputs.parameters.data_year_dir}}/data_year_2026.txt" + - name: PARALLELIZATION_INTERNAL + value: "{{inputs.parameters.parallelization_internal}}" + command: + - bash + - -c + - | + # Use bash-scrict mode. See http://redsymbol.net/articles/unofficial-bash-strict-mode/ + set -euo pipefail + IFS=$'\n\t' + eval "Rscript ./flow.cal.asgn.R $R_ARGS" + + # Generate the change manifest and save to a bucket + - name: main + dependencies: + - assign + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-argo-workflows-data-change-manifest:sha-461c14b + resources: + requests: + memory: 300Mi + cpu: 100m + limits: + memory: 500Mi + cpu: 1 + env: + - name: SOURCE_PATH + value: "{{inputs.parameters.out_path}}" + - name: SOURCE_TYPE + value: "{{inputs.parameters.source_type}}" + - name: RUN_DATE + value: "{{workflow.parameters.run_date}}" + - name: OUTPUT_PATH + value: "{{inputs.parameters.tmp_path}}" + - name: MANIFEST_BUCKET + value: "{{inputs.parameters.output_bucket}}" + - name: MANIFEST_BUCKET_PREFIX + value: "{{inputs.parameters.output_bucket_prefix}}" + - name: MANIFEST_FILE_NAME + value: "{{inputs.parameters.manifest_file_name}}" + - name: CHANGE_MANIFEST_FILE_NAME + value: "{{inputs.parameters.change_manifest_file_name}}" + - name: LOGURU_LEVEL + value: "{{inputs.parameters.log_level}}" + command: + - sh + - -c + - python3 /app/main.py \ No newline at end of file diff --git a/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-config.yaml b/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-config.yaml new file mode 100644 index 0000000000..01097cba12 --- /dev/null +++ b/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-config.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-assignment-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + "config": | + [ + { + "key": "aepg600m", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "cmp22", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "csat3", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "dualfan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "enviroscan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exo2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exoconductivity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exodissolvedoxygen", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exofdom", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exophorp", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exototalalgae", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "exoturbidity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "gascylinder", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "hmp155", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "lt400_hu24", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "li191r", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "li840a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "li7200", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "mcseries", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "metone370380", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "mt300ahrs", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "pluvio", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "pqs1", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "pressuretransducer", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "prt", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "ptb330a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "pump", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "sunav2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "tchain", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "troll", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + }, + { + "key": "windobserverii", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR" + } + ] \ No newline at end of file diff --git a/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-env.yaml b/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-env.yaml new file mode 100644 index 0000000000..969ddf728c --- /dev/null +++ b/argo/assignment_workflows/calibration_assignment/config/configmap-calibration-assignment-env.yaml @@ -0,0 +1,31 @@ +--- +# Environment variables for the location asset assignment workflow. +# To load this configmap in kubernetes: +# kubectl config set-context --current --namespace=argo-workflows-dev +# kubectl apply -f -n argo-workflows-dev +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-assignment-env + namespace: argo-workflows-dev + labels: + # Note: This label is required for the informer to detect this ConfigMap. + workflows.argoproj.io/configmap-type: Parameter +data: + log_level: DEBUG + # R code + relative_path_index: "2" + parallelization_internal: "4" # Internal R parallelization for all R modules. + # Data year file config to restrict processing window. + data_year_file: data_year_2026.txt + data_year_bucket: neon-dev-argo-workflow-test + data_year_bucket_prefix: data_years/data_year_2026.txt + # Calibration bucket + calibration_bucket: neon-test-calibration + calibration_bucket_prefix: files + # Bucket for writing manifests. + output_bucket: neon-dev-argo-workflow-test + output_bucket_prefix: calibration_assignment + # Manifests for changed files. + manifest_file_name: manifest.json + change_manifest_file_name: change_manifest.json diff --git a/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-config.yaml b/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-config.yaml new file mode 100644 index 0000000000..f45c15fb08 --- /dev/null +++ b/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-config.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: group-assignment-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + "config": | + [ + { + "key": "aspirated-single", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "aspirated-triple", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "conc-h2o-soil-salinity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "conc-h2o-soil-salinity-split", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "groundwater-physical", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "l4discharge", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "nitrate-surfacewater", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "par", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "par-quantum-line", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "par-underwater", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "par-water-surface", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "precip-throughfall", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "precip-tipping", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "precip-weighing", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "precip-weighing-v2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "pressure-air", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "pressure-air-buoy", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "rad-short-primary", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "rel-humidity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "rel-humidity-buoy", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "subsurf-moor-temp-cond", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "summary-weather-stats", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "surfacewater-physical", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "temp-air-triple", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "temp-soil", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "temp-specific-depths-lakes", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "temp-specific-depths-lakes-split", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "temp-surfacewater", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "turbulent", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + }, + { + "key": "water-quality", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group" + } + ] \ No newline at end of file diff --git a/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-env.yaml b/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-env.yaml new file mode 100644 index 0000000000..bc0fa3935d --- /dev/null +++ b/argo/assignment_workflows/group_assignment/config/configmap-group-assignment-env.yaml @@ -0,0 +1,35 @@ +--- +# Environment variables for the location active dates assignment workflow. +# To load this configmap in kubernetes: +# kubectl config set-context --current --namespace=argo-workflows-dev +# kubectl apply -f -n argo-workflows-dev +apiVersion: v1 +kind: ConfigMap +metadata: + name: group-assignment-env + namespace: argo-workflows-dev + labels: + # Note: This label is required for the informer to detect this ConfigMap. + workflows.argoproj.io/configmap-type: Parameter +data: + log_level: DEBUG + # R code + relative_path_index: "2" + parallelization_internal: "4" # Internal R parallelization for all R modules. + # Git configuration. + git_secret_name: git-data-processing-metadata-key + git_secret_file_name: ssh-private-key + git_repo_url: git@github.com:BattelleEcology/data-processing-metadata.git + git_branch: main + git_repo_name: data-processing-metadata + git_repo_dir: groups + # Bucket for saving manifests. + output_bucket: neon-dev-argo-workflow-test + output_bucket_prefix: groups_assignment + # Data year file config to restrict processing window. + data_year_file: data_year_2026.txt + data_year_bucket: neon-dev-argo-workflow-test + data_year_bucket_prefix: data_years/data_year_2026.txt + # Manifests for changed files. + manifest_file_name: manifest.json + change_manifest_file_name: change_manifest.json diff --git a/argo/assignment_workflows/group_assignment/group_assignment.yaml b/argo/assignment_workflows/group_assignment/group_assignment.yaml new file mode 100644 index 0000000000..91098e3ac9 --- /dev/null +++ b/argo/assignment_workflows/group_assignment/group_assignment.yaml @@ -0,0 +1,44 @@ +--- +# Workflow for assigning groups to data days. +# The data is loaded to a shared volume. +# The assignment module processes whatever is in the volume. +# The final output is written to a specified path in the specified output bucket. +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: group-assignment +spec: + schedules: + - "* * 31 2 *" + concurrencyPolicy: Forbid + workflowSpec: + entrypoint: main + # This block for granting setting the group permissions on emptyDir volumes + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + arguments: + parameters: + - name: run_date + value: "{{=sprig.date('2006/01/02', workflow.creationTimestamp)}}" + - name: env_config + value: group-assignment-env + - name: assignment_config + valueFrom: + configMapKeyRef: + name: group-assignment-config + key: config + templates: + - name: main + steps: + - - name: run-step + templateRef: + name: assignment-template + template: main + arguments: + parameters: + - name: source_type + value: '{{item.key}}' + - name: r_args + value: '{{item.value}}' + withParam: '{{workflow.parameters.assignment_config}}' \ No newline at end of file diff --git a/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-config.yaml b/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-config.yaml new file mode 100644 index 0000000000..a09c2b1e2e --- /dev/null +++ b/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-config.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: location-active-dates-assignment-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + "config": | + [ + { + "key": "aepg600m", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "cmp22", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "csat3", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "dualfan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "enviroscan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exo2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exoconductivity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exodissolvedoxygen", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exofdom", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exophorp", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exototalalgae", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "exoturbidity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "gascylinder", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "hmp155", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "lt400_hu24", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "li191r", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "li840a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "li7200", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "mcseries", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "metone370380", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "mt300ahrs", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "pluvio", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "pqs1", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "pressuretransducer", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "prt", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "ptb330a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "pump", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "sunav2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "tchain", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "troll", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + }, + { + "key": "windobserverii", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=namedLocation" + } + ] \ No newline at end of file diff --git a/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-env.yaml b/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-env.yaml new file mode 100644 index 0000000000..a0e931e1de --- /dev/null +++ b/argo/assignment_workflows/location_active_dates_assignment/config/configmap-location-active-dates-assignment-env.yaml @@ -0,0 +1,35 @@ +--- +# Environment variables for the location active dates assignment workflow. +# To load this configmap in kubernetes: +# kubectl config set-context --current --namespace=argo-workflows-dev +# kubectl apply -f -n argo-workflows-dev +apiVersion: v1 +kind: ConfigMap +metadata: + name: location-active-dates-assignment-env + namespace: argo-workflows-dev + labels: + # Note: This label is required for the informer to detect this ConfigMap. + workflows.argoproj.io/configmap-type: Parameter +data: + log_level: INFO + # R code + relative_path_index: "2" + parallelization_internal: "4" # Internal R parallelization for all R modules. + # Git configuration. + git_secret_name: git-data-processing-metadata-key + git_secret_file_name: ssh-private-key + git_repo_url: git@github.com:BattelleEcology/data-processing-metadata.git + git_branch: main + git_repo_name: data-processing-metadata + git_repo_dir: locations + # Bucket for saving manifests. + output_bucket: neon-dev-argo-workflow-test + output_bucket_prefix: location_active_dates_assignment + # Data year file config to restrict processing window. + data_year_file: data_year_2026.txt + data_year_bucket: neon-dev-argo-workflow-test + data_year_bucket_prefix: data_years/data_year_2026.txt + # Manifests for changed files. + manifest_file_name: manifest.json + change_manifest_file_name: change_manifest.json diff --git a/argo/assignment_workflows/location_active_dates_assignment/location_active_dates_assignment.yaml b/argo/assignment_workflows/location_active_dates_assignment/location_active_dates_assignment.yaml new file mode 100644 index 0000000000..b6d30c5838 --- /dev/null +++ b/argo/assignment_workflows/location_active_dates_assignment/location_active_dates_assignment.yaml @@ -0,0 +1,44 @@ +--- +# Default workflow for assigning location active dates to data days. +# The data is loaded to a shared volume. +# The assignment module processes whatever is in the volume. +# The final output is written to a specified path in the specified output bucket. +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: location-active-dates-assignment +spec: + schedules: + - "* * 31 2 *" + concurrencyPolicy: Forbid + workflowSpec: + entrypoint: main + # This block for granting setting the group permissions on emptyDir volumes + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + arguments: + parameters: + - name: run_date + value: "{{=sprig.date('2006/01/02', workflow.creationTimestamp)}}" + - name: env_config + value: location-active-dates-assignment-env + - name: assignment_config + valueFrom: + configMapKeyRef: + name: location-active-dates-assignment-config + key: config + templates: + - name: main + steps: + - - name: run-step + templateRef: + name: assignment-template + template: main + arguments: + parameters: + - name: source_type + value: '{{item.key}}' + - name: r_args + value: '{{item.value}}' + withParam: '{{workflow.parameters.assignment_config}}' \ No newline at end of file diff --git a/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-config.yaml b/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-config.yaml new file mode 100644 index 0000000000..5fcec5ca9c --- /dev/null +++ b/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-config.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: location-asset-assignment-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + "config": | + [ + { + "key": "aepg600m", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate'" + }, + { + "key": "cmp22", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations|location_properties:LatTow|location_properties:LonTow'" + }, + { + "key": "csat3", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "dualfan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate'" + }, + { + "key": "enviroscan", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations|locationMetadata:DistZaxsLvlMeasSoil|locationMetadata:LvlMeasSoil'" + }, + { + "key": "exo2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exoconductivity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exodissolvedoxygen", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exofdom", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exophorp", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exototalalgae", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "exoturbidity", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "gascylinder", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "hmp155", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "lt400_hu24", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations'" + }, + { + "key": "li191r", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate'" + }, + { + "key": "li840a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "li7200", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "mcseries", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "metone370380", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|context'" + }, + { + "key": "mt300ahrs", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "pluvio", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate'" + }, + { + "key": "pqs1", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|context'" + }, + { + "key": "pressuretransducer", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "prt", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "ptb330a", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations'" + }, + { + "key": "pump", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset" + }, + { + "key": "sunav2", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate'" + }, + { + "key": "tchain", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations|ThermistorDepth501|ThermistorDepth502|ThermistorDepth503|ThermistorDepth504|ThermistorDepth505|ThermistorDepth506|ThermistorDepth507|ThermistorDepth508|ThermistorDepth509|ThermistorDepth510|ThermistorDepth511'" + }, + { + "key": "troll", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations|context|location_properties:Coordinate uncertainty|location_properties:Coordinate source|location_properties:Elevation uncertainty|location_properties:Real world coordinate uncertainty|location_properties:Real world elevation uncertainty|location_properties:Survey horizontal uncertainty|location_properties:Survey vertical uncertainty|location_properties:Transformation RMSE'" + }, + { + "key": "windobserverii", + "value": "DirIn=$DIR_IN DirOut=$DIR_OUT DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=asset 'Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations'" + } + ] \ No newline at end of file diff --git a/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-env.yaml b/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-env.yaml new file mode 100644 index 0000000000..d7f0784a08 --- /dev/null +++ b/argo/assignment_workflows/location_asset_assignment/config/configmap-location-asset-assignment-env.yaml @@ -0,0 +1,35 @@ +--- +# Environment variables for the location asset assignment workflow. +# To load this configmap in kubernetes: +# kubectl config set-context --current --namespace=argo-workflows-dev +# kubectl apply -f -n argo-workflows-dev +apiVersion: v1 +kind: ConfigMap +metadata: + name: location-asset-assignment-env + namespace: argo-workflows-dev + labels: + # Note: This label is required for the informer to detect this ConfigMap. + workflows.argoproj.io/configmap-type: Parameter +data: + log_level: DEBUG + # R code + relative_path_index: "2" + parallelization_internal: "4" # Internal R parallelization for all R modules. + # Git configuration. + git_secret_name: git-data-processing-metadata-key + git_secret_file_name: ssh-private-key + git_repo_url: git@github.com:BattelleEcology/data-processing-metadata.git + git_branch: main + git_repo_name: data-processing-metadata + git_repo_dir: location-assets + # Bucket for saving manifests. + output_bucket: neon-dev-argo-workflow-test + output_bucket_prefix: location_assignment + # Data year file config to restrict processing window. + data_year_file: data_year_2026.txt + data_year_bucket: neon-dev-argo-workflow-test + data_year_bucket_prefix: data_years/data_year_2026.txt + # Manifests for changed files. + manifest_file_name: manifest.json + change_manifest_file_name: change_manifest.json diff --git a/argo/assignment_workflows/location_asset_assignment/location_asset_assignment.yaml b/argo/assignment_workflows/location_asset_assignment/location_asset_assignment.yaml new file mode 100644 index 0000000000..45bad7e57d --- /dev/null +++ b/argo/assignment_workflows/location_asset_assignment/location_asset_assignment.yaml @@ -0,0 +1,44 @@ +--- +# Default workflow for assigning locations/assets to data days. +# The data is loaded to a shared volume. +# The assignment module processes whatever is in the volume. +# The final output is written to a specified path in the specified output bucket. +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: location-asset-assignment +spec: + schedules: + - "* * 31 2 *" + concurrencyPolicy: Forbid + workflowSpec: + entrypoint: main + # This block for granting setting the group permissions on emptyDir volumes + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + arguments: + parameters: + - name: run_date + value: "{{=sprig.date('2006/01/02', workflow.creationTimestamp)}}" + - name: env_config + value: location-asset-assignment-env + - name: assignment_config + valueFrom: + configMapKeyRef: + name: location-asset-assignment-config + key: config + templates: + - name: main + steps: + - - name: run-step + templateRef: + name: assignment-template + template: main + arguments: + parameters: + - name: source_type + value: '{{item.key}}' + - name: r_args + value: '{{item.value}}' + withParam: '{{workflow.parameters.assignment_config}}' \ No newline at end of file diff --git a/argo/generic_template_design/ARCHITECTURE.md b/argo/generic_template_design/ARCHITECTURE.md new file mode 100644 index 0000000000..8b09666a2a --- /dev/null +++ b/argo/generic_template_design/ARCHITECTURE.md @@ -0,0 +1,90 @@ +## High-Level Architecture + +The workflow structure is organized by workflow family. Each family owns its +own base template and its own overlays, so new workflow templates can be added +without sharing names, ConfigMaps, or patches with existing families. + +```text +workflows/ + calibration-group-and-convert/ + base/ + calibration-group-and-convert.yaml + configmap-*.yaml + kustomization.yaml + overlays/ + cmp22/ + aepg600m/ + aepg600m_heated/ + / + base/ + overlays/ +``` + +```mermaid +flowchart LR + A[Kustomize overlay] --> B[Family base kustomization] + B --> C[WorkflowTemplate] + B --> D[Shared base ConfigMaps] + A --> E[Sensor-specific ConfigMaps and patches] + C --> F[Argo Workflow submission] + E --> F +``` + +## Workflow Execution Flow + +```text +$ kubectl apply -k workflows/calibration-group-and-convert/overlays/cmp22/ +``` + +That render produces one WorkflowTemplate plus the base and overlay ConfigMaps +needed by that sensor. The overlay patches the template to point at the +sensor-specific env and resource ConfigMaps, while the family base supplies the +shared load-data, processing, upload-output, and schema ConfigMaps. + +## Configuration Layout + +The family base contains the reusable pieces: + +```text +workflows/calibration-group-and-convert/base/ + calibration-group-and-convert.yaml + configmap-load-data-instructions.yaml + configmap-processing-instructions.yaml + configmap-resource-request.yaml + configmap-schemas.yaml + configmap-upload-output-instructions.yaml + kustomization.yaml +``` + +Each overlay contributes only the sensor-specific ConfigMaps and a small patch +to the WorkflowTemplate arguments. + +## Runtime Layout + +At runtime, the WorkflowTemplate consumes the rendered ConfigMaps and runs the +same container sequence for each sensor overlay: + +```text +load-data -> processing -> main +``` + +The sensor overlay changes the environment values, resource requests, and +output paths, but the template logic stays the same. + +## Deployment Topology + +```text +Kubernetes cluster + Namespace: argo-workflows-dev + WorkflowTemplate: calibration-group-and-convert + ConfigMaps from family base + ConfigMaps from sensor overlay +``` + +If you later add another workflow family, give it the same `base/` and +`overlays/` shape under `workflows//` and keep its names isolated from +the existing family. + +An optional top-level `workflows/kustomization.yaml` can aggregate multiple +families if you want a single entry point for `kubectl apply -k workflows/`. +If you deploy families independently, you do not need that extra layer. \ No newline at end of file diff --git a/argo/generic_template_design/workflows/README.md b/argo/generic_template_design/workflows/README.md new file mode 100644 index 0000000000..fd87104470 --- /dev/null +++ b/argo/generic_template_design/workflows/README.md @@ -0,0 +1,27 @@ +Workflow templates are organized by family so each template can own its base +resources and its overlays. + +Current pattern: + +```text +workflows/ + / + base/ + .yaml + configmap-*.yaml + kustomization.yaml + overlays/ + / + configmap-env.yaml + configmap-resource-request.yaml + kustomization.yaml +``` + +The initial family is: + +```text +workflows/calibration-group-and-convert/ +``` + +To add a new workflow template, create a sibling directory under `workflows/` +with the same `base/` and `overlays/` shape. \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/calibration-group-and-convert.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/calibration-group-and-convert.yaml new file mode 100644 index 0000000000..c6a107992c --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/calibration-group-and-convert.yaml @@ -0,0 +1,197 @@ +# Calibration group and convert workflow template +# This template is platform-agnostic and sensor-agnostic. +# Sensor-specific configuration is injected via ConfigMap environment variables. +# Processing instructions are still rendered from YAML by an init container. + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: calibration-group-and-convert +spec: + entrypoint: run + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch + + arguments: + parameters: + # - name: datum-manifest + # value: | + # [{"source_type": "aepg600m", "data_date": "2026-05-03","source_id": "16768"},{"source_type": "aepg600m", "data_date": "2026-05-01"}] + - name: datum-manifest + value: | + [{"source_type": "cmp22", "data_date": "2026-02-20", "source_id": "11184", "file_name":"10000000000098_WO83151_397977.xml"},{"source_type": "cmp22", "data_date": "2026-02-20", "source_id": "47913", "file_name":"10000000000098_WO83151_397977.xml"}] + - name: config-map-schemas-name + value: schemas-config + - name: config-map-env-name + value: calibration-group-convert-env-config + - name: config-map-load-data-instructions-name + value: calibration-group-convert-load-data-instructions-config + - name: config-map-processing-instructions-name + value: calibration-group-convert-processing-instructions-config + - name: config-map-upload-output-instructions-name + value: calibration-group-convert-upload-output-instructions-config + - name: config-map-resource-request-name + value: calibration-group-convert-resource-config + + templates: + - name: run + podSpecPatch: | + containers: + - name: processing + resources: + requests: + cpu: "{{inputs.parameters.cpu-request}}" + memory: "{{inputs.parameters.memory-request}}" + limits: + cpu: "{{inputs.parameters.cpu-limit}}" + memory: "{{inputs.parameters.memory-limit}}" + + volumes: + - name: config-load-data-vol + configMap: + name: "{{workflow.parameters.config-map-load-data-instructions-name}}" + - name: config-processing-vol + configMap: + name: "{{workflow.parameters.config-map-processing-instructions-name}}" + - name: config-upload-output-vol + configMap: + name: "{{workflow.parameters.config-map-upload-output-instructions-name}}" + - name: data-vol + emptyDir: { } + - name: tmp-vol + emptyDir: { } + + inputs: + parameters: + - name: datum-manifest + - name: config-map-env-name + - name: cpu-request + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-resource-request-name}}" + key: cpu_request + - name: cpu-limit + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-resource-request-name}}" + key: cpu_limit + - name: memory-request + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-resource-request-name}}" + key: memory_request + - name: memory-limit + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-resource-request-name}}" + key: memory_limit + - name: load-data-image + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-load-data-instructions-name}}" + key: image + - name: processing-image + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-processing-instructions-name}}" + key: image + - name: upload-output-image + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-upload-output-instructions-name}}" + key: image + - name: schema-repo-eng-url + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-schemas-name}}" + key: schema-repo-eng-url + - name: schema-repo-eng-revision + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-schemas-name}}" + key: schema-repo-eng-revision + - name: schema-repo-sci-url + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-schemas-name}}" + key: schema-repo-sci-url + - name: schema-repo-sci-revision + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.config-map-schemas-name}}" + key: schema-repo-sci-revision + artifacts: + - name: schemas-eng + path: /data/schemas-eng + git: + repo: "{{inputs.parameters.schema-repo-eng-url}}" + revision: "{{inputs.parameters.schema-repo-eng-revision}}" + depth: 1 + sshPrivateKeySecret: + name: neon-avro-schemas-secret-readonly + key: sshPrivateKey + - name: schemas-sci + path: /data/schemas-sci + git: + repo: "{{inputs.parameters.schema-repo-sci-url}}" + revision: "{{inputs.parameters.schema-repo-sci-revision}}" + depth: 1 + + containerSet: + volumeMounts: + - name: data-vol + mountPath: /data + - name: config-load-data-vol + mountPath: /scripts-load-data + - name: config-processing-vol + mountPath: /scripts + - name: config-upload-output-vol + mountPath: /scripts-upload-output + - name: tmp-vol + mountPath: /tmp + + containers: + - name: load-data + image: "{{inputs.parameters.load-data-image}}" + env: + - name: MANIFEST + value: "{{inputs.parameters.datum-manifest}}" + envFrom: + - configMapRef: + name: "{{inputs.parameters.config-map-env-name}}" + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + command: ["/bin/bash"] + args: ["/scripts-load-data/load-data-instructions.sh"] + + - name: processing + dependencies: + - load-data + image: "{{inputs.parameters.processing-image}}" + env: + - name: MANIFEST + value: "{{inputs.parameters.datum-manifest}}" + envFrom: + - configMapRef: + name: "{{inputs.parameters.config-map-env-name}}" + command: ["/bin/bash"] + args: ["/scripts/processing-instructions.sh"] + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + + # One container must be named "main". + - name: main + dependencies: + - processing + image: "{{inputs.parameters.upload-output-image}}" + envFrom: + - configMapRef: + name: "{{inputs.parameters.config-map-env-name}}" + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + command: ["/bin/bash"] + args: ["/scripts-upload-output/upload-output-instructions.sh"] \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-load-data-instructions.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-load-data-instructions.yaml new file mode 100644 index 0000000000..43a2797d93 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-load-data-instructions.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-group-convert-load-data-instructions-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-gcs-data:sha-9cf8a5b + load-data-instructions.sh: | + #!/bin/bash + set -euo pipefail + IFS=$'\n\t' + + # Get L0 data from GCS + echo "Retrieving L0 data indicate in the manifest" + python3 -m l0_gcs_loader_by_manifest + + # Get assigned calibrations + echo "Retrieving calibrations" + CAL_REPO="${CAL_BUCKET_PREFIX#/}" + CAL_REPO="${CAL_REPO%/}" + CAL_ROOT=":gcs://${CAL_BUCKET_NAME}/${CAL_REPO}" + + mkdir -p "$OUT_PATH_CAL" + + # Convert manifest records into a normalized {"paths": [...]} object + # consumed by the calibration download loop. + MANIFEST_PATHS="$(python3 -m manifest_paths_builder)" + + echo "Datum Manifest: $MANIFEST_PATHS" + echo "Cal Root: $CAL_ROOT" + echo "Cal Dest: $OUT_PATH_CAL" + echo + + for datum_path in $(printf '%s' "$MANIFEST_PATHS" | jq -r '.paths[]'); do + echo "Getting calibration datum: $datum_path" + + src="${CAL_ROOT}/${datum_path}" + dst="${OUT_PATH_CAL}/${datum_path}" + mkdir -p "$dst" + + rclone \ + --no-check-dest \ + --copy-links \ + --gcs-bucket-policy-only \ + --gcs-no-check-bucket \ + copy \ + --progress \ + "${src}" "${dst}" + done + + echo "Done getting data and calibrations." \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-processing-instructions.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-processing-instructions.yaml new file mode 100644 index 0000000000..b70949b398 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-processing-instructions.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-group-convert-processing-instructions-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-cal-grp-conv:sha-3e7b7b7 + processing-instructions.sh: | + #!/bin/bash + set -euo pipefail + IFS=$'\n\t' + + # Determine if MANIFEST is a file path or JSON string + if [ -f "$MANIFEST" ]; then + json_data=$(cat "$MANIFEST") + else + json_data="$MANIFEST" + fi + + # Run calibration assignment for each date in the manifest + while IFS= read -r data_date; do + export CAL_ASGN_DATE_BEGIN="$data_date" + + # Calculate next day (works with GNU date or macOS date) + export CAL_ASGN_DATE_END=$(date -d "$data_date + 1 day" +%Y-%m-%d) + + echo "CAL_ASGN_DATE_BEGIN=$CAL_ASGN_DATE_BEGIN" + echo "CAL_ASGN_DATE_END=$CAL_ASGN_DATE_END" + + eval "Rscript ./flow.cal.asgn.R $CAL_ASGN_R_ARGS" + + done < <(echo "$json_data" | jq -r '.[] | .data_date') + + # Join files from the archive and from Kafka + export OUT_PATH=$OUT_PATH_JOINER + python3 -m filter_joiner.filter_joiner_main + + # Combine data from Kafka and the archive + eval "Rscript ./flow.kfka.comb.R $KFKA_COMB_R_ARGS" + + # Run calibration conversion module + eval "Rscript ./flow.cal.conv.R $CAL_CONV_R_ARGS" \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-resource-request.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-resource-request.yaml new file mode 100644 index 0000000000..8ee5a1158f --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-resource-request.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-group-convert-resource-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + memory_request: "700Mi" + memory_limit: "1Gi" + cpu_request: "1" + cpu_limit: "1.25" \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-schemas.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-schemas.yaml new file mode 100644 index 0000000000..32e65ab3db --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-schemas.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: schemas-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + # Schema repository URLs and revisions + schema-repo-eng-url: git@github.com:BattelleEcology/neon-avro-schemas.git + schema-repo-eng-revision: develop + schema-repo-sci-url: https://github.com/NEONScience/NEON-IS-avro-schemas.git + schema-repo-sci-revision: master \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-upload-output-instructions.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-upload-output-instructions.yaml new file mode 100644 index 0000000000..0e22e11ffa --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/configmap-upload-output-instructions.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calibration-group-convert-upload-output-instructions-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-gcs-data:sha-9cf8a5b + upload-output-instructions.sh: | + #!/bin/bash + set -euo pipefail + IFS=$'\n\t' + + # Export data to bucket + if [[ -d "$OUT_PATH_DATA_UPLOAD" ]]; then + linkdir=$(mktemp -d) + shopt -s globstar + out_parquet_glob="${OUT_PATH_DATA_UPLOAD}/**/*.*" + echo "Linking output files to ${linkdir}" + for f in $out_parquet_glob; do + [[ "$f" =~ ^$OUT_PATH_DATA_UPLOAD/(.*)$ ]] + rel_path="${BASH_REMATCH[1]}" + fname="${rel_path##*/}" + rel_dir="${rel_path%/*}" + outdir="${linkdir}/${OUTPUT_BUCKET_PREFIX}/${rel_dir}" + mkdir -p "${outdir}" + ln -s "${f}" "${outdir}" + done + + echo "Syncing files to bucket" + rclone \ + --copy-links \ + --gcs-bucket-policy-only \ + copy \ + "${linkdir}/${OUTPUT_BUCKET_PREFIX}" \ + ":gcs://${OUTPUT_BUCKET_NAME}/${OUTPUT_BUCKET_PREFIX}" + + echo "Removing temporary files" + rm -rf $linkdir + + else + echo "No output found." + fi \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/base/kustomization.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/base/kustomization.yaml new file mode 100644 index 0000000000..09a141f820 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - calibration-group-and-convert.yaml + - configmap-load-data-instructions.yaml + - configmap-processing-instructions.yaml + - configmap-resource-request.yaml + - configmap-schemas.yaml + - configmap-upload-output-instructions.yaml \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-env.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-env.yaml new file mode 100644 index 0000000000..d6d6618c46 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-env.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aepg600m-calibration-group-convert-env-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + # Basic + LOG_LEVEL: DEBUG + ERR_PATH: /data/errored_datums + + # Environment vars for loading data + L0_BUCKET_NAME: neon-dev-l0-ingest + L0_BUCKET_VERSION_PATH: v2 # L0 loader has a known path structure. No path indices required + OUT_PATH: /data/DATA_PATH_ARCHIVE + CAL_BUCKET_NAME: neon-dev-argo-workflow-test # Temporary until structured cal bucket is set up + CAL_BUCKET_PREFIX: calibration/files # Temporary until structured cal bucket is set up + OUT_PATH_CAL: /data/calibration + OUT_PATH_SOURCE_TYPE_INDEX: "0" # output path index for source-type for calibration files, after OUT_PATH_CAL. Index is 0-based (0 is first path after OUT_PATH_CAL) + OUT_PATH_SOURCE_ID_INDEX: "1" + + # Arguments for calibration assignment + # Note: Env vars $CAL_ASGN_DATE_BEGIN and $CAL_ASGN_DATE_END and dynamically assigned based on the manifest dates + # See processing-instructions.sh + CAL_ASGN_R_ARGS: >- + "DirIn=$OUT_PATH_CAL" + "DirOut=/data/CALIBRATION_PATH" + "DirErr=$ERR_PATH" + "DateBgn=$CAL_ASGN_DATE_BEGIN" + "DateEnd=$CAL_ASGN_DATE_END" + + # Filter-joiner configuration for merging data and calibrations + CONFIG: | + --- + input_paths: + - path: + name: DATA_PATH_ARCHIVE + glob_pattern: /data/DATA_PATH_ARCHIVE/aepg600m/*/*/*/*/** + join_indices: [7] + outer_join: true + - path: + name: CALIBRATION_PATH + glob_pattern: /data/CALIBRATION_PATH/aepg600m/*/*/*/*/** + join_indices: [7] + outer_join: true + OUT_PATH_JOINER: /data/data_cal_joined + RELATIVE_PATH_INDEX: "3" + LINK_TYPE: SYMLINK + PARALLELISM_INTERNAL: "1" + + # R arguments for kafka combine step + OUT_PATH_KAFKA_COMB: /data/kafka_combined + KFKA_COMB_R_ARGS: >- + "DirIn=$OUT_PATH_JOINER" + "DirOut=$OUT_PATH_KAFKA_COMB" + "DirErr=$ERR_PATH" + "FileSchmL0=/data/schemas-eng/schemas/aepg600m/aepg600m.avsc" + "DirSubCopy=calibration" + + # R arguments for calibration conversion step + OUT_PATH_CAL_CONV: /data/aepg600m_calibration_group_and_convert + CAL_CONV_R_ARGS: >- + "DirIn=$OUT_PATH_KAFKA_COMB" + "DirOut=$OUT_PATH_CAL_CONV" + "DirErr=$ERR_PATH" + "FileSchmData=/data/schemas-sci/avro_schemas/aepg600m/aepg600m_calibrated.avsc" + "FileSchmQf=/data/schemas-sci/avro_schemas/aepg600m/flags_calibration_aepg600m.avsc" + "ConvFuncTerm1=def.cal.conv.poly.aepg600m:strain_gauge1_frequency_raw" + "ConvFuncTerm2=def.cal.conv.poly.aepg600m:strain_gauge2_frequency_raw" + "ConvFuncTerm3=def.cal.conv.poly.aepg600m:strain_gauge3_frequency_raw" + "TermQf=strain_gauge1_frequency_raw|strain_gauge2_frequency_raw|strain_gauge3_frequency_raw" + + # Environment vars for data upload + OUT_PATH_DATA_UPLOAD: /data/aepg600m_calibration_group_and_convert + OUTPUT_BUCKET_NAME: neon-dev-argo-workflow-test + OUTPUT_BUCKET_PREFIX: aepg600m_calibration_group_and_convert \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-resource-request.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-resource-request.yaml new file mode 100644 index 0000000000..91fe7d9f2d --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/configmap-resource-request.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aepg600m-calibration-group-convert-resource-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + memory_request: "500Mi" + memory_limit: "800Mi" + cpu_request: "1" + cpu_limit: "1.25" \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/kustomization.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/kustomization.yaml new file mode 100644 index 0000000000..ab9a59694f --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argo-workflows-dev + +resources: + - ../../base + - configmap-env.yaml + - configmap-resource-request.yaml + +patches: + - |- + apiVersion: argoproj.io/v1alpha1 + kind: WorkflowTemplate + metadata: + name: calibration-group-and-convert + spec: + arguments: + parameters: + - name: config-map-env-name + value: aepg600m-calibration-group-convert-env-config + - name: config-map-resource-request-name + value: aepg600m-calibration-group-convert-resource-config + +commonLabels: + sensor: aepg600m + workflow: calibration-group-and-convert \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-env.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-env.yaml new file mode 100644 index 0000000000..a0a1e6e698 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-env.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aepg600m-heated-calibration-group-convert-env-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + # Basic + LOG_LEVEL: DEBUG + ERR_PATH: /data/errored_datums + + # Environment vars for loading data + L0_BUCKET_NAME: neon-dev-l0-ingest + L0_BUCKET_VERSION_PATH: v2 # L0 loader has a known path structure. No path indices required + OUT_PATH: /data/DATA_PATH_ARCHIVE # Output path for L0 data + CAL_BUCKET_NAME: neon-dev-argo-workflow-test # Temporary until structured cal bucket is set up + CAL_BUCKET_PREFIX: calibration # Temporary until structured cal bucket is set up + OUT_PATH_CAL: /data/CALIBRATION_PATH + OUT_PATH_SOURCE_TYPE_INDEX: "0" # output path index for source-type for calibration files, after OUT_PATH_CAL. Index is 0-based (0 is first path after OUT_PATH_CAL) + OUT_PATH_SOURCE_ID_INDEX: "1" + + # Arguments for calibration assignment + # Note: Env vars $CAL_ASGN_DATE_BEGIN and $CAL_ASGN_DATE_END and dynamically assigned based on the manifest dates + # See processing-instructions.sh + CAL_ASGN_R_ARGS: >- + "DirIn=$OUT_PATH_CAL" + "DirOut=/data/CALIBRATION_PATH" + "DirErr=$ERR_PATH" + "DateBgn=$CAL_ASGN_DATE_BEGIN" + "DateEnd=$CAL_ASGN_DATE_END" + + # Filter-joiner configuration for merging data and calibrations + CONFIG: | + --- + input_paths: + - path: + name: DATA_PATH_ARCHIVE + glob_pattern: /data/DATA_PATH_ARCHIVE/aepg600m_heated/*/*/*/*/** + join_indices: [7] + outer_join: true + - path: + name: CALIBRATION_PATH + glob_pattern: /data/CALIBRATION_PATH/aepg600m_heated/*/*/*/*/** + join_indices: [7] + outer_join: true + OUT_PATH_JOINER: /data/data_cal_joined + RELATIVE_PATH_INDEX: "3" + LINK_TYPE: SYMLINK + PARALLELISM_INTERNAL: "2" + + # Arguments for kafka combine step + OUT_PATH_KAFKA_COMB: /data/kafka_combined + KFKA_COMB_R_ARGS: >- + "DirIn=$OUT_PATH_JOINER" + "DirOut=$OUT_PATH_KAFKA_COMB" + "DirErr=$ERR_PATH" + "FileSchmL0=/data/schemas-eng/schemas/aepg600m/aepg600m_heated.avsc" + "DirSubCopy=calibration" + + # Arguments for calibration conversion step + OUT_PATH_CAL_CONV: /data/aepg600m_heated_calibration_group_and_convert + CAL_CONV_R_ARGS: >- + "DirIn=$OUT_PATH_KAFKA_COMB" + "DirOut=$OUT_PATH_CAL_CONV" + "DirErr=$ERR_PATH" + "FileSchmData=/data/schemas-sci/avro_schemas/aepg600m/aepg600m_heated_calibrated.avsc" + "FileSchmQf=/data/schemas-sci/avro_schemas/aepg600m/flags_calibration_aepg600m.avsc" + "ConvFuncTerm1=def.cal.conv.poly.aepg600m:strain_gauge1_frequency_raw" + "ConvFuncTerm2=def.cal.conv.poly.aepg600m:strain_gauge2_frequency_raw" + "ConvFuncTerm3=def.cal.conv.poly.aepg600m:strain_gauge3_frequency_raw" + "TermQf=strain_gauge1_frequency_raw|strain_gauge2_frequency_raw|strain_gauge3_frequency_raw" + + # Environment vars for data upload + OUT_PATH_DATA_UPLOAD: /data/aepg600m_heated_calibration_group_and_convert + OUTPUT_BUCKET_NAME: neon-dev-argo-workflow-test + OUTPUT_BUCKET_PREFIX: aepg600m_heated_calibration_group_and_convert \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-resource-request.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-resource-request.yaml new file mode 100644 index 0000000000..5c54eefb62 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/configmap-resource-request.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aepg600m-heated-calibration-group-convert-resource-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + memory_request: "900Mi" + memory_limit: "1.5Gi" + cpu_request: "2.2" + cpu_limit: "3" \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/kustomization.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/kustomization.yaml new file mode 100644 index 0000000000..0ded5410db --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/aepg600m_heated/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argo-workflows-dev + +resources: + - ../../base + - configmap-env.yaml + - configmap-resource-request.yaml + +patches: + - |- + apiVersion: argoproj.io/v1alpha1 + kind: WorkflowTemplate + metadata: + name: calibration-group-and-convert + spec: + arguments: + parameters: + - name: config-map-env-name + value: aepg600m-heated-calibration-group-convert-env-config + - name: config-map-resource-request-name + value: aepg600m-heated-calibration-group-convert-resource-config + +commonLabels: + sensor: aepg600m_heated + workflow: calibration-group-and-convert \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-env.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-env.yaml new file mode 100644 index 0000000000..05210cdaa6 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-env.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cmp22-calibration-group-convert-env-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + # Basic + LOG_LEVEL: DEBUG + ERR_PATH: /data/errored_datums + + # Environment vars for loading data + L0_BUCKET_NAME: neon-dev-l0-ingest + L0_BUCKET_VERSION_PATH: v2 # L0 loader has a known path structure. No path indices required + OUT_PATH: /data/DATA_PATH_ARCHIVE + CAL_BUCKET_NAME: neon-dev-argo-workflow-test # Temporary until structured cal bucket is set up + CAL_BUCKET_PREFIX: calibration/files # Temporary until structured cal bucket is set up + OUT_PATH_CAL: /data/calibration + OUT_PATH_SOURCE_TYPE_INDEX: "0" # output path index for source-type for calibration files, after OUT_PATH_CAL. Index is 0-based (0 is first path after OUT_PATH_CAL) + OUT_PATH_SOURCE_ID_INDEX: "1" + + # Arguments for calibration assignment + # Note: Env vars $CAL_ASGN_DATE_BEGIN and $CAL_ASGN_DATE_END and dynamically assigned based on the manifest dates + # See processing-instructions.sh + CAL_ASGN_R_ARGS: >- + "DirIn=$OUT_PATH_CAL" + "DirOut=/data/CALIBRATION_PATH" + "DirErr=$ERR_PATH" + "DateBgn=$CAL_ASGN_DATE_BEGIN" + "DateEnd=$CAL_ASGN_DATE_END" + + # Filter-joiner configuration for merging data and calibrations + CONFIG: | + --- + input_paths: + - path: + name: DATA_PATH_ARCHIVE + glob_pattern: /data/DATA_PATH_ARCHIVE/cmp22/*/*/*/*/** + join_indices: [7] + outer_join: true + - path: + name: CALIBRATION_PATH + glob_pattern: /data/CALIBRATION_PATH/cmp22/*/*/*/*/** + join_indices: [7] + outer_join: true + OUT_PATH_JOINER: /data/data_cal_joined + RELATIVE_PATH_INDEX: "3" + LINK_TYPE: SYMLINK + PARALLELISM_INTERNAL: "2" + + # R arguments for kafka combine step + OUT_PATH_KAFKA_COMB: /data/kafka_combined + KFKA_COMB_R_ARGS: >- + "DirIn=$OUT_PATH_JOINER" + "DirOut=$OUT_PATH_KAFKA_COMB" + "DirErr=$ERR_PATH" + "FileSchmL0=/data/schemas-eng/schemas/cmp22/cmp22.avsc" + "DirSubCopy=calibration" + + # R arguments for calibration conversion step + OUT_PATH_CAL_CONV: /data/cmp22_calibration_group_and_convert + CAL_CONV_R_ARGS: >- + "DirIn=$OUT_PATH_KAFKA_COMB" + "DirOut=$OUT_PATH_CAL_CONV" + "DirErr=$ERR_PATH" + "FileSchmData=/data/schemas-sci/avro_schemas/cmp22/cmp22_calibrated.avsc" + "FileSchmQf=/data/schemas-sci/avro_schemas/cmp22/flags_calibration_cmp22.avsc" + "ConvFuncTerm1=def.cal.conv.poly:voltage" + "TermQf=voltage" + "UcrtFuncTerm1=def.ucrt.meas.mult:voltage" + "FileUcrtFdas=/data/schemas-sci/uncertainty_fdas/fdas_calibration_uncertainty_general.json" + + # Environment vars for data upload + OUT_PATH_DATA_UPLOAD: /data/cmp22_calibration_group_and_convert + OUTPUT_BUCKET_NAME: neon-dev-argo-workflow-test + OUTPUT_BUCKET_PREFIX: cmp22_calibration_group_and_convert \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-resource-request.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-resource-request.yaml new file mode 100644 index 0000000000..18b5d13605 --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/configmap-resource-request.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cmp22-calibration-group-convert-resource-config + labels: + workflows.argoproj.io/configmap-type: Parameter +data: + memory_request: "1.5Gi" + memory_limit: "3Gi" + cpu_request: "3.3" + cpu_limit: "4.5" \ No newline at end of file diff --git a/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/kustomization.yaml b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/kustomization.yaml new file mode 100644 index 0000000000..af7dad169e --- /dev/null +++ b/argo/generic_template_design/workflows/calibration-group-and-convert/overlays/cmp22/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argo-workflows-dev + +resources: + - ../../base + - configmap-env.yaml + - configmap-resource-request.yaml + +patches: + - |- + apiVersion: argoproj.io/v1alpha1 + kind: WorkflowTemplate + metadata: + name: calibration-group-and-convert + spec: + arguments: + parameters: + - name: config-map-env-name + value: cmp22-calibration-group-convert-env-config + - name: config-map-resource-request-name + value: cmp22-calibration-group-convert-resource-config + +commonLabels: + sensor: cmp22 + workflow: calibration-group-and-convert \ No newline at end of file diff --git a/argo/argo_ds_trino_containerset.yaml b/argo/test_workflows/argo_ds_trino_containerset.yaml similarity index 100% rename from argo/argo_ds_trino_containerset.yaml rename to argo/test_workflows/argo_ds_trino_containerset.yaml diff --git a/argo/argo_ds_trino_templateRef.yaml b/argo/test_workflows/argo_ds_trino_templateRef.yaml similarity index 100% rename from argo/argo_ds_trino_templateRef.yaml rename to argo/test_workflows/argo_ds_trino_templateRef.yaml diff --git a/argo/argo_ds_trino_templateRef_loopSite.yaml b/argo/test_workflows/argo_ds_trino_templateRef_loopSite.yaml similarity index 100% rename from argo/argo_ds_trino_templateRef_loopSite.yaml rename to argo/test_workflows/argo_ds_trino_templateRef_loopSite.yaml diff --git a/argo/argo_ds_trino_workflowTemplate.yaml b/argo/test_workflows/argo_ds_trino_workflowTemplate.yaml similarity index 100% rename from argo/argo_ds_trino_workflowTemplate.yaml rename to argo/test_workflows/argo_ds_trino_workflowTemplate.yaml diff --git a/argo/argo_ptb330a_data_source_trino.yaml b/argo/test_workflows/argo_ptb330a_data_source_trino.yaml similarity index 100% rename from argo/argo_ptb330a_data_source_trino.yaml rename to argo/test_workflows/argo_ptb330a_data_source_trino.yaml diff --git a/argo/argo_ptb330a_data_source_trino_PVmount.yaml b/argo/test_workflows/argo_ptb330a_data_source_trino_PVmount.yaml similarity index 100% rename from argo/argo_ptb330a_data_source_trino_PVmount.yaml rename to argo/test_workflows/argo_ptb330a_data_source_trino_PVmount.yaml diff --git a/argo/argo_ptb330a_data_source_trino_emptyDir.yaml b/argo/test_workflows/argo_ptb330a_data_source_trino_emptyDir.yaml similarity index 100% rename from argo/argo_ptb330a_data_source_trino_emptyDir.yaml rename to argo/test_workflows/argo_ptb330a_data_source_trino_emptyDir.yaml diff --git a/argo/argo_ptb330a_ds_trino_containerset.yaml b/argo/test_workflows/argo_ptb330a_ds_trino_containerset.yaml similarity index 100% rename from argo/argo_ptb330a_ds_trino_containerset.yaml rename to argo/test_workflows/argo_ptb330a_ds_trino_containerset.yaml diff --git a/argo/call-which-template.yaml b/argo/test_workflows/call-which-template.yaml similarity index 100% rename from argo/call-which-template.yaml rename to argo/test_workflows/call-which-template.yaml diff --git a/argo/cmp22_calibration_group_and_convert.yaml b/argo/test_workflows/cmp22_calibration_group_and_convert.yaml similarity index 100% rename from argo/cmp22_calibration_group_and_convert.yaml rename to argo/test_workflows/cmp22_calibration_group_and_convert.yaml diff --git a/argo/cmp22_calibration_group_and_convert_chainNext.yaml b/argo/test_workflows/cmp22_calibration_group_and_convert_chainNext.yaml similarity index 100% rename from argo/cmp22_calibration_group_and_convert_chainNext.yaml rename to argo/test_workflows/cmp22_calibration_group_and_convert_chainNext.yaml diff --git a/argo/cmp22_calibration_group_and_convert_loopEachManifest.yaml b/argo/test_workflows/cmp22_calibration_group_and_convert_loopEachManifest.yaml similarity index 100% rename from argo/cmp22_calibration_group_and_convert_loopEachManifest.yaml rename to argo/test_workflows/cmp22_calibration_group_and_convert_loopEachManifest.yaml diff --git a/argo/cmp22_calibration_group_and_convert_manifest.yaml b/argo/test_workflows/cmp22_calibration_group_and_convert_manifest.yaml similarity index 100% rename from argo/cmp22_calibration_group_and_convert_manifest.yaml rename to argo/test_workflows/cmp22_calibration_group_and_convert_manifest.yaml diff --git a/argo/cmp22_calibration_through_fill_date_gaps.yaml b/argo/test_workflows/cmp22_calibration_through_fill_date_gaps.yaml similarity index 100% rename from argo/cmp22_calibration_through_fill_date_gaps.yaml rename to argo/test_workflows/cmp22_calibration_through_fill_date_gaps.yaml diff --git a/argo/cmp22_location_group_and_restructure.yaml b/argo/test_workflows/cmp22_location_group_and_restructure.yaml similarity index 100% rename from argo/cmp22_location_group_and_restructure.yaml rename to argo/test_workflows/cmp22_location_group_and_restructure.yaml diff --git a/argo/configmap-cmp22-next-workflows.yaml b/argo/test_workflows/configmap-cmp22-next-workflows.yaml similarity index 100% rename from argo/configmap-cmp22-next-workflows.yaml rename to argo/test_workflows/configmap-cmp22-next-workflows.yaml diff --git a/argo/configmap-sitelist-small.yaml b/argo/test_workflows/configmap-sitelist-small.yaml similarity index 100% rename from argo/configmap-sitelist-small.yaml rename to argo/test_workflows/configmap-sitelist-small.yaml diff --git a/argo/configmap-sitelist.yaml b/argo/test_workflows/configmap-sitelist.yaml similarity index 100% rename from argo/configmap-sitelist.yaml rename to argo/test_workflows/configmap-sitelist.yaml diff --git a/argo/configmap-sitelist.yaml.tmpl b/argo/test_workflows/configmap-sitelist.yaml.tmpl similarity index 100% rename from argo/configmap-sitelist.yaml.tmpl rename to argo/test_workflows/configmap-sitelist.yaml.tmpl diff --git a/argo/eventsource-transition-workflow-completions.yaml b/argo/test_workflows/eventsource-transition-workflow-completions.yaml similarity index 100% rename from argo/eventsource-transition-workflow-completions.yaml rename to argo/test_workflows/eventsource-transition-workflow-completions.yaml diff --git a/argo/location_asset_template.yaml b/argo/test_workflows/location_asset_template.yaml similarity index 100% rename from argo/location_asset_template.yaml rename to argo/test_workflows/location_asset_template.yaml diff --git a/argo/my-shared-template.yaml b/argo/test_workflows/my-shared-template.yaml similarity index 100% rename from argo/my-shared-template.yaml rename to argo/test_workflows/my-shared-template.yaml diff --git a/argo/sensor-log-everything.yaml b/argo/test_workflows/sensor-log-everything.yaml similarity index 100% rename from argo/sensor-log-everything.yaml rename to argo/test_workflows/sensor-log-everything.yaml diff --git a/argo/sensor-upstream-workflows-succeeded.yaml b/argo/test_workflows/sensor-upstream-workflows-succeeded.yaml similarity index 100% rename from argo/sensor-upstream-workflows-succeeded.yaml rename to argo/test_workflows/sensor-upstream-workflows-succeeded.yaml diff --git a/argo/submit_workflow_with_datum_manifest.yaml b/argo/test_workflows/submit_workflow_with_datum_manifest.yaml similarity index 100% rename from argo/submit_workflow_with_datum_manifest.yaml rename to argo/test_workflows/submit_workflow_with_datum_manifest.yaml diff --git a/argo/test_bucket_input_a.yaml b/argo/test_workflows/test_bucket_input_a.yaml similarity index 100% rename from argo/test_bucket_input_a.yaml rename to argo/test_workflows/test_bucket_input_a.yaml diff --git a/argo/test_bucket_input_b.yaml b/argo/test_workflows/test_bucket_input_b.yaml similarity index 100% rename from argo/test_bucket_input_b.yaml rename to argo/test_workflows/test_bucket_input_b.yaml diff --git a/argo/testInputs.R b/argo/utilities/testInputs.R similarity index 100% rename from argo/testInputs.R rename to argo/utilities/testInputs.R diff --git a/flow/flow.cal.asgn/flow.cal.asgn.R b/flow/flow.cal.asgn/flow.cal.asgn.R index c42dc9d621..2aaf98a728 100644 --- a/flow/flow.cal.asgn/flow.cal.asgn.R +++ b/flow/flow.cal.asgn/flow.cal.asgn.R @@ -59,6 +59,14 @@ #' 2020 #' 2021 #' +#' 4.a "DateBgn=value" (optional, combined with DateEnd is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the start date (inclusive) to assign calibrations. This input will +#' be ignored if FileYear is specified. +#' +#' 4.b "DateEnd=value" (optional, combined with DateBgn is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the end date (NON-inclusive) to assign calibrations. This input will +#' be ignored if FileYear is specified. +#' #' 5. "PadDay=value" (optional), where value contains the integer days to include applicable #' calibration files before/after a given data day. A negative value will copy in the calibration file(s) #' that are applicable to the given data day AND # number of days before the data day. A positive value @@ -102,6 +110,8 @@ # add support for array variables/calibrations that utilize multiple stream IDs for the same term # Cove Sturtevant (2021-08-26) # Add datum error routing +# Cove Sturtevant (2026-07-28) +# Add option to specify date range instaed of reading date year from file ############################################################################################## library(foreach) library(doParallel) @@ -133,8 +143,12 @@ log$debug(paste0(numCoreUse, ' of ',numCoreAvail, ' available cores will be used Para <- NEONprocIS.base::def.arg.pars( arg = arg, - NameParaReqd = c("DirIn", "DirOut","DirErr","FileYear"), - NameParaOptn = c("PadDay","Arry"), + NameParaReqd = c("DirIn", "DirOut","DirErr"), + NameParaOptn = c("FileYear", + "DateBgn", + "DateEnd", + "PadDay", + "Arry"), ValuParaOptn = base::list(PadDay=0, Arry=FALSE), TypePara = base::list(PadDay="integer", @@ -148,14 +162,35 @@ log$debug(base::paste0('Output directory: ', Para$DirOut)) log$debug(base::paste0('Error directory: ', Para$DirErr)) # Parse the file containing the years to populate -log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) -yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) -if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ - log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) - stop() +if(base::length(Para$FileYear) > 0){ + log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) + yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) + if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ + log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) + stop() + } + timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') + timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') +} else { + log$debug(base::paste0('File containing data years to populate not found. Using specified DateBgn: ', + Para$DateBgn,' and DateEnd: ', + Para$DateEnd)) + timeBgn <- base::as.POSIXct(Para$DateBgn,tz='GMT') + timeEnd <- base::as.POSIXct(Para$DateEnd,tz='GMT') + + if(base::length(timeBgn) != 1 || is.na(timeBgn)){ + log$fatal(base::paste0('Cannot determine start date from input DateBgn. Check parameter.')) + stop() + } + if(base::length(timeEnd) != 1 || is.na(timeEnd)){ + log$fatal(base::paste0('Cannot determine end date from input DateEnd. Check parameter.')) + stop() + } + if(timeBgn > timeEnd){ + log$fatal(base::paste0('DateBgn is after DateEnd... illogical! Check input parameters.')) + stop() + } } -timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') -timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') # Parse the days to pad if(base::length(Para$PadDay) == 1 && !base::is.na(Para$PadDay)){ diff --git a/flow/flow.cal.asgn/wrap.cal.asgn.R b/flow/flow.cal.asgn/wrap.cal.asgn.R index 6c05b9a357..b03323e4ee 100644 --- a/flow/flow.cal.asgn/wrap.cal.asgn.R +++ b/flow/flow.cal.asgn/wrap.cal.asgn.R @@ -28,11 +28,15 @@ #' Input path = /scratch/pfs/proc_group/prt/27134/resistenace \cr #' #' @param DirOutBase Character value. The output path that will replace the #/pfs/BASE_REPO portion of DirIn. +#' #' @param DirErrBase (optional) Character value. The output path for errored datums that will replace the #' #/pfs/BASE_REPO portion of DirIn. Default is a directory named "errored_datums" appended to the end of DirOutBase. #' Paths to calibration files that failed for any reason will be placed in this directory. +#' #' @param TimeBgn POSIX. The minimum date for which to assign calibration files. +#' #' @param TimeEnd POSIX. The maximum date for which to assign calibration files (non-inclusive). +#' #' @param PadDay (optional). 2-element difftime object with units of days indicating the days to include applicable #' calibration files before/after a given data day. A negative value will copy in the calibration file(s) #' that are applicable to the given data day AND # number of days before the data day. A positive value @@ -41,6 +45,7 @@ #' that are applicable between 2019-01-13 00:00 and 2019-01-15 24:00. "PadDay=2" will copy in calibration file(s) #' that are applicable between 2019-01-15 00:00 and 2019-01-17 24:00. To provide both negative and positive pads #' (a window around a given day), separate the values with pipes (e.g. "PadDay=-2|2"). +#' #' @param Arry (optional). Logical value indicating whether the calibration files should be assigned separately for #' each stream ID. (Normally there would be a single stream ID corresponding to a given TERM, so no checking for #' multiple stream IDs is needed). A value of TRUE would be needed if the TERM is an array, meaning that calibrations @@ -48,6 +53,7 @@ #' (one for each stream ID). This is relatively rare, but is the case for e.g. the tchain source type. The default is #' FALSE, but there is also really no harm in setting it to TRUE unless the stream ID for a SOURCE_ID changes over #' its lifetime. +#' #' @param log A logger object as produced by NEONprocIS.base::def.log.init to produce structured log #' output. Defaults to NULL, in which the logger will be created and used within the function. #' diff --git a/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R b/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R index 622cf21625..db2c2f25b2 100644 --- a/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R +++ b/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R @@ -52,6 +52,14 @@ #' 2019 #' 2020 #' 2021 +#' +#' 4.a "DateBgn=value" (optional, combined with DateEnd is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the start date (inclusive) to assign files. This input will +#' be ignored if FileYear is specified. +#' +#' 4.b "DateEnd=value" (optional, combined with DateBgn is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the end date (NON-inclusive) to assign files. This input will +#' be ignored if FileYear is specified. #' #' 5. "TypeFile=value", where value is the type of file. #' Options are 'asset', 'namedLocation', and 'group'. Only one may be specified. 'asset' corresponds to a @@ -99,6 +107,8 @@ # Allow good records to pass through, while removing bad records and routing to errored datums # Cove Sturtevant (2025-03-20) # Accommodate location_properties: syntax for Prop argument (see def.loc.filt) +# Cove Sturtevant (2026-07-29) +# Add option to specify date range instead of reading date year from file ############################################################################################## library(foreach) library(doParallel) @@ -130,8 +140,11 @@ log$debug(paste0(numCoreUse, ' of ',numCoreAvail, ' available cores will be used Para <- NEONprocIS.base::def.arg.pars( arg = arg, - NameParaReqd = c("DirIn", "DirOut","DirErr","FileYear","TypeFile"), - NameParaOpt = c('Prop'), + NameParaReqd = c("DirIn", "DirOut","DirErr","TypeFile"), + NameParaOpt = c("FileYear", + "DateBgn", + "DateEnd", + "Prop"), ValuParaOpt = list(Prop = 'all'), log = log ) @@ -151,14 +164,35 @@ log$debug(base::paste0('Error directory: ', Para$DirErr)) log$debug(base::paste0('Properties to retain: ', base::paste0(Para$Prop,collapse=','))) # Parse the file containing the years to populate -log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) -yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) -if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ - log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) - stop() +if(base::length(Para$FileYear) > 0){ + log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) + yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) + if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ + log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) + stop() + } + timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') + timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') +} else { + log$debug(base::paste0('File containing data years to populate not found. Using specified DateBgn: ', + Para$DateBgn,' and DateEnd: ', + Para$DateEnd)) + timeBgn <- base::as.POSIXct(Para$DateBgn,tz='GMT') + timeEnd <- base::as.POSIXct(Para$DateEnd,tz='GMT') + + if(base::length(timeBgn) != 1 || is.na(timeBgn)){ + log$fatal(base::paste0('Cannot determine start date from input DateBgn. Check parameter.')) + stop() + } + if(base::length(timeEnd) != 1 || is.na(timeEnd)){ + log$fatal(base::paste0('Cannot determine end date from input DateEnd. Check parameter.')) + stop() + } + if(timeBgn > timeEnd){ + log$fatal(base::paste0('DateBgn is after DateEnd... illogical! Check input parameters.')) + stop() + } } -timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') -timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') # Check that TypeFile is one of 'asset', 'namedLocation', or 'group' log$debug(base::paste0('Type of location files: ', Para$TypeFile)) diff --git a/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R b/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R index 1bd36950d0..6ed242f04a 100644 --- a/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R +++ b/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R @@ -22,8 +22,11 @@ #' Input path = /scratch/pfs/proc_group/prt/27134 \cr #' #' @param DirOutBase Character value. The output path that will replace the #/pfs/BASE_REPO portion of DirIn. +#' #' @param TimeBgn POSIX. The minimum date for which to assign location files. +#' #' @param TimeEnd POSIX. The maximum date for which to assign location files. +#' #' @param TypeFile String value. The type of file that is being distributed and filtered. #' Options are 'asset', 'namedLocation', and 'group'. Only one may be specified. 'asset' corresponds to a #' location file for a particular asset, which includes information about where and for how long @@ -31,9 +34,11 @@ #' location file specific to a named location, including the properties of that named location and #' the dates over which it was active (should have been producing data). 'group' corresponds to a group #' file specific to a group member, including what groups the member is in and properties of the group. +#' #' @param Prop character vector of the properties in the file to retain. The meaning of this input changes #' according to TypeFile. See the filtering functions for each type for details. Defaults to 'all'. Currently #' relevant for TypeFile='asset' and TypeFile='namedLocation'. +#' #' @param log A logger object as produced by NEONprocIS.base::def.log.init to produce structured log #' output. Defaults to NULL, in which the logger will be created and used within the function. #' diff --git a/flow/flow.srf.asgn/flow.srf.asgn.R b/flow/flow.srf.asgn/flow.srf.asgn.R index e27c6cd0c8..f09c8cf8a1 100644 --- a/flow/flow.srf.asgn/flow.srf.asgn.R +++ b/flow/flow.srf.asgn/flow.srf.asgn.R @@ -52,6 +52,14 @@ #' 2020 #' 2021 #' +#' 4.a "DateBgn=value" (optional, combined with DateEnd is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the start date (inclusive) to assign files. This input will +#' be ignored if FileYear is specified. +#' +#' 4.b "DateEnd=value" (optional, combined with DateBgn is an alternative to FileYear), where value is a date +#' string formatted "YYYY-mm-dd" specifying the end date (NON-inclusive) to assign files. This input will +#' be ignored if FileYear is specified. +#' #' Note: This script implements optional parallelization as well as logging (described in #' \code{\link[NEONprocIS.base]{def.log.init}}), both of which use system environment variables if available. @@ -76,6 +84,8 @@ # original creation, refactored from flow.loc.grp.asgn # Cove Sturtevant (2024-11-27) # Allow good records to pass through, while removing bad records and routing to errored datums +# Cove Sturtevant (2026-07-29) +# Add option to specify date range instead of reading date year from file ############################################################################################## library(foreach) library(doParallel) @@ -107,7 +117,10 @@ log$debug(paste0(numCoreUse, ' of ',numCoreAvail, ' available cores will be used Para <- NEONprocIS.base::def.arg.pars( arg = arg, - NameParaReqd = c("DirIn", "DirOut","DirErr","FileYear"), + NameParaReqd = c("DirIn", "DirOut","DirErr"), + NameParaOptn = c("FileYear", + "DateBgn", + "DateEnd"), log = log ) @@ -117,14 +130,35 @@ log$debug(base::paste0('Output directory: ', Para$DirOut)) log$debug(base::paste0('Error directory: ', Para$DirErr)) # Parse the file containing the years to populate -log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) -yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) -if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ - log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) - stop() +if(base::length(Para$FileYear) > 0){ + log$debug(base::paste0('File containing data years to populate: ', Para$FileYear)) + yearFill <- base::as.integer(base::readLines(con=Para$FileYear)) + if(base::length(yearFill) == 0 || base::any(base::is.na(yearFill))){ + log$fatal(base::paste0('Cannot determine years to populate from file: ', Para$FileYear,'. Check file contents.')) + stop() + } + timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') + timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') +} else { + log$debug(base::paste0('File containing data years to populate not found. Using specified DateBgn: ', + Para$DateBgn,' and DateEnd: ', + Para$DateEnd)) + timeBgn <- base::as.POSIXct(Para$DateBgn,tz='GMT') + timeEnd <- base::as.POSIXct(Para$DateEnd,tz='GMT') + + if(base::length(timeBgn) != 1 || is.na(timeBgn)){ + log$fatal(base::paste0('Cannot determine start date from input DateBgn. Check parameter.')) + stop() + } + if(base::length(timeEnd) != 1 || is.na(timeEnd)){ + log$fatal(base::paste0('Cannot determine end date from input DateEnd. Check parameter.')) + stop() + } + if(timeBgn > timeEnd){ + log$fatal(base::paste0('DateBgn is after DateEnd... illogical! Check input parameters.')) + stop() + } } -timeBgn <- base::as.POSIXct(x=paste0(min(yearFill),'-01-01'),tz='GMT') -timeEnd <- base::as.POSIXct(x=paste0(max(yearFill)+1,'-01-01'),tz='GMT') # Find all the input paths (terminal directories). We will process each one. DirIn <- diff --git a/flow/flow.srf.asgn/wrap.srf.asgn.R b/flow/flow.srf.asgn/wrap.srf.asgn.R index 227652ab5a..79f4234505 100644 --- a/flow/flow.srf.asgn/wrap.srf.asgn.R +++ b/flow/flow.srf.asgn/wrap.srf.asgn.R @@ -22,8 +22,11 @@ #' Input path = /scratch/pfs/proc_group/surfacewater-physical_PRLA130100 \cr #' #' @param DirOutBase Character value. The output path that will replace the #/pfs/BASE_REPO portion of DirIn. +#' #' @param TimeBgn POSIX. The minimum date for which to assign location files. +#' #' @param TimeEnd POSIX. The maximum date for which to assign location files. +#' #' @param log A logger object as produced by NEONprocIS.base::def.log.init to produce structured log #' output. Defaults to NULL, in which the logger will be created and used within the function. #' diff --git a/modules/gcs_data/Dockerfile b/modules/gcs_data/Dockerfile index 7dfa4ef2ec..51ed41752f 100644 --- a/modules/gcs_data/Dockerfile +++ b/modules/gcs_data/Dockerfile @@ -5,34 +5,56 @@ # docker build -t neon-is-gcs-data -f ./modules/gcs_data/Dockerfile . # ### -# Use ubi9 as the base image -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7 +# Use Ubuntu as the base image +FROM ubuntu:24.04 ARG MODULE_DIR="./modules" +ARG APP_DIR="gcs_data" ARG CONTAINER_APP_DIR="/usr/src/app" +ARG DEBIAN_FRONTEND=noninteractive +ARG PYTHON_VERSION=3.13 # For rclone ARG TARGETPLATFORM ARG RCLONE_VERSION=v1.72.1 -# Install system dependencies -RUN microdnf update -y --disableplugin=subscription-manager && \ - microdnf install -y --disableplugin=subscription-manager \ - jq \ - findutils \ - gzip \ - libzstd \ - shadow-utils \ - tar && \ - microdnf clean all --disableplugin=subscription-manager &&\ +ENV PYTHON_BIN="python${PYTHON_VERSION}" + +# Install system dependencies and Python 3.13 from deadsnakes +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + findutils \ + gnupg \ + gzip \ + jq \ + libzstd1 \ + passwd \ + software-properties-common \ + tar \ + unzip && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + ${PYTHON_BIN} \ + ${PYTHON_BIN}-venv && \ + curl -sS https://bootstrap.pypa.io/get-pip.py | ${PYTHON_BIN} && \ + ${PYTHON_BIN} -m pip install --no-cache-dir --upgrade pip setuptools wheel && \ + apt-get purge -y --auto-remove software-properties-common gnupg && \ + rm -rf /var/lib/apt/lists/* && \ groupadd appuser -g 1001 && \ - useradd appuser -d ${CONTAINER_APP_DIR} -g appuser -u 1001 &&\ - rm -rf ${CONTAINER_APP_DIR}/venv + useradd appuser -d ${CONTAINER_APP_DIR} -g appuser -u 1001 && \ + rm -rf ${CONTAINER_APP_DIR}/venv && \ + ln -s /usr/bin/${PYTHON_BIN} /usr/bin/python3 # Install yq (v4.x example) for manipulating json ENV YQ_VERSION=v4.44.1 -RUN curl -L "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ - -o /usr/bin/yq && \ +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then YQ_ARCH=amd64; \ + elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then YQ_ARCH=arm64; \ + else YQ_ARCH=amd64; fi && \ + curl -L "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${YQ_ARCH}" \ + -o /usr/bin/yq && \ chmod +x /usr/bin/yq @@ -40,9 +62,20 @@ RUN curl -L "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; \ elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; \ else ARCHITECTURE=amd64; fi &&\ - rpm -Uvh "https://github.com/rclone/rclone/releases/download/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-${ARCHITECTURE}.rpm" + curl -L "https://github.com/rclone/rclone/releases/download/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-${ARCHITECTURE}.zip" \ + -o /tmp/rclone.zip && \ + unzip /tmp/rclone.zip -d /tmp && \ + cp "/tmp/rclone-${RCLONE_VERSION}-linux-${ARCHITECTURE}/rclone" /usr/bin/rclone && \ + chmod +x /usr/bin/rclone && \ + rm -rf /tmp/rclone* WORKDIR ${CONTAINER_APP_DIR} +COPY ${MODULE_DIR}/${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/requirements.txt +RUN ${PYTHON_BIN} -m pip install --no-cache-dir -r ${CONTAINER_APP_DIR}/requirements.txt + +COPY ${MODULE_DIR}/${APP_DIR} ${CONTAINER_APP_DIR} +COPY ${MODULE_DIR}/common ${CONTAINER_APP_DIR}/common + USER appuser \ No newline at end of file diff --git a/modules/gcs_data/l0_gcs_loader_by_manifest.py b/modules/gcs_data/l0_gcs_loader_by_manifest.py new file mode 100644 index 0000000000..0189ab8beb --- /dev/null +++ b/modules/gcs_data/l0_gcs_loader_by_manifest.py @@ -0,0 +1,242 @@ + +"""Load L0 parquet files from GCS using manifest-driven record selectors. + +Manifest input can be provided by either: +1. MANIFEST: JSON-formatted string. +2. MANIFEST_FILE: Path to a file containing JSON. + +If both are set, MANIFEST is used. + +Accepted JSON format: a JSON array of objects. Each object must contain the +keys "source_type" and "data_date". The optional "source_id" key narrows the +query to a specific source; when omitted it is wildcarded. Any additional keys +in a record are ignored. + +The "data_date" value may be: +- "YYYY-mm-dd": filters to an exact day. +- "YYYY-mm": filters to a month (day wildcarded). +- "YYYY": filters to a year (month and day wildcarded). + +Example manifest: +[ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10-02", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10"}, + {"source_type": "cmp22", "data_date": "2026"} +] + +When source_id is present, the bucket prefix includes: +{L0_BUCKET_VERSION_PATH}/{source_type}/ms={download_year}-{download_month}/source_id={source_id} +""" + +from google.cloud import storage +from pathlib import Path +import environs +import os +import sys +import re +import json +from datetime import datetime + +import structlog +import common.log_config as log_config + + +def _parse_manifest_data(manifest_data: object, log) -> list[dict]: + if not isinstance(manifest_data, list): + log.error('Invalid manifest format', manifest_type=type(manifest_data).__name__) + sys.exit('Manifest must be a JSON array of objects with "source_type" and "data_date" keys.') + + records = [] + for i, record in enumerate(manifest_data): + if not isinstance(record, dict): + log.warning('Skipping non-object manifest entry', index=i) + continue + if 'source_type' not in record: + log.warning('Skipping manifest record missing required key "source_type"', index=i) + continue + if 'data_date' not in record: + log.warning('Skipping manifest record missing required key "data_date"', index=i) + continue + records.append(record) + return records + + +def l0_gcs_loader_by_manifest() -> None: + + env = environs.Env() + log_level: str = env.log_level('LOG_LEVEL', 'INFO') + log_config.configure(log_level) + log = structlog.get_logger() + + ingest_bucket_name = env.str('L0_BUCKET_NAME') + bucket_version_path = env.str('L0_BUCKET_VERSION_PATH') + source_type_out = env.str('SOURCE_TYPE_OUT', None) + manifest_inline = env.str('MANIFEST', None) + manifest_file_raw = env.str('MANIFEST_FILE', None) + output_directory: Path = env.path('OUT_PATH') + + log.debug('Configuration loaded', + bucket_name=ingest_bucket_name, + output_directory=output_directory) + + if manifest_inline and manifest_inline.strip(): + try: + manifest_data = json.loads(manifest_inline) + log.debug('Manifest loaded from MANIFEST environment variable') + except json.JSONDecodeError as exc: + log.error('Invalid JSON in MANIFEST', error=str(exc)) + sys.exit(f'Invalid JSON in MANIFEST: {exc}') + manifest_records = _parse_manifest_data(manifest_data, log) + else: + if not manifest_file_raw: + log.error('One of MANIFEST or MANIFEST_FILE environment variables is required') + sys.exit('One of MANIFEST or MANIFEST_FILE environment variables is required.') + manifest_file = Path(manifest_file_raw) + if not manifest_file.exists(): + log.error('MANIFEST_FILE does not exist', manifest_file=str(manifest_file)) + sys.exit(f'MANIFEST_FILE does not exist: {manifest_file}') + + log.debug('Loading manifest from file', manifest_file=str(manifest_file)) + with open(manifest_file, 'r', encoding='utf-8') as manifest_handle: + try: + manifest_data = json.load(manifest_handle) + log.debug('Manifest loaded from file', manifest_file=str(manifest_file)) + except json.JSONDecodeError as exc: + log.error('Invalid JSON in MANIFEST_FILE', manifest_file=str(manifest_file), error=str(exc)) + sys.exit(f'Invalid JSON in MANIFEST_FILE {manifest_file}: {exc}') + manifest_records = _parse_manifest_data(manifest_data, log) + + if not manifest_records: + log.warning('No valid records found in MANIFEST input') + return + + log.info('Processing manifest records', record_count=len(manifest_records)) + + storage_client = storage.Client() + ingest_bucket = storage_client.bucket(ingest_bucket_name) + log.debug('Connected to GCS bucket', bucket_name=ingest_bucket_name) + + def parse_blob_metadata(blob_name: str) -> tuple[str | None, str | None, str | None, str | None]: + blob_pattern = re.compile( + rf"^{re.escape(bucket_version_path)}/([^/]+)/ms=(\d{{4}})-(\d{{2}})/source_id=([^/]+)/" + ) + match = blob_pattern.search(blob_name) + if not match: + return None, None, None, None + return match.group(1), match.group(2), match.group(3), match.group(4) + + downloaded_blob_names = set() + + for record in manifest_records: + source_type = record['source_type'] + data_date = record['data_date'] + manifest_source_id = record.get('source_id', None) + + log.debug('Processing manifest record', source_type=source_type, data_date=data_date, source_id=manifest_source_id) + + date_parts = data_date.split('-') + download_year = date_parts[0] if len(date_parts) >= 1 else None + download_month = date_parts[1] if len(date_parts) >= 2 else None + download_day = date_parts[2] if len(date_parts) >= 3 else None + + prefix = bucket_version_path + if source_type: + prefix = f"{prefix}/{source_type}" + if download_year and download_month: + prefix = f"{prefix}/ms={download_year}-{download_month}" + + prefixes = [prefix] + if manifest_source_id: + normalized_source_id = manifest_source_id.replace('source_id=', '') + prefixes = [f"{prefix}/source_id={normalized_source_id}"] + + files_downloaded_for_path = 0 + for list_prefix in prefixes: + for blob in ingest_bucket.list_blobs(prefix=list_prefix): + if blob.name in downloaded_blob_names: + continue + + file_path_bucket = os.path.splitext(blob.name)[0] + file_name_bucket = re.split('/', file_path_bucket)[-1] + + file_date_match = re.search(r'[0-9]{4}-[0-1]{1}[0-9]{1}-[0-3]{1}[0-9]{1}', file_name_bucket) + if not file_date_match: + continue + + file_date = file_date_match.group(0) + file_date_parts = file_date.split('-') + file_year = file_date_parts[0] + file_month = file_date_parts[1] + file_day = file_date_parts[2] + + if download_year and download_year != file_year: + continue + if download_month and download_month != file_month: + continue + if download_day and download_day != file_day: + continue + + blob_source_type, blob_year, blob_month, blob_source_id = parse_blob_metadata(blob.name) + if blob_year and download_year and blob_year != download_year: + continue + if blob_month and download_month and blob_month != download_month: + continue + if blob_source_id and manifest_source_id and blob_source_id != manifest_source_id.replace('source_id=', ''): + continue + + if download_day: + if not download_year or not download_month: + continue + trigger_date = datetime(int(download_year), int(download_month), int(download_day)) + bucket_file_date = datetime(int(file_year), int(file_month), int(file_day)) + if trigger_date != bucket_file_date: + continue + + resolved_source_type = source_type_out or source_type or blob_source_type + resolved_source_id = blob_source_id or manifest_source_id + if resolved_source_id is not None: + resolved_source_id = resolved_source_id.replace('source_id=', '') + + if not resolved_source_type or not resolved_source_id: + continue + + file_name = file_name_bucket + '.parquet' + file_path = Path( + output_directory, + resolved_source_type, + file_year, + file_month, + file_day, + resolved_source_id, + 'data', + file_name, + ) + + log.debug('Downloading file to local path', + file_path=str(file_path), + blob_name=blob.name, + resolved_source_type=resolved_source_type, + resolved_source_id=resolved_source_id, + file_date=file_date) + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'wb') as l0_data_file: + l0_data_file.write(blob.download_as_bytes()) + log.info('File downloaded successfully', + file_path=str(file_path), + blob_name=blob.name) + + downloaded_blob_names.add(blob.name) + files_downloaded_for_path += 1 + + if files_downloaded_for_path == 0: + log.warning('No files found in bucket for manifest record', + source_type=source_type, + data_date=data_date, + source_id=manifest_source_id, + bucket_prefix=prefixes[0] if prefixes else 'N/A') + + log.info('Manifest processing completed', total_files_downloaded=len(downloaded_blob_names)) + +if __name__ == '__main__': + l0_gcs_loader_by_manifest() diff --git a/modules/gcs_data/manifest_paths_builder.py b/modules/gcs_data/manifest_paths_builder.py new file mode 100644 index 0000000000..e3f948f8ee --- /dev/null +++ b/modules/gcs_data/manifest_paths_builder.py @@ -0,0 +1,228 @@ +"""Build path selectors from a JSON manifest. + +Manifest input can be provided by either: +1. MANIFEST: JSON-formatted string. +2. MANIFEST_FILE: Path to a file containing JSON. + +If both are set, MANIFEST is used. + +Accepted manifest format: a JSON array of objects. Each object must contain the +keys "source_type" and "data_date". The optional "source_id" key narrows the +query to a specific source; when omitted it is wildcarded. Any additional keys +in a record are ignored. + +The "data_date" value may be: +- "YYYY-mm-dd": filters to an exact day. +- "YYYY-mm": filters to a month. +- "YYYY": filters to a year. + +When data_date is truncated, any larger PATH indices are ignored. + +Example manifest: +[ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10-02", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-11"}, + {"source_type": "cmp22", "data_date": "2026"} +] + +Output path segment order is controlled by environment variables that map fields to +0-based indexes in each output path. Any subset of these variables may be set; only +configured components are included in output paths: +- OUT_PATH_SOURCE_TYPE_INDEX +- OUT_PATH_YEAR_INDEX +- OUT_PATH_MONTH_INDEX +- OUT_PATH_DAY_INDEX +- OUT_PATH_SOURCE_ID_INDEX + +For example, if: +OUT_PATH_SOURCE_TYPE_INDEX=0 +OUT_PATH_YEAR_INDEX=1 +OUT_PATH_MONTH_INDEX=2 +OUT_PATH_DAY_INDEX=3 +OUT_PATH_SOURCE_ID_INDEX=4 + +Then: +{ + "paths": ["source_type/year/month/day/source_id", ...] +} + +So, for the example manifest, the output would be: +{ + "paths": ["cmp22/2025/10/01/11185", "cmp22/2025/10/02/11185", "cmp22/2025/11", "cmp22/2026"] +} + +""" + +from __future__ import annotations + +from pathlib import Path +import json +import re +import sys + +import environs + + +DATE_RE = re.compile(r"^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$") + + +def _load_manifest(env: environs.Env) -> list[dict]: + manifest_inline = env.str("MANIFEST", None) + manifest_file_raw = env.str("MANIFEST_FILE", None) + + if manifest_inline and manifest_inline.strip(): + try: + manifest_data = json.loads(manifest_inline) + except json.JSONDecodeError as exc: + sys.exit(f"Invalid JSON in MANIFEST: {exc}") + else: + if not manifest_file_raw: + sys.exit("One of MANIFEST or MANIFEST_FILE environment variables is required.") + manifest_file = Path(manifest_file_raw) + if not manifest_file.exists(): + sys.exit(f"MANIFEST_FILE does not exist: {manifest_file}") + + with open(manifest_file, "r", encoding="utf-8") as manifest_handle: + try: + manifest_data = json.load(manifest_handle) + except json.JSONDecodeError as exc: + sys.exit(f"Invalid JSON in MANIFEST_FILE {manifest_file}: {exc}") + + if not isinstance(manifest_data, list): + sys.exit("Manifest must be a JSON array of objects.") + + records: list[dict] = [] + for idx, record in enumerate(manifest_data): + if not isinstance(record, dict): + sys.exit(f"Manifest record at index {idx} is not an object.") + if "source_type" not in record: + sys.exit(f"Manifest record at index {idx} is missing required key 'source_type'.") + if "data_date" not in record: + sys.exit(f"Manifest record at index {idx} is missing required key 'data_date'.") + records.append(record) + + return records + + +def _read_index_env(env: environs.Env) -> dict[str, int]: + raw_index_map = { + "source_type": env.int("OUT_PATH_SOURCE_TYPE_INDEX", None), + "year": env.int("OUT_PATH_YEAR_INDEX", None), + "month": env.int("OUT_PATH_MONTH_INDEX", None), + "day": env.int("OUT_PATH_DAY_INDEX", None), + "source_id": env.int("OUT_PATH_SOURCE_ID_INDEX", None), + } + + index_map = {key: value for key, value in raw_index_map.items() if value is not None} + if not index_map: + sys.exit( + "At least one OUT_PATH_*_INDEX environment variable is required to build output paths." + ) + + for key, value in index_map.items(): + if value < 0: + sys.exit(f"Index for {key} must be >= 0, got {value}.") + + if len(set(index_map.values())) != len(index_map): + sys.exit("All configured OUT_PATH_*_INDEX values must be unique.") + + return index_map + + +def _parse_data_date(data_date: object) -> tuple[str, str | None, str | None]: + if not isinstance(data_date, str): + sys.exit("Manifest key 'data_date' must be a string.") + + match = DATE_RE.fullmatch(data_date.strip()) + if not match: + sys.exit( + "Manifest key 'data_date' must match one of: YYYY-mm-dd, YYYY-mm, or YYYY." + ) + + year, month, day = match.group(1), match.group(2), match.group(3) + + if month is not None and not ("01" <= month <= "12"): + sys.exit(f"Invalid month in data_date '{data_date}'.") + if day is not None and not ("01" <= day <= "31"): + sys.exit(f"Invalid day in data_date '{data_date}'.") + + return year, month, day + + +def _build_path(record: dict, index_map: dict[str, int]) -> str: + source_type = record.get("source_type") + if not isinstance(source_type, str) or not source_type.strip(): + sys.exit("Manifest key 'source_type' must be a non-empty string.") + + year, month, day = _parse_data_date(record["data_date"]) + + values: dict[str, str] = { + "source_type": source_type.strip(), + "year": year, + } + + if month is not None: + values["month"] = month + if day is not None: + values["day"] = day + + # source_id is only used for full dates (YYYY-mm-dd). + if day is not None: + source_id = record.get("source_id") + if source_id is not None: + source_id_str = str(source_id).strip() + if source_id_str: + values["source_id"] = source_id_str + + # Determine the cutoff index using only configured date/source_id indices. + # If no relevant date/source_id indices are configured, no cutoff is applied. + cutoff: int | None = None + if "source_id" in values: + for key in ("source_id", "day", "month", "year"): + if key in index_map: + cutoff = index_map[key] + break + elif day is not None: + for key in ("day", "month", "year"): + if key in index_map: + cutoff = index_map[key] + break + elif month is not None: + for key in ("month", "year"): + if key in index_map: + cutoff = index_map[key] + break + elif "year" in index_map: + cutoff = index_map["year"] + + indexed_values = [ + (index_map[key], value) + for key, value in values.items() + if key in index_map and (cutoff is None or index_map[key] <= cutoff) + ] + indexed_values.sort(key=lambda item: item[0]) + + return "/".join(value for _, value in indexed_values) + + +def manifest_paths_builder() -> None: + env = environs.Env() + index_map = _read_index_env(env) + manifest_records = _load_manifest(env) + + seen_paths: set[str] = set() + output_paths: list[str] = [] + + for record in manifest_records: + path = _build_path(record, index_map) + if path not in seen_paths: + seen_paths.add(path) + output_paths.append(path) + + json.dump({"paths": output_paths}, sys.stdout, indent=4) + sys.stdout.write("\n") + + +if __name__ == "__main__": + manifest_paths_builder() \ No newline at end of file diff --git a/modules/gcs_data/requirements.txt b/modules/gcs_data/requirements.txt new file mode 100644 index 0000000000..40b9a90eba --- /dev/null +++ b/modules/gcs_data/requirements.txt @@ -0,0 +1,3 @@ +google-cloud-storage==3.9.0 +environs==14.4.0 +structlog==21.5.0 diff --git a/modules/gcs_data/tests/__init__.py b/modules/gcs_data/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/gcs_data/tests/test_manifest_paths_builder.py b/modules/gcs_data/tests/test_manifest_paths_builder.py new file mode 100644 index 0000000000..8b30356ccd --- /dev/null +++ b/modules/gcs_data/tests/test_manifest_paths_builder.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path +from unittest import mock + +import pytest +from pyfakefs.fake_filesystem_unittest import TestCase + +from gcs_data.manifest_paths_builder import ( + _build_path, + _load_manifest, + _parse_data_date, + _read_index_env, + manifest_paths_builder, +) + + +class ManifestPathsBuilderTest(TestCase): + + def setUp(self): + self.setUpPyfakefs() + + def test_parse_data_date_full_date(self): + """Test parsing full date YYYY-mm-dd.""" + year, month, day = _parse_data_date("2025-10-01") + self.assertEqual(year, "2025") + self.assertEqual(month, "10") + self.assertEqual(day, "01") + + def test_parse_data_date_month(self): + """Test parsing month date YYYY-mm.""" + year, month, day = _parse_data_date("2025-11") + self.assertEqual(year, "2025") + self.assertEqual(month, "11") + self.assertIsNone(day) + + def test_parse_data_date_year_only(self): + """Test parsing year-only date YYYY.""" + year, month, day = _parse_data_date("2026") + self.assertEqual(year, "2026") + self.assertIsNone(month) + self.assertIsNone(day) + + def test_parse_data_date_with_whitespace(self): + """Test parsing date with leading/trailing whitespace.""" + year, month, day = _parse_data_date(" 2025-10-01 ") + self.assertEqual(year, "2025") + self.assertEqual(month, "10") + self.assertEqual(day, "01") + + def test_parse_data_date_invalid_format(self): + """Test parsing invalid date format.""" + with self.assertRaises(SystemExit) as cm: + _parse_data_date("25-10-01") + self.assertIn("YYYY-mm-dd", str(cm.exception)) + + def test_parse_data_date_invalid_month(self): + """Test parsing with invalid month.""" + with self.assertRaises(SystemExit) as cm: + _parse_data_date("2025-13-01") + self.assertIn("Invalid month", str(cm.exception)) + + def test_parse_data_date_invalid_day(self): + """Test parsing with invalid day.""" + with self.assertRaises(SystemExit) as cm: + _parse_data_date("2025-10-32") + self.assertIn("Invalid day", str(cm.exception)) + + def test_parse_data_date_not_string(self): + """Test parsing with non-string data_date.""" + with self.assertRaises(SystemExit) as cm: + _parse_data_date(20251001) + self.assertIn("must be a string", str(cm.exception)) + + def test_read_index_env_valid(self): + """Test reading valid index environment variables.""" + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + import environs + env = environs.Env() + index_map = _read_index_env(env) + + self.assertEqual(index_map["source_type"], 0) + self.assertEqual(index_map["year"], 1) + self.assertEqual(index_map["month"], 2) + self.assertEqual(index_map["day"], 3) + self.assertEqual(index_map["source_id"], 4) + + def test_read_index_env_negative_index(self): + """Test that negative indices cause an error.""" + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "-1" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _read_index_env(env) + self.assertIn("must be >= 0", str(cm.exception)) + + def test_read_index_env_duplicate_indices(self): + """Test that duplicate indices cause an error.""" + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "0" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _read_index_env(env) + self.assertIn("must be unique", str(cm.exception)) + + def test_load_manifest_from_inline(self): + """Test loading manifest from inline JSON in MANIFEST env var.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10-02"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + + import environs + env = environs.Env() + records = _load_manifest(env) + + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["source_type"], "cmp22") + self.assertEqual(records[0]["source_id"], "11185") + + def test_load_manifest_from_file(self): + """Test loading manifest from file.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01"}, + ] + manifest_file = Path("/manifest.json") + with open(manifest_file, "w") as f: + json.dump(manifest_data, f) + + os.environ.pop("MANIFEST", None) + os.environ["MANIFEST_FILE"] = str(manifest_file) + + import environs + env = environs.Env() + records = _load_manifest(env) + + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["source_type"], "cmp22") + + def test_load_manifest_missing_required_field(self): + """Test that missing required fields cause an error.""" + manifest_data = [ + {"source_type": "cmp22"}, # Missing data_date + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _load_manifest(env) + self.assertIn("data_date", str(cm.exception)) + + def test_load_manifest_not_array(self): + """Test that non-array manifest causes an error.""" + os.environ["MANIFEST"] = json.dumps({"source_type": "cmp22", "data_date": "2025-10-01"}) + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _load_manifest(env) + self.assertIn("must be a JSON array", str(cm.exception)) + + def test_load_manifest_invalid_json(self): + """Test that invalid JSON causes an error.""" + os.environ["MANIFEST"] = "{ invalid json }" + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _load_manifest(env) + self.assertIn("Invalid JSON", str(cm.exception)) + + def test_load_manifest_file_not_found(self): + """Test that missing file causes an error.""" + os.environ.pop("MANIFEST", None) + os.environ["MANIFEST_FILE"] = "/nonexistent.json" + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _load_manifest(env) + self.assertIn("does not exist", str(cm.exception)) + + def test_load_manifest_priority_inline_over_file(self): + """Test that MANIFEST is used when both MANIFEST and MANIFEST_FILE are set.""" + inline_data = [{"source_type": "inline", "data_date": "2025-01-01"}] + file_data = [{"source_type": "file", "data_date": "2025-02-01"}] + + os.environ["MANIFEST"] = json.dumps(inline_data) + manifest_file = Path("/manifest.json") + with open(manifest_file, "w") as f: + json.dump(file_data, f) + os.environ["MANIFEST_FILE"] = str(manifest_file) + + import environs + env = environs.Env() + records = _load_manifest(env) + + self.assertEqual(records[0]["source_type"], "inline") + + def test_build_path_full_date_with_source_id(self): + """Test building path with full date and source_id.""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": "11185", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2025/10/01/11185") + + def test_build_path_full_date_without_source_id(self): + """Test building path with full date but no source_id.""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2025/10/01") + + def test_build_path_month_only_date(self): + """Test building path with month-only date (source_id ignored).""" + record = { + "source_type": "cmp22", + "data_date": "2025-11", + "source_id": "11185", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + path = _build_path(record, index_map) + # source_id should not be included when day is missing + self.assertEqual(path, "cmp22/2025/11") + + def test_build_path_year_only_date(self): + """Test building path with year-only date.""" + record = { + "source_type": "cmp22", + "data_date": "2026", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2026") + + def test_build_path_custom_index_order(self): + """Test building path with custom index ordering (source_id at end).""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": "11185", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 5, # source_id at higher index + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2025/10/01/11185") + + def test_build_path_empty_source_type(self): + """Test that empty source_type causes an error.""" + record = { + "source_type": " ", + "data_date": "2025-10-01", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + with self.assertRaises(SystemExit) as cm: + _build_path(record, index_map) + self.assertIn("non-empty string", str(cm.exception)) + + def test_build_path_empty_source_id_ignored(self): + """Test that empty source_id is ignored.""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": " ", + } + index_map = { + "source_type": 0, + "year": 1, + "month": 2, + "day": 3, + "source_id": 4, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2025/10/01") + + def test_manifest_paths_builder_basic(self): + """Test end-to-end manifest_paths_builder with basic example.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10-02", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-11"}, + {"source_type": "cmp22", "data_date": "2026"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + # Capture stdout + from io import StringIO + captured_output = StringIO() + sys.stdout = captured_output + + try: + manifest_paths_builder() + output = captured_output.getvalue() + result = json.loads(output) + + self.assertIn("paths", result) + paths = result["paths"] + self.assertEqual(len(paths), 4) + self.assertIn("cmp22/2025/10/01/11185", paths) + self.assertIn("cmp22/2025/10/02/11185", paths) + self.assertIn("cmp22/2025/11", paths) + self.assertIn("cmp22/2026", paths) + finally: + sys.stdout = sys.__stdout__ + + def test_manifest_paths_builder_deduplication(self): + """Test that duplicate paths are deduplicated.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + from io import StringIO + captured_output = StringIO() + sys.stdout = captured_output + + try: + manifest_paths_builder() + output = captured_output.getvalue() + result = json.loads(output) + + paths = result["paths"] + # Should have only one path, not two + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], "cmp22/2025/10/01/11185") + finally: + sys.stdout = sys.__stdout__ + + def test_manifest_paths_builder_multiple_source_types(self): + """Test with multiple source types.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "co2", "data_date": "2025-10-01", "source_id": "12345"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ["OUT_PATH_MONTH_INDEX"] = "2" + os.environ["OUT_PATH_DAY_INDEX"] = "3" + os.environ["OUT_PATH_SOURCE_ID_INDEX"] = "4" + + from io import StringIO + captured_output = StringIO() + sys.stdout = captured_output + + try: + manifest_paths_builder() + output = captured_output.getvalue() + result = json.loads(output) + + paths = result["paths"] + self.assertEqual(len(paths), 2) + self.assertIn("cmp22/2025/10/01/11185", paths) + self.assertIn("co2/2025/10/01/12345", paths) + finally: + sys.stdout = sys.__stdout__ + + # --- Subset OUT_PATH_*_INDEX scenarios --- + + def test_read_index_env_subset_only_source_type_and_year(self): + """Test _read_index_env with only source_type and year indices set.""" + os.environ.pop("OUT_PATH_MONTH_INDEX", None) + os.environ.pop("OUT_PATH_DAY_INDEX", None) + os.environ.pop("OUT_PATH_SOURCE_ID_INDEX", None) + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + + import environs + env = environs.Env() + index_map = _read_index_env(env) + + self.assertEqual(index_map["source_type"], 0) + self.assertEqual(index_map["year"], 1) + self.assertNotIn("month", index_map) + self.assertNotIn("day", index_map) + self.assertNotIn("source_id", index_map) + + def test_read_index_env_subset_no_vars_raises(self): + """Test _read_index_env with no indices set raises an error.""" + for key in ( + "OUT_PATH_SOURCE_TYPE_INDEX", + "OUT_PATH_YEAR_INDEX", + "OUT_PATH_MONTH_INDEX", + "OUT_PATH_DAY_INDEX", + "OUT_PATH_SOURCE_ID_INDEX", + ): + os.environ.pop(key, None) + + import environs + env = environs.Env() + with self.assertRaises(SystemExit) as cm: + _read_index_env(env) + self.assertIn("required", str(cm.exception)) + + def test_build_path_subset_source_type_and_year_only(self): + """Test _build_path when only source_type and year are indexed.""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": "11185", + } + index_map = { + "source_type": 0, + "year": 1, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/2025") + + def test_build_path_subset_year_and_month_only(self): + """Test _build_path when only year and month are indexed (no source_type).""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": "11185", + } + index_map = { + "year": 0, + "month": 1, + } + path = _build_path(record, index_map) + self.assertEqual(path, "2025/10") + + def test_build_path_subset_source_type_and_source_id_only(self): + """Test _build_path when only source_type and source_id are indexed.""" + record = { + "source_type": "cmp22", + "data_date": "2025-10-01", + "source_id": "11185", + } + index_map = { + "source_type": 0, + "source_id": 1, + } + path = _build_path(record, index_map) + self.assertEqual(path, "cmp22/11185") + + def test_manifest_paths_builder_subset_indices(self): + """Test end-to-end manifest_paths_builder with only source_type and year indices.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "cmp22", "data_date": "2025-11", "source_id": "11185"}, + {"source_type": "co2", "data_date": "2025-10-01", "source_id": "12345"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + os.environ["OUT_PATH_SOURCE_TYPE_INDEX"] = "0" + os.environ["OUT_PATH_YEAR_INDEX"] = "1" + os.environ.pop("OUT_PATH_MONTH_INDEX", None) + os.environ.pop("OUT_PATH_DAY_INDEX", None) + os.environ.pop("OUT_PATH_SOURCE_ID_INDEX", None) + + from io import StringIO + captured_output = StringIO() + sys.stdout = captured_output + + try: + manifest_paths_builder() + output = captured_output.getvalue() + result = json.loads(output) + + paths = result["paths"] + # All 2025 records collapse to source_type/year; co2 is distinct + self.assertIn("cmp22/2025", paths) + self.assertIn("co2/2025", paths) + # Duplicates deduplicated: cmp22/2025 appears once despite two records + self.assertEqual(paths.count("cmp22/2025"), 1) + finally: + sys.stdout = sys.__stdout__ + + def test_manifest_paths_builder_subset_year_month_only(self): + """Test end-to-end manifest_paths_builder with only year and month indices.""" + manifest_data = [ + {"source_type": "cmp22", "data_date": "2025-10-01", "source_id": "11185"}, + {"source_type": "co2", "data_date": "2025-10-15", "source_id": "12345"}, + ] + os.environ["MANIFEST"] = json.dumps(manifest_data) + os.environ.pop("OUT_PATH_SOURCE_TYPE_INDEX", None) + os.environ["OUT_PATH_YEAR_INDEX"] = "0" + os.environ["OUT_PATH_MONTH_INDEX"] = "1" + os.environ.pop("OUT_PATH_DAY_INDEX", None) + os.environ.pop("OUT_PATH_SOURCE_ID_INDEX", None) + + from io import StringIO + captured_output = StringIO() + sys.stdout = captured_output + + try: + manifest_paths_builder() + output = captured_output.getvalue() + result = json.loads(output) + + paths = result["paths"] + # Both records map to 2025/10 (different source_type/source_id ignored) + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], "2025/10") + finally: + sys.stdout = sys.__stdout__ diff --git a/modules_combined/calibration_group_and_convert/Dockerfile b/modules_combined/calibration_group_and_convert/Dockerfile index f278a66b4a..ea2b289fb1 100644 --- a/modules_combined/calibration_group_and_convert/Dockerfile +++ b/modules_combined/calibration_group_and_convert/Dockerfile @@ -1,4 +1,5 @@ -# Dockerfile for NEON IS Data Processing - combined filter-joiner, kafka combiner, array parser, and Calibration Conversion +# Dockerfile for NEON IS Data Processing - combined filter-joiner, kafka combiner, array parser, +# calibration assignment and calibration Conversion # Example command (must be run from project parent directory to include modules/ and flow/ paths in Docker context): # docker build -t neon-is-cal-grp-conv -f ./modules_combined/calibration_group_and_convert/Dockerfile . @@ -26,7 +27,8 @@ COPY ${MODULE_DIR}/${APP_DIR_2}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR_ RUN apt update && \ apt-get install -y --no-install-recommends \ software-properties-common \ - gnupg && \ + gnupg \ + jq && \ rm -rf /var/lib/apt/lists/* RUN add-apt-repository ppa:deadsnakes/ppa -y @@ -63,12 +65,16 @@ COPY ./flow/flow.kfka.comb/wrap.kfka.comb.R . # Load calibration conversion module # Copy the lockfile and restore known working versions of R dependency packages # ENSURE that the renv.lock file is up-to-date and thus has all listed dependencies prior to creating this docker image +COPY ./flow/flow.cal.asgn/renv.lock . +RUN R -e 'renv::restore(lockfile="./renv.lock")' COPY ./flow/flow.cal.conv/renv.lock . RUN R -e 'renv::restore(lockfile="./renv.lock")' -# Copy in R code +# Copy in calibration R code COPY ./flow/flow.cal.conv/flow.cal.conv.R . COPY ./flow/flow.cal.conv/wrap.cal.conv.R . +COPY ./flow/flow.cal.asgn/flow.cal.asgn.R . +COPY ./flow/flow.cal.asgn/wrap.cal.asgn.R . # Run as app user USER appuser diff --git a/modules_combined/fill_date_gaps_and_regularize/Dockerfile b/modules_combined/fill_date_gaps_and_regularize/Dockerfile index c8e8f6070e..2a51570eb1 100644 --- a/modules_combined/fill_date_gaps_and_regularize/Dockerfile +++ b/modules_combined/fill_date_gaps_and_regularize/Dockerfile @@ -23,7 +23,8 @@ COPY ${MODULE_DIR}/${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/r RUN apt update && \ apt-get install -y --no-install-recommends \ - python3.8 && \ + python3.8 \ + jq && \ apt install -y python3-pip && \ python3 -mpip install --no-cache-dir --upgrade pip setuptools wheel && \ python3 -mpip install --no-cache-dir -r ${CONTAINER_APP_DIR}/${APP_DIR}/requirements.txt && \ @@ -41,6 +42,8 @@ ARG MODULE_DIR="flow" # Copy the lockfile and restore known working versions of R dependency packages # ENSURE that the renv.lock file is up-to-date and thus has all listed dependencies prior to creating this docker image +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/renv.lock ./renv.lock.loc.grp.asgn +RUN R -e 'renv::restore(lockfile="./renv.lock.loc.grp.asgn")' COPY ./${MODULE_DIR}/flow.rglr/renv.lock ./renv.lock RUN R -e 'renv::restore(lockfile="./renv.lock")' #RUN git clone -b deve https://github.com/NEONScience/eddy4R.git @@ -54,6 +57,8 @@ WORKDIR /usr/src/app # Copy in application code COPY ./${MODULE_DIR}/flow.rglr/wrap.rglr.R . COPY ./${MODULE_DIR}/flow.rglr/flow.rglr.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/flow.loc.grp.asgn.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/wrap.loc.grp.asgn.R . # Run as app user USER appuser diff --git a/modules_combined/fill_date_gaps_nonregularized/Dockerfile b/modules_combined/fill_date_gaps_nonregularized/Dockerfile index a8d4d50177..e27deb81cc 100644 --- a/modules_combined/fill_date_gaps_nonregularized/Dockerfile +++ b/modules_combined/fill_date_gaps_nonregularized/Dockerfile @@ -21,7 +21,8 @@ COPY ${MODULE_DIR}/${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/r RUN apt update && \ apt-get install -y --no-install-recommends \ - python3.8 && \ + python3.8 \ + jq && \ apt install -y python3-pip && \ python3 -mpip install --no-cache-dir --upgrade pip setuptools wheel && \ python3 -mpip install --no-cache-dir -r ${CONTAINER_APP_DIR}/${APP_DIR}/requirements.txt && \ @@ -39,6 +40,8 @@ ARG MODULE_DIR="flow" # Copy the lockfile and restore known working versions of R dependency packages # ENSURE that the renv.lock file is up-to-date and thus has all listed dependencies prior to creating this docker image +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/renv.lock ./renv.lock.loc.grp.asgn +RUN R -e 'renv::restore(lockfile="./renv.lock.loc.grp.asgn")' COPY ./${MODULE_DIR}/flow.gap.fill.nonrglr/renv.lock ./renv.lock RUN R -e 'renv::restore(lockfile="./renv.lock")' #RUN git clone -b deve https://github.com/NEONScience/eddy4R.git @@ -52,6 +55,8 @@ WORKDIR /usr/src/app # Copy in application code COPY ./${MODULE_DIR}/flow.gap.fill.nonrglr/wrap.gap.fill.nonrglr.R . COPY ./${MODULE_DIR}/flow.gap.fill.nonrglr/flow.gap.fill.nonrglr.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/flow.loc.grp.asgn.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/wrap.loc.grp.asgn.R . # Run as app user USER appuser diff --git a/modules_combined/level1_group_consolidate_srf/Dockerfile b/modules_combined/level1_group_consolidate_srf/Dockerfile index 663de6d733..7efeee3286 100644 --- a/modules_combined/level1_group_consolidate_srf/Dockerfile +++ b/modules_combined/level1_group_consolidate_srf/Dockerfile @@ -1,6 +1,6 @@ #### # -# This dockerfile will build the combined module for level1_consolidate, filter_joiner, and flow.pub.tabl. +# This dockerfile will build the combined module for level1_consolidate, filter_joiner, flow.pub.tabl, and flow.srf.asgn # It also includes rclone for exporting output to a bucket # Example command (must be run from project root directory to include modules and modules_combined path in Docker context): # docker build -t neon-is-levl1-grp-cons-srf -f ./modules_combined/level1_group_consolidate_srf/Dockerfile . @@ -34,7 +34,7 @@ COPY ${MODULE_DIR}/${COMMON_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${COMMON_ COPY ${MODULE_DIR}/${DATA_ACCESS_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${DATA_ACCESS_DIR}/requirements.txt RUN apt update && \ - apt-get install -y python3.9 python3.9-venv python3-pip && \ + apt-get install -y python3.9 python3.9-venv python3-pip jq && \ python3.9 -m venv /opt/venv ENV VIRTUAL_ENV=/opt/venv @@ -76,12 +76,16 @@ ARG MODULE_DIR="flow" # Copy the lockfile and restore known working versions of R dependency packages # ENSURE that the renv.lock file is up-to-date and thus has all listed dependencies prior to creating this docker image -COPY ./${MODULE_DIR}/flow.pub.tabl.srf/renv.lock ./renv.lock -RUN R -e 'renv::restore(lockfile="./renv.lock")' +COPY ./${MODULE_DIR}/flow.srf.asgn/renv.lock ./renv.lock.srf.asgn +RUN R -e 'renv::restore(lockfile="./renv.lock.srf.asgn")' +COPY ./${MODULE_DIR}/flow.pub.tabl.srf/renv.lock ./renv.lock.pub.tabl.srf +RUN R -e 'renv::restore(lockfile="./renv.lock.pub.tabl.srf")' # Copy in application code COPY ./${MODULE_DIR}/flow.pub.tabl.srf/flow.pub.tabl.srf.R . COPY ./${MODULE_DIR}/flow.pub.tabl.srf/wrap.pub.tabl.srf.R . +COPY ./${MODULE_DIR}/flow.srf.asgn/flow.srf.asgn.R . +COPY ./${MODULE_DIR}/flow.srf.asgn/wrap.srf.asgn.R . # Run as app user USER appuser diff --git a/modules_combined/location_group_and_restructure/Dockerfile b/modules_combined/location_group_and_restructure/Dockerfile index ea71b6cc3c..4ac13e047d 100644 --- a/modules_combined/location_group_and_restructure/Dockerfile +++ b/modules_combined/location_group_and_restructure/Dockerfile @@ -1,5 +1,6 @@ # Dockerfile for NEON IS Data Processing - Group in location metadata and restructure+merge data files by location -# This image combines four modules: filter-joiner, kafka combiner, flow.loc.repo.strc, and flow.loc.data.trnc.comb +# This image combines four modules: filter-joiner, kafka combiner, flow.loc.repo.strc, flow.loc.data.trnc.comb, +# and flow.loc.grp.asgn # Build with the following command # docker build --no-cache -t neon-is-loc-grp-strc-comb -f @@ -23,7 +24,8 @@ COPY ${MODULE_DIR}/${APP_DIR}/requirements.txt ${CONTAINER_APP_DIR}/${APP_DIR}/r RUN apt update && \ apt-get install -y --no-install-recommends \ - python3.8 && \ + python3.8 \ + jq && \ apt install -y python3-pip && \ python3 -mpip install --no-cache-dir --upgrade pip setuptools wheel && \ python3 -mpip install --no-cache-dir -r ${CONTAINER_APP_DIR}/${APP_DIR}/requirements.txt && \ @@ -53,16 +55,21 @@ COPY ./${MODULE_DIR}/flow.kfka.comb/wrap.kfka.comb.R . # Copy the lockfile and restore known working versions of R dependency packages # ENSURE that the renv.lock file is up-to-date and thus has all listed dependencies prior to creating this docker image +COPY ./flow/flow.loc.grp.asgn/renv.lock ./renv.lock.loc.grp.asgn +RUN R -e 'renv::restore(lockfile="./renv.lock.loc.grp.asgn")' COPY ./${MODULE_DIR}/flow.loc.repo.strc/renv.lock ./renv.lock.repo.strc RUN R -e 'renv::restore(lockfile="./renv.lock.repo.strc")' COPY ./${MODULE_DIR}/flow.loc.data.trnc.comb/renv.lock ./renv.lock.comb.trnc RUN R -e 'renv::restore(lockfile="./renv.lock.comb.trnc")' + # Copy in application code COPY ./${MODULE_DIR}/flow.loc.repo.strc/wrap.loc.repo.strc.R . COPY ./${MODULE_DIR}/flow.loc.repo.strc/flow.loc.repo.strc.R . COPY ./${MODULE_DIR}/flow.loc.data.trnc.comb/wrap.loc.data.trnc.comb.R . COPY ./${MODULE_DIR}/flow.loc.data.trnc.comb/flow.loc.data.trnc.comb.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/wrap.loc.grp.asgn.R . +COPY ./${MODULE_DIR}/flow.loc.grp.asgn/flow.loc.grp.asgn.R . # Run as app user USER appuser diff --git a/pipe/cmp22/cmp22_location_active_dates_assignment.yaml b/pipe/cmp22/cmp22_location_active_dates_assignment.yaml index ad56766598..e5ddc96b59 100644 --- a/pipe/cmp22/cmp22_location_active_dates_assignment.yaml +++ b/pipe/cmp22/cmp22_location_active_dates_assignment.yaml @@ -14,7 +14,7 @@ transform: FileYear=$FILE_YEAR TypeFile=namedLocation "Prop=HOR|VER|name|description|site|Data Rate|active_periods" - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:v1.3.4 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:sha-69bab6b # image_pull_secrets: # - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/cmp22/cmp22_location_asset_assignment.yaml b/pipe/cmp22/cmp22_location_asset_assignment.yaml index 7d5309f531..b7cb3d7c60 100644 --- a/pipe/cmp22/cmp22_location_asset_assignment.yaml +++ b/pipe/cmp22/cmp22_location_asset_assignment.yaml @@ -14,7 +14,7 @@ transform: FileYear=$FILE_YEAR TypeFile=asset "Prop=HOR|VER|install_date|remove_date|name|site|Data Rate|locations|location_properties:LatTow|location_properties:LonTow" - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:v1.3.4 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:sha-69bab6b # image_pull_secrets: # - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/radShortPrimary/radShortPrimary_group_assignment.yaml b/pipe/radShortPrimary/radShortPrimary_group_assignment.yaml index f5d88bb224..9de683408e 100644 --- a/pipe/radShortPrimary/radShortPrimary_group_assignment.yaml +++ b/pipe/radShortPrimary/radShortPrimary_group_assignment.yaml @@ -13,7 +13,7 @@ transform: DirErr=$ERR_PATH FileYear=$FILE_YEAR TypeFile=group - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:v1.3.4 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-loc-grp-asgn:sha-69bab6b # image_pull_secrets: # - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/radShortPrimary/radShortPrimary_srf_assignment.yaml b/pipe/radShortPrimary/radShortPrimary_srf_assignment.yaml index 5b7a885a22..e3495e0e05 100644 --- a/pipe/radShortPrimary/radShortPrimary_srf_assignment.yaml +++ b/pipe/radShortPrimary/radShortPrimary_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-srf-asgn:v1.1.6 + image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-srf-asgn:sha-69bab6b # image_pull_secrets: # - battelleecology-quay-read-all-pull-secret env: