From 87c3d8c1c6a3a074eef56a0eec2bd532874da234 Mon Sep 17 00:00:00 2001 From: Teresa Burlingame Date: Thu, 30 Jul 2026 10:11:57 -0700 Subject: [PATCH 1/2] tune pipeline specs for tempSoil / concH2oSoilSalinity / prt / enviroscan Reduce parallelism_spec and adjust resource requests/limits and disk allocations to relieve cluster scheduling pressure. --- ...oilSalinity_analyze_pad_qaqc_join_custom.yaml | 13 +++++++------ .../concH2oSoilSalinity_group_path.yaml | 3 ++- .../concH2oSoilSalinity_group_split.yaml | 8 ++++---- ...oilSalinity_level1_group_consolidate_srf.yaml | 9 +++++---- .../concH2oSoilSalinity_qm_join_stats.yaml | 9 +++++---- ...cH2oSoilSalinity_stats_group_and_compute.yaml | 11 ++++++----- ...concH2oSoilSalinity_thresh_select_ts_pad.yaml | 8 ++++---- pipe/enviroscan/enviroscan_array_parser.yaml | 12 ++++++------ .../enviroscan_calibration_assignment.yaml | 2 +- ...enviroscan_calibration_group_and_convert.yaml | 8 ++++---- pipe/enviroscan/enviroscan_data_source_gcs.yaml | 2 +- ...enviroscan_fill_date_gaps_and_regularize.yaml | 8 ++++---- ...iroscan_location_active_dates_assignment.yaml | 2 +- ...nviroscan_location_group_and_restructure.yaml | 6 +++--- pipe/prt/prt_calibration_assignment.yaml | 3 ++- pipe/prt/prt_calibration_group_and_convert.yaml | 9 +++++---- pipe/prt/prt_calibration_list_files.yaml | 1 + pipe/prt/prt_calibration_loader.yaml | 1 + pipe/prt/prt_data_source_gcs.yaml | 11 ++++++----- pipe/prt/prt_fill_date_gaps_and_regularize.yaml | 16 +++++++++------- pipe/prt/prt_location_group_and_restructure.yaml | 15 ++++++++------- .../tempSoil_analyze_pad_and_qaqc_plau.yaml | 8 ++++---- pipe/tempSoil/tempSoil_group_path.yaml | 3 ++- .../tempSoil_level1_group_consolidate_srf.yaml | 9 +++++---- pipe/tempSoil/tempSoil_qm_group_and_compute.yaml | 11 ++++++----- .../tempSoil_stats_group_and_compute.yaml | 11 ++++++----- pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml | 8 ++++---- 27 files changed, 112 insertions(+), 95 deletions(-) diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml index 5baf27195..4f079c7b8 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml @@ -110,7 +110,7 @@ transform: ERR_PATH: /pfs/out/errored_datums RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for filter-joiner CONFIG: | --- @@ -148,14 +148,15 @@ input: joinOn: $1/$2/$3 empty_files: false parallelism_spec: - constant: 8 + constant: 30 autoscaling: true resource_requests: - memory: 8G - cpu: 8.5 + memory: 5G + cpu: 2.5 + disk: 4G resource_limits: - memory: 10G - cpu: 10 + memory: 6G + cpu: 4 sidecar_resource_requests: memory: 3G cpu: 0.3 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml index f873b4837..028450f39 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml @@ -52,11 +52,12 @@ input: glob: /(*/*/*) joinOn: $1 parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G cpu: 0.6 + disk: 2G resource_limits: memory: 2.5G cpu: 1.2 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml index dec818f9f..43149baa7 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml @@ -66,7 +66,7 @@ transform: GROUP_FOCUS_MONTH_INDEX: '4' GROUP_FOCUS_DAY_INDEX: '5' GROUP_FOCUS_GROUP_INDEX: '8' - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -91,14 +91,14 @@ input: glob: /(*/*/*) joinOn: $1 parallelism_spec: - constant: 8 + constant: 24 autoscaling: true resource_requests: memory: 4G - cpu: 7 + cpu: 2 resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml index ffa21cbb5..16d0a14ff 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml @@ -134,7 +134,7 @@ transform: # Environment variables for pub table and srf module PUB_WORKBOOKS: /tmp/pub_workbooks - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for the L1 archiver GROUP_PREFIX: conc-h2o-soil-salinity-split # no ending "_" @@ -162,14 +162,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 8 + constant: 16 autoscaling: true resource_requests: memory: 4G - cpu: 7 + cpu: 2 + disk: 4G resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml index abdc88b5f..b01814f4f 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml @@ -163,7 +163,7 @@ transform: LOG_LEVEL: INFO RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY - PARALLELIZATION_INTERNAL: '3' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -190,14 +190,15 @@ input: joinOn: $1 empty_files: false parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G - cpu: 4 + cpu: 2 + disk: 3G resource_limits: memory: 3G - cpu: 5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml index f6aac050e..b4fbf0f99 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml @@ -81,7 +81,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '5' # Option for stats module + PARALLELIZATION_INTERNAL: '2' # Option for stats module input: cross: - pfs: @@ -102,13 +102,14 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 5 -resource_requests: + constant: 14 +resource_requests: memory: 1.8G - cpu: 6 + cpu: 2 + disk: 3G resource_limits: memory: 3G - cpu: 7 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml index da4be0d0d..872026144 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml @@ -87,7 +87,7 @@ transform: PAD_DIR: data COPY_DIR: location,uncertainty_coef,uncertainty_data # Can be multiple, separated by commas without spaces. Directories other than the pad directory to copy to the output RELATIVE_PATH_INDEX: '3' - PARALLELIZATION_INTERNAL: '5' # For threshold select module + PARALLELIZATION_INTERNAL: '2' # For threshold select module output_branch: master input: cross: @@ -100,14 +100,14 @@ input: repo: concH2oSoilSalinity_threshold glob: /thresholds.json parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: memory: 2.5G - cpu: 5.5 + cpu: 2 resource_limits: memory: 4G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1 diff --git a/pipe/enviroscan/enviroscan_array_parser.yaml b/pipe/enviroscan/enviroscan_array_parser.yaml index 62204962b..8ce636826 100644 --- a/pipe/enviroscan/enviroscan_array_parser.yaml +++ b/pipe/enviroscan/enviroscan_array_parser.yaml @@ -128,17 +128,17 @@ input: repo: enviroscan_data_source_gcs glob: "/enviroscan/*/*/*" parallelism_spec: - constant: 3 + constant: 16 autoscaling: true resource_requests: - memory: 300M - cpu: 1.6 + memory: 600M + cpu: 1.2 resource_limits: - memory: 1.5G + memory: 2G cpu: 2 sidecar_resource_requests: - memory: 2G - cpu: 0.5 + memory: 2.5G + cpu: 0.1 datum_set_spec: number: 1 scheduling_spec: diff --git a/pipe/enviroscan/enviroscan_calibration_assignment.yaml b/pipe/enviroscan/enviroscan_calibration_assignment.yaml index f1feb70ed..ced2550fb 100644 --- a/pipe/enviroscan/enviroscan_calibration_assignment.yaml +++ b/pipe/enviroscan/enviroscan_calibration_assignment.yaml @@ -27,7 +27,7 @@ input: repo: enviroscan_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 5 + constant: 10 autoscaling: true resource_requests: memory: 500M diff --git a/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml b/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml index 9236b8057..b705bfda9 100644 --- a/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml +++ b/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml @@ -114,7 +114,7 @@ transform: OUT_PATH_ARRAY_PARSER: /tmp/pfs/array_parsed # R modules must have pfs in the repo structure # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '3' # Option for calibration conversion module + PARALLELIZATION_INTERNAL: '2' # Option for calibration conversion module input: cross: # - pfs: @@ -164,14 +164,14 @@ input: outer_join: true empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 4 + constant: 12 autoscaling: true resource_requests: memory: 2.5G - cpu: 5 + cpu: 2 resource_limits: memory: 4G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1 diff --git a/pipe/enviroscan/enviroscan_data_source_gcs.yaml b/pipe/enviroscan/enviroscan_data_source_gcs.yaml index c074cb4a8..63b46dc31 100644 --- a/pipe/enviroscan/enviroscan_data_source_gcs.yaml +++ b/pipe/enviroscan/enviroscan_data_source_gcs.yaml @@ -62,7 +62,7 @@ input: branch: master output_branch: master parallelism_spec: - constant: 5 + constant: 15 autoscaling: true resource_requests: memory: 400M diff --git a/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml b/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml index 3ab828c45..bcb059678 100644 --- a/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml +++ b/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml @@ -52,7 +52,7 @@ transform: EMPTY_FILE_TYPE_INDEX: '4' LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. # Environment variables for regularizer - PARALLELIZATION_INTERNAL: '3' # Parallelization within R. If increased, adjust resource requests appropriately. + PARALLELIZATION_INTERNAL: '2' # Parallelization within R. If increased, adjust resource requests appropriately. input: cross: - pfs: @@ -89,14 +89,14 @@ input: group_by: $1 empty_files: true # This can remain true even if LINK_TYPE=COPY parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G - cpu: 3.3 + cpu: 2 resource_limits: memory: 3G - cpu: 4.5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml b/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml index 814cf2624..fc0144221 100644 --- a/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml +++ b/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml @@ -30,7 +30,7 @@ input: repo: enviroscan_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 2 + constant: 8 autoscaling: true resource_requests: memory: 210M diff --git a/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml b/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml index eb3633b3d..fc6c8ea15 100644 --- a/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml +++ b/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml @@ -50,14 +50,14 @@ input: empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2.2G - cpu: 3.3 + cpu: 1.5 resource_limits: memory: 4G - cpu: 4.5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/prt/prt_calibration_assignment.yaml b/pipe/prt/prt_calibration_assignment.yaml index c2a35c68c..01b608288 100644 --- a/pipe/prt/prt_calibration_assignment.yaml +++ b/pipe/prt/prt_calibration_assignment.yaml @@ -28,11 +28,12 @@ input: repo: prt_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 5 + constant: 10 autoscaling: true resource_requests: memory: 300M cpu: 1.5 + disk: 2G resource_limits: memory: 1G cpu: 2 diff --git a/pipe/prt/prt_calibration_group_and_convert.yaml b/pipe/prt/prt_calibration_group_and_convert.yaml index 332a5e882..ec2741cf3 100644 --- a/pipe/prt/prt_calibration_group_and_convert.yaml +++ b/pipe/prt/prt_calibration_group_and_convert.yaml @@ -77,7 +77,7 @@ transform: RELATIVE_PATH_INDEX: "3" # Must be consistent across inputs LINK_TYPE: COPY # options are COPY or SYMLINK. MUST BE SIMLINK IF USING COMBINED MODULE. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '5' # Option for calibration conversion module + PARALLELIZATION_INTERNAL: '2' # Option for calibration conversion module input: cross: - pfs: @@ -120,14 +120,15 @@ input: # outer_join: true # empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 10 + constant: 16 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 + disk: 2G resource_limits: memory: 5G - cpu: 7.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/prt/prt_calibration_list_files.yaml b/pipe/prt/prt_calibration_list_files.yaml index ef364e167..412a16af9 100644 --- a/pipe/prt/prt_calibration_list_files.yaml +++ b/pipe/prt/prt_calibration_list_files.yaml @@ -20,6 +20,7 @@ autoscaling: true resource_requests: memory: 800M cpu: 0.4 + disk: 2G resource_limits: memory: 1.2G cpu: 1.5 diff --git a/pipe/prt/prt_calibration_loader.yaml b/pipe/prt/prt_calibration_loader.yaml index ab97cdd97..a3e3c0250 100644 --- a/pipe/prt/prt_calibration_loader.yaml +++ b/pipe/prt/prt_calibration_loader.yaml @@ -31,6 +31,7 @@ autoscaling: true resource_requests: memory: 500M cpu: 0.5 + disk: 2G resource_limits: memory: 1G cpu: 1.5 diff --git a/pipe/prt/prt_data_source_gcs.yaml b/pipe/prt/prt_data_source_gcs.yaml index 33624a8b3..ac3d33c26 100644 --- a/pipe/prt/prt_data_source_gcs.yaml +++ b/pipe/prt/prt_data_source_gcs.yaml @@ -60,16 +60,17 @@ input: branch: master output_branch: master parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: - memory: 600M - cpu: 0.5 -resource_limits: memory: 1G + cpu: 0.35 + disk: 2G +resource_limits: + memory: 3G cpu: 1.5 sidecar_resource_requests: - memory: 2.8G + memory: 3.5G cpu: 0.5 datum_set_spec: number: 1 diff --git a/pipe/prt/prt_fill_date_gaps_and_regularize.yaml b/pipe/prt/prt_fill_date_gaps_and_regularize.yaml index af96e5886..407f8d1f1 100644 --- a/pipe/prt/prt_fill_date_gaps_and_regularize.yaml +++ b/pipe/prt/prt_fill_date_gaps_and_regularize.yaml @@ -48,7 +48,7 @@ transform: EMPTY_FILE_TYPE_INDEX: '4' LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. # Environment variables for regularizer - PARALLELIZATION_INTERNAL: '3' # Parallelization within R. If increased, adjust resource requests appropriately. + PARALLELIZATION_INTERNAL: '2' # Parallelization within R. If increased, adjust resource requests appropriately. input: cross: - pfs: @@ -85,19 +85,21 @@ input: group_by: $1 empty_files: true # This can remain true even if LINK_TYPE=COPY parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: - memory: 5G - cpu: 4.3 + memory: 5.5G + cpu: 2.5 + disk: 2G resource_limits: - memory: 7G - cpu: 5 + memory: 8G + cpu: 3 sidecar_resource_requests: - memory: 4G + memory: 5.5G cpu: 1.2 datum_set_spec: number: 1 scheduling_spec: node_selector: cloud.google.com/compute-class: pach-pipeline-class-no-spot + diff --git a/pipe/prt/prt_location_group_and_restructure.yaml b/pipe/prt/prt_location_group_and_restructure.yaml index 4643f7541..2ba83dd8c 100644 --- a/pipe/prt/prt_location_group_and_restructure.yaml +++ b/pipe/prt/prt_location_group_and_restructure.yaml @@ -62,7 +62,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for R modules - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: join: - pfs: @@ -79,16 +79,17 @@ input: joinOn: $1 empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: - memory: 5G - cpu: 6 + memory: 5.5G + cpu: 2.5 + disk: 2G resource_limits: - memory: 6.5G - cpu: 7 + memory: 8G + cpu: 3 sidecar_resource_requests: - memory: 5G + memory: 5.5G cpu: 1.2 datum_set_spec: number: 1 diff --git a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml index efd05c590..e38e422d7 100644 --- a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml +++ b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml @@ -37,7 +37,7 @@ transform: RELATIVE_PATH_INDEX: '3' ERR_PATH: /pfs/out/errored_datums # Environment variables for qaqc plausibility - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -49,14 +49,14 @@ input: repo: tempSoil_avro_schemas glob: /tempSoil/flags_plausibility_tempSoil.avsc parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/tempSoil/tempSoil_group_path.yaml b/pipe/tempSoil/tempSoil_group_path.yaml index 8a1e45cfa..73c82a3b0 100644 --- a/pipe/tempSoil/tempSoil_group_path.yaml +++ b/pipe/tempSoil/tempSoil_group_path.yaml @@ -55,11 +55,12 @@ input: # glob: /(*/*/*) # joinOn: $1 parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G cpu: 0.6 + disk: 2G resource_limits: memory: 3.5G cpu: 1.5 diff --git a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml index 86d81f306..e3d5ec124 100644 --- a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml +++ b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml @@ -174,7 +174,7 @@ transform: # Environment variables for pub table and srf module PUB_WORKBOOKS: /tmp/pub_workbooks - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for the L1 archiver GROUP_PREFIX: temp-soil # no ending "_" @@ -217,14 +217,15 @@ input: empty_files: false # Make sure this is false for LINK_TYPE=COPY output_branch: master parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 + disk: 4G resource_limits: memory: 5.5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml b/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml index d38225bef..e69b5238b 100644 --- a/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml +++ b/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml @@ -53,7 +53,7 @@ transform: LOG_LEVEL: INFO RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. - PARALLELIZATION_INTERNAL: '6' # Option for quality metrics module + PARALLELIZATION_INTERNAL: '2' # Option for quality metrics module input: cross: - pfs: @@ -74,14 +74,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 10 + constant: 16 autoscaling: true -resource_requests: +resource_requests: memory: 3G - cpu: 6.5 + cpu: 2 + disk: 3G resource_limits: memory: 4.5G - cpu: 7 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.6 diff --git a/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml b/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml index 8d28a4443..03cabb453 100644 --- a/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml +++ b/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml @@ -50,7 +50,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '4' # Option for stats module + PARALLELIZATION_INTERNAL: '2' # Option for stats module input: cross: - pfs: @@ -71,14 +71,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 8 + constant: 20 autoscaling: true -resource_requests: +resource_requests: memory: 3G - cpu: 5 + cpu: 2 + disk: 3G resource_limits: memory: 4.5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1.2 diff --git a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml index 258e7d2ef..1920a5901 100644 --- a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml +++ b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml @@ -36,7 +36,7 @@ transform: PAD_DIR: data COPY_DIR: none # Can be multiple, separated by commas without spaces. Directories other than the pad directory to copy to the output RELATIVE_PATH_INDEX: '3' - PARALLELIZATION_INTERNAL: '5' # For threshold select module + PARALLELIZATION_INTERNAL: '2' # For threshold select module output_branch: master input: cross: @@ -49,14 +49,14 @@ input: repo: tempSoil_threshold glob: /thresholds.json parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: memory: 1.5G - cpu: 5.5 + cpu: 2 resource_limits: memory: 3G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1 From 4194bbd257829d40edea19b8c746d4282bfb7beb Mon Sep 17 00:00:00 2001 From: Teresa Burlingame Date: Thu, 30 Jul 2026 12:38:06 -0700 Subject: [PATCH 2/2] reduce paralellism on some very high ones. Claude likely overshooting spec availability. --- .../concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml | 2 +- pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml | 2 +- .../concH2oSoilSalinity_thresh_select_ts_pad.yaml | 2 +- pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml | 2 +- pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml index 4f079c7b8..3c2b8eec7 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml @@ -148,7 +148,7 @@ input: joinOn: $1/$2/$3 empty_files: false parallelism_spec: - constant: 30 + constant: 20 autoscaling: true resource_requests: memory: 5G diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml index 43149baa7..8b09ebb15 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml @@ -91,7 +91,7 @@ input: glob: /(*/*/*) joinOn: $1 parallelism_spec: - constant: 24 + constant: 16 autoscaling: true resource_requests: memory: 4G diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml index 872026144..d62bab6fb 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml @@ -100,7 +100,7 @@ input: repo: concH2oSoilSalinity_threshold glob: /thresholds.json parallelism_spec: - constant: 20 + constant: 14 autoscaling: true resource_requests: memory: 2.5G diff --git a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml index e38e422d7..6f6a23967 100644 --- a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml +++ b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml @@ -49,7 +49,7 @@ input: repo: tempSoil_avro_schemas glob: /tempSoil/flags_plausibility_tempSoil.avsc parallelism_spec: - constant: 20 + constant: 14 autoscaling: true resource_requests: memory: 4G diff --git a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml index 1920a5901..55877fca5 100644 --- a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml +++ b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml @@ -49,7 +49,7 @@ input: repo: tempSoil_threshold glob: /thresholds.json parallelism_spec: - constant: 20 + constant: 14 autoscaling: true resource_requests: memory: 1.5G