diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml index 5baf27195..3c2b8eec7 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_analyze_pad_qaqc_join_custom.yaml @@ -110,7 +110,7 @@ transform: ERR_PATH: /pfs/out/errored_datums RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for filter-joiner CONFIG: | --- @@ -148,14 +148,15 @@ input: joinOn: $1/$2/$3 empty_files: false parallelism_spec: - constant: 8 + constant: 20 autoscaling: true resource_requests: - memory: 8G - cpu: 8.5 + memory: 5G + cpu: 2.5 + disk: 4G resource_limits: - memory: 10G - cpu: 10 + memory: 6G + cpu: 4 sidecar_resource_requests: memory: 3G cpu: 0.3 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml index f873b4837..028450f39 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_path.yaml @@ -52,11 +52,12 @@ input: glob: /(*/*/*) joinOn: $1 parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G cpu: 0.6 + disk: 2G resource_limits: memory: 2.5G cpu: 1.2 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml index dec818f9f..8b09ebb15 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_group_split.yaml @@ -66,7 +66,7 @@ transform: GROUP_FOCUS_MONTH_INDEX: '4' GROUP_FOCUS_DAY_INDEX: '5' GROUP_FOCUS_GROUP_INDEX: '8' - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -91,14 +91,14 @@ input: glob: /(*/*/*) joinOn: $1 parallelism_spec: - constant: 8 + constant: 16 autoscaling: true resource_requests: memory: 4G - cpu: 7 + cpu: 2 resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml index f2ecf5491..e4d3fc107 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_level1_group_consolidate_srf.yaml @@ -134,7 +134,7 @@ transform: # Environment variables for pub table and srf module PUB_WORKBOOKS: /tmp/pub_workbooks - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for the L1 archiver GROUP_PREFIX: conc-h2o-soil-salinity-split # no ending "_" @@ -162,14 +162,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 8 + constant: 16 autoscaling: true resource_requests: memory: 4G - cpu: 7 + cpu: 2 + disk: 4G resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml index abdc88b5f..b01814f4f 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_qm_join_stats.yaml @@ -163,7 +163,7 @@ transform: LOG_LEVEL: INFO RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY - PARALLELIZATION_INTERNAL: '3' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -190,14 +190,15 @@ input: joinOn: $1 empty_files: false parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G - cpu: 4 + cpu: 2 + disk: 3G resource_limits: memory: 3G - cpu: 5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml index f6aac050e..b4fbf0f99 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_stats_group_and_compute.yaml @@ -81,7 +81,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '5' # Option for stats module + PARALLELIZATION_INTERNAL: '2' # Option for stats module input: cross: - pfs: @@ -102,13 +102,14 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 5 -resource_requests: + constant: 14 +resource_requests: memory: 1.8G - cpu: 6 + cpu: 2 + disk: 3G resource_limits: memory: 3G - cpu: 7 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml index da4be0d0d..d62bab6fb 100644 --- a/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml +++ b/pipe/concH2oSoilSalinity/concH2oSoilSalinity_thresh_select_ts_pad.yaml @@ -87,7 +87,7 @@ transform: PAD_DIR: data COPY_DIR: location,uncertainty_coef,uncertainty_data # Can be multiple, separated by commas without spaces. Directories other than the pad directory to copy to the output RELATIVE_PATH_INDEX: '3' - PARALLELIZATION_INTERNAL: '5' # For threshold select module + PARALLELIZATION_INTERNAL: '2' # For threshold select module output_branch: master input: cross: @@ -100,14 +100,14 @@ input: repo: concH2oSoilSalinity_threshold glob: /thresholds.json parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: memory: 2.5G - cpu: 5.5 + cpu: 2 resource_limits: memory: 4G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1 diff --git a/pipe/enviroscan/enviroscan_array_parser.yaml b/pipe/enviroscan/enviroscan_array_parser.yaml index 62204962b..8ce636826 100644 --- a/pipe/enviroscan/enviroscan_array_parser.yaml +++ b/pipe/enviroscan/enviroscan_array_parser.yaml @@ -128,17 +128,17 @@ input: repo: enviroscan_data_source_gcs glob: "/enviroscan/*/*/*" parallelism_spec: - constant: 3 + constant: 16 autoscaling: true resource_requests: - memory: 300M - cpu: 1.6 + memory: 600M + cpu: 1.2 resource_limits: - memory: 1.5G + memory: 2G cpu: 2 sidecar_resource_requests: - memory: 2G - cpu: 0.5 + memory: 2.5G + cpu: 0.1 datum_set_spec: number: 1 scheduling_spec: diff --git a/pipe/enviroscan/enviroscan_calibration_assignment.yaml b/pipe/enviroscan/enviroscan_calibration_assignment.yaml index 9b844e47a..6acf29316 100644 --- a/pipe/enviroscan/enviroscan_calibration_assignment.yaml +++ b/pipe/enviroscan/enviroscan_calibration_assignment.yaml @@ -27,7 +27,7 @@ input: repo: enviroscan_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 5 + constant: 10 autoscaling: true resource_requests: memory: 500M diff --git a/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml b/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml index 46a9a49cc..fe154283a 100644 --- a/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml +++ b/pipe/enviroscan/enviroscan_calibration_group_and_convert.yaml @@ -114,7 +114,7 @@ transform: OUT_PATH_ARRAY_PARSER: /tmp/pfs/array_parsed # R modules must have pfs in the repo structure # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '3' # Option for calibration conversion module + PARALLELIZATION_INTERNAL: '2' # Option for calibration conversion module input: cross: # - pfs: @@ -164,14 +164,14 @@ input: outer_join: true empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 4 + constant: 12 autoscaling: true resource_requests: memory: 2.5G - cpu: 5 + cpu: 2 resource_limits: memory: 4G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1 diff --git a/pipe/enviroscan/enviroscan_data_source_gcs.yaml b/pipe/enviroscan/enviroscan_data_source_gcs.yaml index c074cb4a8..63b46dc31 100644 --- a/pipe/enviroscan/enviroscan_data_source_gcs.yaml +++ b/pipe/enviroscan/enviroscan_data_source_gcs.yaml @@ -62,7 +62,7 @@ input: branch: master output_branch: master parallelism_spec: - constant: 5 + constant: 15 autoscaling: true resource_requests: memory: 400M diff --git a/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml b/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml index e051a4c5f..bfa14e275 100644 --- a/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml +++ b/pipe/enviroscan/enviroscan_fill_date_gaps_and_regularize.yaml @@ -52,7 +52,7 @@ transform: EMPTY_FILE_TYPE_INDEX: '4' LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. # Environment variables for regularizer - PARALLELIZATION_INTERNAL: '3' # Parallelization within R. If increased, adjust resource requests appropriately. + PARALLELIZATION_INTERNAL: '2' # Parallelization within R. If increased, adjust resource requests appropriately. input: cross: - pfs: @@ -89,14 +89,14 @@ input: group_by: $1 empty_files: true # This can remain true even if LINK_TYPE=COPY parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G - cpu: 3.3 + cpu: 2 resource_limits: memory: 3G - cpu: 4.5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml b/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml index 40ed57933..e6b165548 100644 --- a/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml +++ b/pipe/enviroscan/enviroscan_location_active_dates_assignment.yaml @@ -30,7 +30,7 @@ input: repo: enviroscan_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 2 + constant: 8 autoscaling: true resource_requests: memory: 210M diff --git a/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml b/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml index 7a3d48611..c18e3d342 100644 --- a/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml +++ b/pipe/enviroscan/enviroscan_location_group_and_restructure.yaml @@ -50,14 +50,14 @@ input: empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2.2G - cpu: 3.3 + cpu: 1.5 resource_limits: memory: 4G - cpu: 4.5 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.5 diff --git a/pipe/prt/prt_calibration_assignment.yaml b/pipe/prt/prt_calibration_assignment.yaml index eb5a56957..0314b5d82 100644 --- a/pipe/prt/prt_calibration_assignment.yaml +++ b/pipe/prt/prt_calibration_assignment.yaml @@ -28,11 +28,12 @@ input: repo: prt_cron_daily_and_date_control glob: /data_year*.txt parallelism_spec: - constant: 5 + constant: 10 autoscaling: true resource_requests: memory: 300M cpu: 1.5 + disk: 2G resource_limits: memory: 1G cpu: 2 diff --git a/pipe/prt/prt_calibration_group_and_convert.yaml b/pipe/prt/prt_calibration_group_and_convert.yaml index 4a82bbf53..8ad2ec341 100644 --- a/pipe/prt/prt_calibration_group_and_convert.yaml +++ b/pipe/prt/prt_calibration_group_and_convert.yaml @@ -77,7 +77,7 @@ transform: RELATIVE_PATH_INDEX: "3" # Must be consistent across inputs LINK_TYPE: COPY # options are COPY or SYMLINK. MUST BE SIMLINK IF USING COMBINED MODULE. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '5' # Option for calibration conversion module + PARALLELIZATION_INTERNAL: '2' # Option for calibration conversion module input: cross: - pfs: @@ -120,14 +120,15 @@ input: # outer_join: true # empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 10 + constant: 16 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 + disk: 2G resource_limits: memory: 5G - cpu: 7.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/prt/prt_calibration_list_files.yaml b/pipe/prt/prt_calibration_list_files.yaml index ef364e167..412a16af9 100644 --- a/pipe/prt/prt_calibration_list_files.yaml +++ b/pipe/prt/prt_calibration_list_files.yaml @@ -20,6 +20,7 @@ autoscaling: true resource_requests: memory: 800M cpu: 0.4 + disk: 2G resource_limits: memory: 1.2G cpu: 1.5 diff --git a/pipe/prt/prt_calibration_loader.yaml b/pipe/prt/prt_calibration_loader.yaml index ab97cdd97..a3e3c0250 100644 --- a/pipe/prt/prt_calibration_loader.yaml +++ b/pipe/prt/prt_calibration_loader.yaml @@ -31,6 +31,7 @@ autoscaling: true resource_requests: memory: 500M cpu: 0.5 + disk: 2G resource_limits: memory: 1G cpu: 1.5 diff --git a/pipe/prt/prt_data_source_gcs.yaml b/pipe/prt/prt_data_source_gcs.yaml index 33624a8b3..ac3d33c26 100644 --- a/pipe/prt/prt_data_source_gcs.yaml +++ b/pipe/prt/prt_data_source_gcs.yaml @@ -60,16 +60,17 @@ input: branch: master output_branch: master parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: - memory: 600M - cpu: 0.5 -resource_limits: memory: 1G + cpu: 0.35 + disk: 2G +resource_limits: + memory: 3G cpu: 1.5 sidecar_resource_requests: - memory: 2.8G + memory: 3.5G cpu: 0.5 datum_set_spec: number: 1 diff --git a/pipe/prt/prt_fill_date_gaps_and_regularize.yaml b/pipe/prt/prt_fill_date_gaps_and_regularize.yaml index 49ff60237..c743aff95 100644 --- a/pipe/prt/prt_fill_date_gaps_and_regularize.yaml +++ b/pipe/prt/prt_fill_date_gaps_and_regularize.yaml @@ -48,7 +48,7 @@ transform: EMPTY_FILE_TYPE_INDEX: '4' LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. # Environment variables for regularizer - PARALLELIZATION_INTERNAL: '3' # Parallelization within R. If increased, adjust resource requests appropriately. + PARALLELIZATION_INTERNAL: '2' # Parallelization within R. If increased, adjust resource requests appropriately. input: cross: - pfs: @@ -85,19 +85,21 @@ input: group_by: $1 empty_files: true # This can remain true even if LINK_TYPE=COPY parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: - memory: 5G - cpu: 4.3 + memory: 5.5G + cpu: 2.5 + disk: 2G resource_limits: - memory: 7G - cpu: 5 + memory: 8G + cpu: 3 sidecar_resource_requests: - memory: 4G + memory: 5.5G cpu: 1.2 datum_set_spec: number: 1 scheduling_spec: node_selector: cloud.google.com/compute-class: pach-pipeline-class-no-spot + diff --git a/pipe/prt/prt_location_group_and_restructure.yaml b/pipe/prt/prt_location_group_and_restructure.yaml index 37110e1c4..b35f3382e 100644 --- a/pipe/prt/prt_location_group_and_restructure.yaml +++ b/pipe/prt/prt_location_group_and_restructure.yaml @@ -62,7 +62,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for R modules - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: join: - pfs: @@ -79,16 +79,17 @@ input: joinOn: $1 empty_files: false # Make sure to use false if LINK_TYPE=COPY. Can also be set to false for LINK_TYPE=SYMLINK. parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: - memory: 5G - cpu: 6 + memory: 5.5G + cpu: 2.5 + disk: 2G resource_limits: - memory: 6.5G - cpu: 7 + memory: 8G + cpu: 3 sidecar_resource_requests: - memory: 5G + memory: 5.5G cpu: 1.2 datum_set_spec: number: 1 diff --git a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml index efd05c590..6f6a23967 100644 --- a/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml +++ b/pipe/tempSoil/tempSoil_analyze_pad_and_qaqc_plau.yaml @@ -37,7 +37,7 @@ transform: RELATIVE_PATH_INDEX: '3' ERR_PATH: /pfs/out/errored_datums # Environment variables for qaqc plausibility - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' input: cross: - pfs: @@ -49,14 +49,14 @@ input: repo: tempSoil_avro_schemas glob: /tempSoil/flags_plausibility_tempSoil.avsc parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 resource_limits: memory: 5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1.2 diff --git a/pipe/tempSoil/tempSoil_group_path.yaml b/pipe/tempSoil/tempSoil_group_path.yaml index 8a1e45cfa..73c82a3b0 100644 --- a/pipe/tempSoil/tempSoil_group_path.yaml +++ b/pipe/tempSoil/tempSoil_group_path.yaml @@ -55,11 +55,12 @@ input: # glob: /(*/*/*) # joinOn: $1 parallelism_spec: - constant: 5 + constant: 12 autoscaling: true resource_requests: memory: 2G cpu: 0.6 + disk: 2G resource_limits: memory: 3.5G cpu: 1.5 diff --git a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml index a61c635f7..7ac6149d1 100644 --- a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml +++ b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml @@ -174,7 +174,7 @@ transform: # Environment variables for pub table and srf module PUB_WORKBOOKS: /tmp/pub_workbooks - PARALLELIZATION_INTERNAL: '5' + PARALLELIZATION_INTERNAL: '2' # Environment variables for the L1 archiver GROUP_PREFIX: temp-soil # no ending "_" @@ -217,14 +217,15 @@ input: empty_files: false # Make sure this is false for LINK_TYPE=COPY output_branch: master parallelism_spec: - constant: 5 + constant: 20 autoscaling: true resource_requests: memory: 4G - cpu: 6.5 + cpu: 2 + disk: 4G resource_limits: memory: 5.5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1 diff --git a/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml b/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml index d38225bef..e69b5238b 100644 --- a/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml +++ b/pipe/tempSoil/tempSoil_qm_group_and_compute.yaml @@ -53,7 +53,7 @@ transform: LOG_LEVEL: INFO RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined modules. - PARALLELIZATION_INTERNAL: '6' # Option for quality metrics module + PARALLELIZATION_INTERNAL: '2' # Option for quality metrics module input: cross: - pfs: @@ -74,14 +74,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 10 + constant: 16 autoscaling: true -resource_requests: +resource_requests: memory: 3G - cpu: 6.5 + cpu: 2 + disk: 3G resource_limits: memory: 4.5G - cpu: 7 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 0.6 diff --git a/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml b/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml index 8d28a4443..03cabb453 100644 --- a/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml +++ b/pipe/tempSoil/tempSoil_stats_group_and_compute.yaml @@ -50,7 +50,7 @@ transform: RELATIVE_PATH_INDEX: "3" LINK_TYPE: COPY # options are COPY or SYMLINK. Use COPY for combined module. # Environment variables for calibration module - PARALLELIZATION_INTERNAL: '4' # Option for stats module + PARALLELIZATION_INTERNAL: '2' # Option for stats module input: cross: - pfs: @@ -71,14 +71,15 @@ input: joinOn: $1 empty_files: false # Make sure this is false for LINK_TYPE=COPY parallelism_spec: - constant: 8 + constant: 20 autoscaling: true -resource_requests: +resource_requests: memory: 3G - cpu: 5 + cpu: 2 + disk: 3G resource_limits: memory: 4.5G - cpu: 8 + cpu: 2.5 sidecar_resource_requests: memory: 3G cpu: 1.2 diff --git a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml index 258e7d2ef..55877fca5 100644 --- a/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml +++ b/pipe/tempSoil/tempSoil_thresh_select_ts_pad.yaml @@ -36,7 +36,7 @@ transform: PAD_DIR: data COPY_DIR: none # Can be multiple, separated by commas without spaces. Directories other than the pad directory to copy to the output RELATIVE_PATH_INDEX: '3' - PARALLELIZATION_INTERNAL: '5' # For threshold select module + PARALLELIZATION_INTERNAL: '2' # For threshold select module output_branch: master input: cross: @@ -49,14 +49,14 @@ input: repo: tempSoil_threshold glob: /thresholds.json parallelism_spec: - constant: 5 + constant: 14 autoscaling: true resource_requests: memory: 1.5G - cpu: 5.5 + cpu: 2 resource_limits: memory: 3G - cpu: 6.5 + cpu: 2.5 sidecar_resource_requests: memory: 4G cpu: 1