diff --git a/applications/lfric2lfric/source/initialisation/lfric2lfric_init_mesh.f90 b/applications/lfric2lfric/source/initialisation/lfric2lfric_init_mesh.f90 index 734ca96d17..2d417fa803 100644 --- a/applications/lfric2lfric/source/initialisation/lfric2lfric_init_mesh.f90 +++ b/applications/lfric2lfric/source/initialisation/lfric2lfric_init_mesh.f90 @@ -335,34 +335,67 @@ subroutine init_mesh( config, & !=========================================================== if (mesh_file(dst) == mesh_file(src)) then call load_global_mesh( mesh_file(dst), mesh_names ) + + ! Partition the global meshes + !=========================================================== + call create_local_mesh( mesh_names(dst:dst), & + local_rank, total_ranks, & + decomposition_dst, & + stencil_depths, & + generate_inner_halos(dst), & + partitioner_dst, & + enforce_constraints = .false. ) + call create_local_mesh( mesh_names(src:src), & + local_rank, total_ranks, & + decomposition_src, & + stencil_depths, & + generate_inner_halos(src), & + partitioner_src, & + enforce_constraints = .false. ) + + ! Create the associated local mesh maps + !=========================================================== + if (regrid_method == regrid_method_map) then + call create_local_mesh_maps( mesh_file(dst) ) + end if + + ! Clear the + call global_mesh_collection%clear() + else - call load_global_mesh( mesh_file(dst), mesh_names(dst) ) + + ! SOURCE call load_global_mesh( mesh_file(src), mesh_names(src) ) - endif + write( log_scratch_space, '(A)' ) & + 'Loaded source mesh' + call log_event(log_scratch_space, log_level_debug) + + call create_local_mesh( mesh_names(src:src), & + local_rank, total_ranks, & + decomposition_src, & + stencil_depths, & + generate_inner_halos(src), & + partitioner_src, & + enforce_constraints = .false. ) + + call global_mesh_collection%clear() + + ! DESTINATION + call load_global_mesh( mesh_file(dst), mesh_names(dst) ) + write( log_scratch_space, '(A)' ) & + 'Loaded destination mesh' + call log_event(log_scratch_space, log_level_debug) + + call create_local_mesh( mesh_names(dst:dst), & + local_rank, total_ranks, & + decomposition_dst, & + stencil_depths, & + generate_inner_halos(dst), & + partitioner_dst, & + enforce_constraints = .false. ) + + call global_mesh_collection%clear() - ! Partition the global meshes - !=========================================================== - call create_local_mesh( mesh_names(dst:dst), & - local_rank, total_ranks, & - decomposition_dst, & - stencil_depths, & - generate_inner_halos(dst), & - partitioner_dst, & - enforce_constraints = .false. ) - - call create_local_mesh( mesh_names(src:src), & - local_rank, total_ranks, & - decomposition_src, & - stencil_depths, & - generate_inner_halos(src), & - partitioner_src, & - enforce_constraints = .false. ) - - ! Read in the global intergrid mesh mappings, - ! then create the associated local mesh maps - !=========================================================== - if (regrid_method == regrid_method_map) then - call create_local_mesh_maps( mesh_file(dst) ) end if end if ! prepartitioned diff --git a/rose-stem/app/lfric2lfric/file/iodef_diag_file.xml b/rose-stem/app/lfric2lfric/file/iodef_diag_file.xml new file mode 100644 index 0000000000..e292200d47 --- /dev/null +++ b/rose-stem/app/lfric2lfric/file/iodef_diag_file.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + performance + 1.0 + + + + false + false + 50 + true + lfric2lfric lfric2lfric + + + + + diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-clim_gal9.conf b/rose-stem/app/lfric2lfric/opt/rose-app-clim_gal9.conf index 723f79bc39..9aa1e4d4e6 100644 --- a/rose-stem/app/lfric2lfric/opt/rose-app-clim_gal9.conf +++ b/rose-stem/app/lfric2lfric/opt/rose-app-clim_gal9.conf @@ -9,5 +9,4 @@ stretching_height=17507.0 stretching_method='smooth' [namelist:files] -checkpoint_stem_name='restart_lfric2lfric_clim_gal9_C12_MG' start_dump_filename='lfric2lfric_clim_gal9_C12_MG' diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-dst_melbourne.conf b/rose-stem/app/lfric2lfric/opt/rose-app-dst_melbourne.conf new file mode 100644 index 0000000000..9e058ec329 --- /dev/null +++ b/rose-stem/app/lfric2lfric/opt/rose-app-dst_melbourne.conf @@ -0,0 +1,10 @@ +[namelist:lfric2lfric] +destination_mesh_name='melbourne' +destination_meshfile_prefix='${MESH_DIR}/melbourne/mesh_melbourne' +destination_topology='non_periodic' +dst_ancil_directory='$BIG_DATA_DIR/ancils/proto-ral/Ticket255/Melbourne' +dst_orography_mean_ancil_path='' +target_domain='LAM' + +[namelist:partitioning(destination)] +partitioner='planar' diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-dst_ukv.conf b/rose-stem/app/lfric2lfric/opt/rose-app-dst_ukv.conf new file mode 100644 index 0000000000..1a08a7de07 --- /dev/null +++ b/rose-stem/app/lfric2lfric/opt/rose-app-dst_ukv.conf @@ -0,0 +1,10 @@ +[namelist:lfric2lfric] +destination_mesh_name='ukv' +destination_meshfile_prefix='${MESH_DIR}/ukv/mesh_ukv' +destination_topology='non_periodic' +dst_ancil_directory='' +dst_orography_mean_ancil_path='' +target_domain='LAM' + +[namelist:partitioning(destination)] +partitioner='planar' diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-ral3.conf b/rose-stem/app/lfric2lfric/opt/rose-app-ral3.conf index 4235bafd08..6a2d1f9340 100644 --- a/rose-stem/app/lfric2lfric/opt/rose-app-ral3.conf +++ b/rose-stem/app/lfric2lfric/opt/rose-app-ral3.conf @@ -12,7 +12,6 @@ stretching_height=16641.984 stretching_method='smooth' [namelist:files] -checkpoint_stem_name='restart_lfric2lfric_ral3_seuk' !!lbc_directory='$BIG_DATA_DIR/lbcs/proto-ral/Ticket3510/seuk' !!lbc_filename='checkpoint_um2lfric_000001_fixedtime_date' start_dump_filename='lfric2lfric_ral3_seuk' diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-src_C224_L70.conf b/rose-stem/app/lfric2lfric/opt/rose-app-src_C224_L70.conf new file mode 100644 index 0000000000..841e265cc1 --- /dev/null +++ b/rose-stem/app/lfric2lfric/opt/rose-app-src_C224_L70.conf @@ -0,0 +1,30 @@ +[file:iodef.xml] +source=$ROSE_SUITE_DIR/app/lfric2lfric/file/iodef_orography.xml + +[file:source_data_C224.nc] +mode=symlink +source="/data/users/christine.johnson/lfric_data/lfric2lfric/startdump_C224.nc" + +[namelist:extrusion] +domain_height=80000.0 +method='um_L70_50t_20s_80km' +number_of_layers=70 +stretching_height=17507.0 +stretching_method='smooth' + +[namelist:files] +start_dump_filename='source_data_C224' + +[namelist:lfric2lfric] +source_mesh_name='dynamics' +source_meshfile_prefix='${MESH_DIR}/C224_MG/mesh_C224_MG' +src_ancil_directory='$BIG_DATA_DIR/ancils/basic-gal/yak/C224' +src_orography_mean_ancil_path='orography/gmted_ramp2/qrparm.orog.ugrid' + +[namelist:orography] +orog_init_option='none' + +[namelist:partitioning(source)] +panel_decomposition='custom' +panel_xproc=6 +panel_yproc=6 diff --git a/rose-stem/app/lfric2lfric/opt/rose-app-src_C896_L70.conf b/rose-stem/app/lfric2lfric/opt/rose-app-src_C896_L70.conf new file mode 100644 index 0000000000..fe5f381a69 --- /dev/null +++ b/rose-stem/app/lfric2lfric/opt/rose-app-src_C896_L70.conf @@ -0,0 +1,30 @@ +[file:iodef.xml] +source=$ROSE_SUITE_DIR/app/lfric2lfric/file/iodef_diag_file.xml + +[file:source_data_C896.nc] +mode=symlink +source="/data/users/christine.johnson/lfric_data/lfric2lfric/rho_theta_C896_t1.nc" + +[namelist:extrusion] +domain_height=80000.0 +method='um_L70_50t_20s_80km' +number_of_layers=70 +stretching_height=17507.0 +stretching_method='smooth' + +[namelist:files] +start_dump_filename='source_data_C896' + +[namelist:lfric2lfric] +source_mesh_name='dynamics' +source_meshfile_prefix='${MESH_DIR}/C896_MG/mesh_C896_MG' +src_ancil_directory='' +src_orography_mean_ancil_path='' + +[namelist:orography] +orog_init_option='none' + +[namelist:partitioning(source)] +panel_decomposition='custom' +panel_xproc=16 +panel_yproc=8 diff --git a/rose-stem/app/mesh/opt/rose-app-melbourne.conf b/rose-stem/app/mesh/opt/rose-app-melbourne.conf new file mode 100644 index 0000000000..8eb9c79bd2 --- /dev/null +++ b/rose-stem/app/mesh/opt/rose-app-melbourne.conf @@ -0,0 +1,23 @@ +[command] +default=mkdir -p $MESH_DIR ; mpiexec -n 1 $BIN_DIR/planar_mesh_generator mesh_generation.nml + +[env] +mesh_generator=Planar + +[!!namelist:cubedsphere_mesh] + +[namelist:mesh] +mesh_names='melbourne' +rotate_mesh=.true. +topology='non_periodic' + +[namelist:planar_mesh] +domain_centre=144.578,-37.472 +domain_size=3.456,3.456 +edge_cells_x=256 +edge_cells_y=256 +periodic_x=.false. +periodic_y=.false. + +[namelist:rotation] +target_north_pole=180.0,90.0 diff --git a/rose-stem/app/mesh/opt/rose-app-ukv.conf b/rose-stem/app/mesh/opt/rose-app-ukv.conf new file mode 100644 index 0000000000..6aa2501344 --- /dev/null +++ b/rose-stem/app/mesh/opt/rose-app-ukv.conf @@ -0,0 +1,32 @@ +[command] +default=mkdir -p $MESH_DIR ; mpiexec -n 1 $BIN_DIR/planar_mesh_generator mesh_generation.nml + +[env] +mesh_generator=Planar + +[!!namelist:cubedsphere_mesh] + +[namelist:mesh] +mesh_names='ukv' +rotate_mesh=.true. +topology='non_periodic' + +[namelist:planar_mesh] +domain_centre=-0.872500,1.6968 +domain_size=1,1 +edge_cells_x=960 +edge_cells_y=1024 +periodic_x=.false. +periodic_y=.false. +stretch_function='inflation' + +[namelist:rotation] +target_north_pole=177.5,37.5 + +[namelist:stretch_transform] +cell_size_inner=0.0135,0.0135 +cell_size_outer=0.036,0.036 +n_cells_outer=140,85 +n_cells_stretch=21,21 +stretching_on='p_points' +transform_mesh='ukv' diff --git a/rose-stem/site/common/lfric2lfric/tasks_lfric2lfric.cylc b/rose-stem/site/common/lfric2lfric/tasks_lfric2lfric.cylc index 855e37c85e..b28c83657f 100644 --- a/rose-stem/site/common/lfric2lfric/tasks_lfric2lfric.cylc +++ b/rose-stem/site/common/lfric2lfric/tasks_lfric2lfric.cylc @@ -216,6 +216,51 @@ "xios_min_buffer_size": 2097152, }) %} +{% elif task_ns.conf_name == "oasis_C224_to_Melbourne" %} + + {% do task_dict.update({ + "opt_confs": ["oasis","src_C224_L70","dst_melbourne"], + "resolution": ["melbourne","C224_MG"], + "dst_mesh": "melbourne", + "dst_name": "melbourne", + "dst_type": "regional", + "src_mesh": "C224_MG", + "src_name": "dynamics", + "src_type": "global", + "mpi_parts": 216, + "kgo_checks": [], + }) %} + +{% elif task_ns.conf_name == "oasis_C896_to_Melbourne" %} + + {% do task_dict.update({ + "opt_confs": ["oasis","src_C896_L70","dst_melbourne"], + "resolution": ["melbourne","C896_MG"], + "dst_mesh": "melbourne", + "dst_name": "melbourne", + "dst_type": "regional", + "src_mesh": "C896_MG", + "src_name": "dynamics", + "src_type": "global", + "mpi_parts": 768, + "memory": [250, "GB"], + "kgo_checks": [], + }) %} + +{% elif task_ns.conf_name == "oasis_C896_to_ukv" %} + + {% do task_dict.update({ + "opt_confs": ["oasis","src_C896_L70","dst_ukv"], + "resolution": ["ukv","C896_MG"], + "dst_mesh": "ukv", + "dst_name": "ukv", + "dst_type": "regional", + "src_mesh": "C896_MG", + "src_name": "dynamics", + "src_type": "global", + "mpi_parts": 768, + "kgo_checks": [], + }) %} {% elif task_ns.conf_name == "canned" %} {% do task_dict.update({ diff --git a/rose-stem/site/meto/groups/groups_lfric2lfric.cylc b/rose-stem/site/meto/groups/groups_lfric2lfric.cylc index 8ccdff8070..873ff90e41 100644 --- a/rose-stem/site/meto/groups/groups_lfric2lfric.cylc +++ b/rose-stem/site/meto/groups/groups_lfric2lfric.cylc @@ -61,6 +61,11 @@ "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_4cpu_ex1a_cce_fast-debug-64bit", "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_4cpu_ex1a_cce_fast-debug-64bit", ], + "lfric2lfric_ex1a_weekly": [ + "lfric2lfric_oasis_C224_to_Melbourne_ex1a_cce_fast-debug-64bit", + "lfric2lfric_oasis_C896_to_Melbourne_ex1a_cce_fast-debug-64bit", + "lfric2lfric_oasis_C896_to_ukv_ex1a_cce_fast-debug-64bit", + ], "lfric2lfric_ex1a": [ "lfric2lfric_ex1a_developer", "lfric2lfric_ex1a_extra", diff --git a/rose-stem/templates/default_directives.cylc b/rose-stem/templates/default_directives.cylc index 8abf4df340..d0c2a5b5f4 100644 --- a/rose-stem/templates/default_directives.cylc +++ b/rose-stem/templates/default_directives.cylc @@ -17,7 +17,7 @@ "tech-tests_memory": [6, "GB"], "tech-tests_wallclock": 20, "tech-tests_cpus": 1, - "gen_weights_tasks": 8, + "gen_weights_tasks": 12, "gen_weights_memory": [16, "GB"], } %} diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index 77b0552d2d..09be04d5df 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -81,6 +81,8 @@ 'test -f *-checksums.txt && cp *-checksums.txt $TASK_OUTPUT_DIR/checksum.txt', 'touch $CYLC_TASK_WORK_DIR/tmp.m', 'mv $CYLC_TASK_WORK_DIR/*.m $TASK_OUTPUT_DIR/results', + 'test -f $CYLC_TASK_WORK_DIR/lfric_checkpoint_write.nc && '~ + 'mv $CYLC_TASK_WORK_DIR/lfric_checkpoint_write.nc $TASK_OUTPUT_DIR/results || true', 'test -f $CYLC_TASK_WORK_DIR/timer.txt && '~ 'cp $CYLC_TASK_WORK_DIR/timer.txt $TASK_OUTPUT_DIR || true' ] %}