From dde5991189c7293584bb1f0d0dbfacd2e7f00eff Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Mon, 4 May 2026 15:47:47 -0600 Subject: [PATCH 01/24] Generalize the name of the 'mp_thompson_aers_in' package in init_atmosphere core The processing of GOCART-based water- and ice-friendly aerosols in the init_atmosphere core currently supports only the aerosol-aware Thompson microphysics scheme, but planned additions to MPAS-Atmosphere (specifically, the TEMPO microphysics scheme) will make use of these same aerosol fields. Accordingly, this commit generalizes the name of the 'mp_thompson_aers_in' package to 'microphysics_aerosols' in the init_atmosphere core. --- src/core_init_atmosphere/Registry.xml | 19 ++++++++-------- .../mpas_init_atm_core_interface.F | 22 +++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/core_init_atmosphere/Registry.xml b/src/core_init_atmosphere/Registry.xml index d45a115e33..8834c402a9 100644 --- a/src/core_init_atmosphere/Registry.xml +++ b/src/core_init_atmosphere/Registry.xml @@ -385,7 +385,7 @@ - + @@ -599,7 +599,7 @@ - + @@ -1025,15 +1025,15 @@ + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> + @@ -1210,11 +1211,11 @@ + packages="microphysics_aerosols"/> + packages="microphysics_aerosols"/> diff --git a/src/core_init_atmosphere/mpas_init_atm_core_interface.F b/src/core_init_atmosphere/mpas_init_atm_core_interface.F index f277a4a72f..288ab3b246 100644 --- a/src/core_init_atmosphere/mpas_init_atm_core_interface.F +++ b/src/core_init_atmosphere/mpas_init_atm_core_interface.F @@ -120,7 +120,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul logical, pointer :: config_native_gwd_static, config_static_interp, config_vertical_grid, config_met_interp logical, pointer :: config_native_gwd_gsl_static logical, pointer :: first_guess_field - logical, pointer :: mp_thompson_aers_in + logical, pointer :: microphysics_aerosols integer, pointer :: config_init_case logical, pointer :: noahmp, config_noahmp_static @@ -167,8 +167,8 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul nullify(met_stage_out) call mpas_pool_get_package(packages, 'met_stage_outActive', met_stage_out) - nullify(mp_thompson_aers_in) - call mpas_pool_get_package(packages, 'mp_thompson_aers_inActive', mp_thompson_aers_in) + nullify(microphysics_aerosols) + call mpas_pool_get_package(packages, 'microphysics_aerosolsActive', microphysics_aerosols) if (.not. associated(initial_conds) .or. & .not. associated(sfc_update) .or. & @@ -179,7 +179,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul .not. associated(vertical_stage_out) .or. & .not. associated(met_stage_in) .or. & .not. associated(met_stage_out) .or. & - .not. associated(mp_thompson_aers_in)) then + .not. associated(microphysics_aerosols)) then call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_ERR) call mpas_log_write('* Error while setting up packages for init_atmosphere core.', messageType=MPAS_LOG_ERR) call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_ERR) @@ -197,12 +197,12 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul if (config_init_case == 9) then lbcs = .true. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if(lexist) mp_thompson_aers_in = .true. + if(lexist) microphysics_aerosols = .true. else lbcs = .false. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. end if if (config_init_case == 7) then @@ -226,9 +226,9 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul (.not. config_vertical_grid) met_stage_out = config_met_interp - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) mp_thompson_aers_in = .true. + if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) microphysics_aerosols = .true. else if (config_init_case == 8) then gwd_stage_in = .false. @@ -252,9 +252,9 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul met_stage_in = .true. met_stage_out = .true. - mp_thompson_aers_in = .false. + microphysics_aerosols = .false. inquire(file="QNWFA_QNIFA_SIGMA_MONTHLY.dat",exist=lexist) - if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) mp_thompson_aers_in = .true. + if((lexist .and. met_stage_out) .or. (lexist .and. met_stage_in)) microphysics_aerosols = .true. initial_conds = .false. ! Also, turn off the initial_conds package to avoid writing the IC "output" stream From 0fd81c4c1d7e854eb9f819f646facf1697a892e7 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 09:17:54 -0400 Subject: [PATCH 02/24] Fix memory leak in mpas_io.F In the `mpas_io` module, the `MPAS_io_inq_var` subroutine allocates the `dimnames` array and returns it to a calling routine. In this case, the calling routine is `MPAS_io_def_var`, which has a local pointer array, `inq_dimnames`, that corresponds to the allocated `dimnames` and must be deallocated before this routine returns. Direct leak of 152576 byte(s) in 162 object(s) allocated from: #0 0x7fe7202ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000024709b7 in __mpas_io_MOD_mpas_io_inq_var /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:1123 #2 0x000002468a5b in __mpas_io_MOD_mpas_io_def_var /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:1222 #3 0x000002613e0a in __mpas_io_streams_MOD_mpas_streamaddfield_generic /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io_streams.F:1761 #4 0x000002639406 in __mpas_io_streams_MOD_mpas_streamaddfield_1dreal /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io_streams.F:959 #5 0x0000024a69fa in build_stream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:4363 #6 0x0000024b13d7 in write_stream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:3251 #7 0x0000024b6afd in __mpas_stream_manager_MOD_mpas_stream_mgr_write /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:2858 #8 0x00000094afb5 in __atm_core_MOD_atm_core_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/core_atmosphere/mpas_atm_core.F:620 #9 0x000000406341 in __mpas_subdriver_MOD_mpas_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:416 #10 0x000000402942 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:20 #11 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #12 0x7fe71f208680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #13 0x7fe71f208797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #14 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: c0332077ce07f118fef380e98497523209704432) --- src/framework/mpas_io.F | 1 + 1 file changed, 1 insertion(+) diff --git a/src/framework/mpas_io.F b/src/framework/mpas_io.F index 5b3b5642bb..d5483be69c 100644 --- a/src/framework/mpas_io.F +++ b/src/framework/mpas_io.F @@ -1256,6 +1256,7 @@ subroutine MPAS_io_def_var(handle, fieldname, fieldtype, dimnames, precision, ie ! TODO: Can we get the dimension sizes to see whether they match those from the file? end if + if (associated(inq_dimnames)) deallocate(inq_dimnames) return end if From 022e5ca541393d616c36cc6fe3193d3721e50418 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 09:37:33 -0400 Subject: [PATCH 03/24] Fix memory leak in mpas_stream_manager.F This one was difficult to find. Backtrace, for some reason, did not point to the exact locations in the source code where subroutine calls are located. In `mpas_stream_list.F` in the `MPAS_stream_list_create` subroutine, memory is allocated for `list` pointer. This routine is called from MPAS_stream_mgr_add_alarmat line 1271 and stored in new_alarm % streamList. The new_alarm itself is allocated a few lines before (1269), and inserted into manager % alarms_in at line 1279: 1266 ! If alarm is not already defined, we need to create a new alarm node 1267 nullify(new_alarm) 1268 if (.not. MPAS_stream_list_query(manager % alarms_in, alarmID, new_alarm, ierr=err_local)) then 1269 allocate(new_alarm) 1270 new_alarm % name = alarmID 1271 call MPAS_stream_list_create(new_alarm % streamList, ierr=err_local) 1272 if (err_local /= MPAS_STREAM_LIST_NOERR) then 1273 if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR 1274 STREAM_ERROR_WRITE('Problems while creating stream list for alarm') 1275 return 1276 end if 1277 nullify(new_alarm % next) 1278 1279 call MPAS_stream_list_insert(manager % alarms_in, new_alarm, ierr=err_local) 1280 if (err_local /= MPAS_STREAM_LIST_NOERR) then 1281 if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR 1282 STREAM_ERROR_WRITE('Problems while adding input alarm to list') 1283 return 1284 end if 1285 end if Similar actions are taken for manager % alarms_out. To prevent memory leaks, before freeing up the list of input/output alarms we must first destroy the list of "sub" alarms added to these two lists (manager % alarms_in and manager % alarms_out) in MPAS_stream_mgr_finalize. Direct leak of 15456 byte(s) in 7 object(s) allocated from: #0 0x7f59c3eef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x000002580ef6 in __mpas_stream_list_MOD_mpas_stream_list_create /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_list.F:38 #2 0x0000024c7397 in __mpas_stream_manager_MOD_mpas_stream_mgr_add_alarm /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:1242 #3 0x0000024d6bd7 in stream_mgr_add_alarm_c /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:6198 #4 0x0000024fc7c1 in xml_stream_parser /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/xml_stream_parser.c:1348 #5 0x00000040c916 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:42 #6 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #7 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #8 0x7f59c2e08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x7f59c2e08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #10 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 3a2362d27fd37ec758776277d1ca8e2b27dea473) --- src/framework/mpas_stream_manager.F | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/framework/mpas_stream_manager.F b/src/framework/mpas_stream_manager.F index 8a4a1b4ad3..609663ed58 100644 --- a/src/framework/mpas_stream_manager.F +++ b/src/framework/mpas_stream_manager.F @@ -245,6 +245,19 @@ subroutine MPAS_stream_mgr_finalize(manager, ierr)!{{{ STREAM_ERROR_WRITE('Problems while destroying stream list') end if + ! + ! Remove all items from manager % alarms_in(put) list + ! + stream_cursor => manager % alarms_in % head + do while (associated(stream_cursor)) + call MPAS_stream_list_destroy(stream_cursor % streamList, ierr=err_local) + if (err_local /= MPAS_STREAM_LIST_NOERR) then + if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + STREAM_ERROR_WRITE('Problems while destroying input alarms item') + end if + stream_cursor => stream_cursor % next + end do + ! ! Free up list of input alarms ! @@ -254,6 +267,19 @@ subroutine MPAS_stream_mgr_finalize(manager, ierr)!{{{ STREAM_ERROR_WRITE('Problems while destroying input alarms list') end if + ! + ! Remove all items from manager % alarms_out(put) list + ! + stream_cursor => manager % alarms_out % head + do while (associated(stream_cursor)) + call MPAS_stream_list_destroy(stream_cursor % streamList, ierr=err_local) + if (err_local /= MPAS_STREAM_LIST_NOERR) then + if (present(ierr)) ierr = MPAS_STREAM_MGR_ERROR + STREAM_ERROR_WRITE('Problems while destroying output alarms list') + end if + stream_cursor => stream_cursor % next + end do + ! ! Free up list of output alarms ! From b464355a33f238682178cabe1edfe3a6ab51c9cb Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 09:45:11 -0400 Subject: [PATCH 04/24] Fix memory leak in stream_inquiry.c Memory is allocated in `read_and_broadcast` function in stream_inquiry.c and its address returned to `parse_streams_file` via `xml_buf` pointer. This pointer is then passed to `ezxml_parse_str` function, but it is never freed in `parse_streams_file` nor returned to the function that called `parse_streams_file`. This means the ownership of the memory that xml_buf points to is transferred to ezxml_parse_str. It is responsible for freeing that memory. In `ezxml_parse_str` the pointer to that memory buffer (char *s) is saved in `root->m` so that it can be freed later in `ezxml_free()`. To indicate that root->m should be freed, root->len must be set to -1. This is how it's done in other functions. Direct leak of 2915 byte(s) in 1 object(s) allocated from: #0 0x7f329f6ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000024ff074 in read_and_broadcast /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/stream_inquiry.c:83 #2 0x0000024ff2da in parse_streams_file /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/stream_inquiry.c:146 #3 0x000002489642 in __mpas_stream_inquiry_MOD_streaminfo_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_inquiry.F:137 #4 0x000000409cad in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:267 #5 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #6 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #7 0x7f329e408680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #8 0x7f329e408797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: ab61abfcc5d576737ec24283b11ac29c6cd28819) --- src/external/ezxml/ezxml.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/external/ezxml/ezxml.c b/src/external/ezxml/ezxml.c index 91ae2bc1d9..88f0002b0c 100644 --- a/src/external/ezxml/ezxml.c +++ b/src/external/ezxml/ezxml.c @@ -481,6 +481,7 @@ ezxml_t ezxml_parse_str(char *s, size_t len) int l, i, j; root->m = s; + root->len = -1; // so we know to free s in ezxml_free() if (! len) return ezxml_err(root, NULL, "root tag missing"); root->u = ezxml_str2utf8(&s, &len); // convert utf-16 to utf-8 root->e = (root->s = s) + len; // record start and end of work area From 51ee1c49502d2ddf73a24f3ced70ee903ed62969 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 10:02:33 -0400 Subject: [PATCH 05/24] Fix memory leak in mpas_io.F In `MPAS_io_put_att_text` (and also in other *_io_put_att_*) subroutine `new_attlist_node` is allocated and added as a variable attribute to a field (if fieldname is present) or as a global attribute to a handle. It should be deleted in `MPAS_io_close`. Direct leak of 2592 byte(s) in 162 object(s) allocated from: #0 0x7fd0fbaef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000023be0e0 in __mpas_io_MOD_mpas_io_put_att_text /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:6066 #2 0x000002581ccd in __mpas_io_streams_MOD_put_get_field_atts /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io_streams.F:4341 #3 0x00000263a195 in __mpas_io_streams_MOD_mpas_streamaddfield_1dreal /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io_streams.F:976 #4 0x0000024a6aac in build_stream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:4389 #5 0x0000024b1489 in write_stream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:3277 #6 0x0000024b6baf in __mpas_stream_manager_MOD_mpas_stream_mgr_write /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:2884 #7 0x00000094df25 in __atm_core_MOD_atm_core_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/core_atmosphere/mpas_atm_core.F:754 #8 0x000000406341 in __mpas_subdriver_MOD_mpas_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:416 #9 0x000000402942 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:20 #10 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #11 0x7fd0fa808680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #12 0x7fd0fa808797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #13 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: d86a2755908f751ab1aef6095ced60ba65202ab8) --- src/framework/mpas_io.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework/mpas_io.F b/src/framework/mpas_io.F index d5483be69c..fa7e35598c 100644 --- a/src/framework/mpas_io.F +++ b/src/framework/mpas_io.F @@ -6336,6 +6336,7 @@ subroutine MPAS_io_close(handle, ierr) if (attlist_del % atthandle % attType == MPAS_ATT_INTA) deallocate(attlist_del % atthandle % attValueIntA) if (attlist_del % atthandle % attType == MPAS_ATT_REALA) deallocate(attlist_del % atthandle % attValueRealA) deallocate(attlist_del % atthandle) + deallocate(attlist_del) end do nullify(fieldlist_del % fieldhandle % attlist_head) nullify(fieldlist_del % fieldhandle % attlist_tail) @@ -6363,6 +6364,7 @@ subroutine MPAS_io_close(handle, ierr) if (attlist_del % atthandle % attType == MPAS_ATT_INTA) deallocate(attlist_del % atthandle % attValueIntA) if (attlist_del % atthandle % attType == MPAS_ATT_REALA) deallocate(attlist_del % atthandle % attValueRealA) deallocate(attlist_del % atthandle) + deallocate(attlist_del) end do nullify(handle % attlist_head) nullify(handle % attlist_tail) From ec9349954c0d0d15c933f26c4df58a97612306ef Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 10:31:18 -0400 Subject: [PATCH 06/24] Fix memory leak in mpas_pool_routines.F In all subroutines that add a new field to a pool, like `mpas_pool_add_field_2d_real` in this case, a new mpas_pool_member_type is allocated, with `allocate(newmem)`, and then a `mpas_pool_data_type` is allocated as a `data` component of newmem, with `allocate(newmem % data)`. When pool members are deallocated (in `pool_remove_member`) before deallocating the pointer that points to a pool member type, a pool data type component must also be deallocated. Direct leak of 1056 byte(s) in 1 object(s) allocated from: #0 0x7f94afcef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000021f2834 in __mpas_pool_routines_MOD_mpas_pool_add_field_2d_real /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_pool_routines.F:2215 #2 0x0000023af006 in __mpas_halo_MOD_mpas_halo_exch_group_add_field /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_halo.F:511 #3 0x000000964b6b in __mpas_atm_halos_MOD_atm_build_halo_groups /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/core_atmosphere/mpas_atm_halos.F:191 #4 0x00000095a6f8 in __atm_core_MOD_atm_core_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/core_atmosphere/mpas_atm_core.F:103 #5 0x00000040ce79 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:394 #6 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #7 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #8 0x7f94aec08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x7f94aec08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #10 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 1c56ff6f6b8a8860cf4bb09f1474a7cfce529289) --- src/framework/mpas_pool_routines.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework/mpas_pool_routines.F b/src/framework/mpas_pool_routines.F index 80339e3112..4f57211deb 100644 --- a/src/framework/mpas_pool_routines.F +++ b/src/framework/mpas_pool_routines.F @@ -5795,6 +5795,7 @@ logical function pool_remove_member(inPool, key, memType)!{{{ end if !TODO: are there cases where we need to delete more data here? + if (associated(ptr_prev % data)) deallocate(ptr_prev % data) deallocate(ptr_prev) end if pool_remove_member = .true. @@ -5828,6 +5829,7 @@ logical function pool_remove_member(inPool, key, memType)!{{{ end if !TODO: are there cases where we need to delete more data here? + if (associated(ptr % data)) deallocate(ptr % data) deallocate(ptr) end if pool_remove_member = .true. From e0969a6c862b28aec7adb77bdd914978978500ff Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 11:17:01 -0400 Subject: [PATCH 07/24] Fix memory leak in mpas_framework.F and mpas_domain_routines.F The `dminfo` component of the `dom` object is allocated twice, first in `mpas_allocate_domain` and then in `mpas_framework_init_phase1`. It is never deallocated. The second allocation is unnecessary and should be removed. The deallocation should be added to `mpas_deallocate_domain`. Also, `mpas_deallocate_domain` is called too early in `mpas_framework_finalize`, it should be called after calls to `mpas_dmpar_finalize` and `mpas_finish_block_proc_list`. In fact I think `mpas_deallocate_domain` should not be called at all from `mpas_framework_finalize`, but from the module that called `mpas_allocate_domain`, which is `mpas_subdriver`. Direct leak of 160 byte(s) in 1 object(s) allocated from: #0 0x7f8370cef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000215d335 in __mpas_domain_routines_MOD_mpas_allocate_domain /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_domain_routines.F:46 #2 0x00000040908c in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:194 #3 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #4 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #5 0x7f836fa08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #6 0x7f836fa08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #7 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: e8ced93cd7d700e184ed309fa9f3c945cd2d3e8a) Direct leak of 160 byte(s) in 1 object(s) allocated from: #0 0x7f8370cef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000236c37d in __mpas_framework_MOD_mpas_framework_init_phase1 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_framework.F:68 #2 0x000000409131 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:202 #3 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #4 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #5 0x7f836fa08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #6 0x7f836fa08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #7 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: e8ced93cd7d700e184ed309fa9f3c945cd2d3e8a) --- src/framework/mpas_domain_routines.F | 2 ++ src/framework/mpas_framework.F | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/framework/mpas_domain_routines.F b/src/framework/mpas_domain_routines.F index 5d6c563cf7..0a4ee8eddf 100644 --- a/src/framework/mpas_domain_routines.F +++ b/src/framework/mpas_domain_routines.F @@ -138,6 +138,8 @@ subroutine mpas_deallocate_domain(dom)!{{{ deallocate(dom % clock) deallocate(dom % ioContext) + deallocate(dom % dminfo) + end subroutine mpas_deallocate_domain!}}} diff --git a/src/framework/mpas_framework.F b/src/framework/mpas_framework.F index 10367a3d9c..3aa4ad7b2f 100644 --- a/src/framework/mpas_framework.F +++ b/src/framework/mpas_framework.F @@ -65,7 +65,6 @@ subroutine mpas_framework_init_phase1(dminfo, external_comm)!{{{ integer, intent(in), optional :: external_comm #endif - allocate(dminfo) call mpas_dmpar_init(dminfo, external_comm) end subroutine mpas_framework_init_phase1!}}} @@ -182,14 +181,14 @@ subroutine mpas_framework_finalize(dminfo, domain, io_system)!{{{ call MPAS_io_finalize(domain % ioContext, .false.) - call mpas_deallocate_domain(domain) - call mpas_dmpar_finalize(dminfo) call mpas_finish_block_proc_list(dminfo) call mpas_timekeeping_finalize() + call mpas_deallocate_domain(domain) + end subroutine mpas_framework_finalize!}}} From af276d249b60f5f0007ebe2787812b578df9e800 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 5 May 2026 12:06:23 -0400 Subject: [PATCH 08/24] Fix memory leak in mpas_domain_routines.F In `mpas_domain_routines.F` several pool objects which are components of the domain and block structure are allocated explicitly in `mpas_allocate_domain` and `mpas_allocate_block` and then again in `mpas_pool_create_pool` subroutiens called from these two functions. To avoid memory leaks the explicit allocations should be removed. Direct leak of 96 byte(s) in 1 object(s) allocated from: #0 0x7f5ba74ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000215d0f0 in __mpas_domain_routines_MOD_mpas_allocate_block /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_domain_routines.F:91 #2 0x000002529bfd in __mpas_block_creator_MOD_mpas_block_creator_finalize_block_phase1 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_block_creator.F:1017 #3 0x000002158a90 in __mpas_bootstrapping_MOD_mpas_bootstrap_framework_phase1 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_bootstrapping.F:385 #4 0x00000040c49c in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:356 #5 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #6 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #7 0x7f5ba6208680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #8 0x7f5ba6208797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: f25761ad95db227da9a5bcfb5c7400f929e4a29e) Direct leak of 96 byte(s) in 1 object(s) allocated from: #0 0x7f5ba74ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000215dafc in __mpas_domain_routines_MOD_mpas_allocate_domain /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_domain_routines.F:50 #2 0x00000040908c in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:194 #3 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #4 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #5 0x7f5ba6208680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #6 0x7f5ba6208797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #7 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: f25761ad95db227da9a5bcfb5c7400f929e4a29e) --- src/framework/mpas_domain_routines.F | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/framework/mpas_domain_routines.F b/src/framework/mpas_domain_routines.F index 0a4ee8eddf..f0fb97ed9e 100644 --- a/src/framework/mpas_domain_routines.F +++ b/src/framework/mpas_domain_routines.F @@ -46,10 +46,7 @@ subroutine mpas_allocate_domain(dom)!{{{ allocate(dom % dminfo) nullify(dom % blocklist) - allocate(dom % configs) - allocate(dom % packages) allocate(dom % clock) - allocate(dom % streamManager) allocate(dom % ioContext) call mpas_pool_create_pool(dom % configs) @@ -87,9 +84,6 @@ subroutine mpas_allocate_block(nHaloLayers, b, dom, blockID) !{{{ b % domain => dom - allocate(b % structs) - allocate(b % dimensions) - allocate(b % allFields) call mpas_pool_create_pool(b % structs) call mpas_pool_create_pool(b % dimensions) call mpas_pool_create_pool(b % allFields) From e3ed44654f19dfff64db06fb15b8734a400cbd4b Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 13:50:15 -0400 Subject: [PATCH 09/24] Fix memory leak in xml_stream_parser In `xml_stream_parser` function there is this block of code: packages = strdup(packagelist); package = strsep(&packages, ";"); ... // more calls to strsep free(packages); Here, `strdup` allocates memory for a copy of `packagelist` and saves a pointer to that memory block in `packages`, which is later used in `free`. However, the `strsep` function modifies that pointer by advancing it through the allocated memory as it parses the string. It eventually returns NULL, which caused free to fail to deallocate memory. Solution is simply to keep a copy of the original pointer and use it to free the allocated memory. Direct leak of 58 byte(s) in 5 object(s) allocated from: #0 0x7fcf6c6e8ac4 in strdup (/lib64/libasan.so.8+0xe8ac4) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000024fc518 in xml_stream_parser /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/xml_stream_parser.c:1381 #2 0x00000040c916 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:42 #3 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #4 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #5 0x7fcf6b608680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #6 0x7fcf6b608797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #7 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 66ceb08b660eaf75bed2d3da86b4743a3541832e) --- src/framework/xml_stream_parser.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/framework/xml_stream_parser.c b/src/framework/xml_stream_parser.c index 00b22fd009..470cc0fe63 100644 --- a/src/framework/xml_stream_parser.c +++ b/src/framework/xml_stream_parser.c @@ -1057,7 +1057,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) const char *streamID2, *interval_in2, *interval_out2; char interval_name[256]; char match_stream_name[256]; - char *packages, *package; + char *packages, *packages_ptr, *package; char filename_interval_string[256]; char ref_time_local[256]; char rec_intv_local[256]; @@ -1379,6 +1379,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) if (packagelist != NULL) { packages = strdup(packagelist); + packages_ptr = packages; package = strsep(&packages, ";"); stream_mgr_add_pkg_c(manager, streamID, package, &err); @@ -1403,7 +1404,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) } } - free(packages); + free(packages_ptr); } } @@ -1688,6 +1689,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) if (packagelist != NULL) { packages = strdup(packagelist); + packages_ptr = packages; package = strsep(&packages, ";"); stream_mgr_add_pkg_c(manager, streamID, package, &err); @@ -1712,7 +1714,7 @@ void xml_stream_parser(char *fname, void *manager, int *mpi_comm, int *status) } } - free(packages); + free(packages_ptr); } for (varfile_xml = ezxml_child(stream_xml, "file"); varfile_xml; varfile_xml = ezxml_next(varfile_xml)) { From 15d769e321b15bd6cee514bee046217dabe04a61 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 14:14:52 -0400 Subject: [PATCH 10/24] Fix memory leak in mpas_io.F These two leaks are similar. In `MPAS_io_inq_dim` new dimlist_type node is allocated (line 594), in `MPAS_io_inq_var` new fieldlist_type node is allocated (line 860). They must be deallocated in `MPAS_io_close` Direct leak of 48 byte(s) in 3 object(s) allocated from: #0 0x7fcf4c0ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x000002472c16 in __mpas_io_MOD_mpas_io_inq_dim /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:594 #2 0x00000214f5ad in __mpas_bootstrapping_MOD_mpas_bootstrap_framework_phase2 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_bootstrapping.F:612 #3 0x00000040cdd6 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:389 #4 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #5 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #6 0x7fcf4ae08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #7 0x7fcf4ae08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #8 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 002a28b14293b75e2eca1ab9aeac8be59516cd63) Direct leak of 32 byte(s) in 2 object(s) allocated from: #0 0x7fcf4c0ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000246bba9 in __mpas_io_MOD_mpas_io_inq_var /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:860 #2 0x000002650ba8 in __mpas_io_streams_MOD_mpas_seekstream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io_streams.F:339 #3 0x0000024b2409 in write_stream /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:3353 #4 0x0000024b640d in __mpas_stream_manager_MOD_mpas_stream_mgr_write /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:2884 #5 0x00000094df25 in __atm_core_MOD_atm_core_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/core_atmosphere/mpas_atm_core.F:754 #6 0x000000406341 in __mpas_subdriver_MOD_mpas_run /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:416 #7 0x000000402942 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:20 #8 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #9 0x7fcf4ae08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #10 0x7fcf4ae08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #11 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 002a28b14293b75e2eca1ab9aeac8be59516cd63) --- src/framework/mpas_io.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework/mpas_io.F b/src/framework/mpas_io.F index fa7e35598c..a96907b7cb 100644 --- a/src/framework/mpas_io.F +++ b/src/framework/mpas_io.F @@ -6344,6 +6344,7 @@ subroutine MPAS_io_close(handle, ierr) deallocate(fieldlist_del % fieldhandle % dims) deallocate(fieldlist_del % fieldhandle) + deallocate(fieldlist_del) end do nullify(handle % fieldlist_head) nullify(handle % fieldlist_tail) @@ -6353,6 +6354,7 @@ subroutine MPAS_io_close(handle, ierr) dimlist_del => dimlist_ptr dimlist_ptr => dimlist_ptr % next deallocate(dimlist_del % dimhandle) + deallocate(dimlist_del) end do nullify(handle % dimlist_head) nullify(handle % dimlist_tail) From 8ddd20f9438290f4a32508b779e6ce828b6ec07c Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 15:21:42 -0400 Subject: [PATCH 11/24] Fix memory leak in mpas_domain_routines.F In `mpas_dmpar_init_multihalo_exchange_list` subrotine called from mpas_block_creator.F exchList is allocated. The exchLists has halo components, which are also exchLists. These are passed back to mpas_block_creator and are eventually saved into the block's exchange lists: cellsToSend, cellsToRecv, cellsToCopy, edgesToSend, edgesToRecv, verticesToSend, verticesToRecv and verticesToCopy. When these lists are deallocated in `mpas_deallocate_block`, instead of just deallocating exchList pointers, a subroutine `mpas_dmpar_destroy_mulithalo_exchange_list` should be used to properly destroy and deallocate halo subcomponents. Direct leak of 24 byte(s) in 1 object(s) allocated from: #0 0x7fdff4aef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000022ce9d8 in __mpas_dmpar_MOD_mpas_dmpar_init_multihalo_exchange_list /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_dmpar.F:6466 #2 0x0000025501f4 in __mpas_block_creator_MOD_mpas_block_creator_build_0_and_1halo_edge_fields /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_block_creator.F:334 #3 0x000002156ba9 in __mpas_bootstrapping_MOD_mpas_bootstrap_framework_phase1 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_bootstrapping.F:266 #4 0x00000040c49c in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:356 #5 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #6 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #7 0x7fdff3808680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #8 0x7fdff3808797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 94eaf12a29734d8a2ed692e6fb9b037dcf35135f) --- src/framework/mpas_domain_routines.F | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/framework/mpas_domain_routines.F b/src/framework/mpas_domain_routines.F index f0fb97ed9e..7d2c289c12 100644 --- a/src/framework/mpas_domain_routines.F +++ b/src/framework/mpas_domain_routines.F @@ -163,17 +163,17 @@ subroutine mpas_deallocate_block(b)!{{{ call mpas_pool_destroy_pool(b % structs) call mpas_pool_destroy_pool(b % dimensions) - deallocate(b % parinfo % cellsToSend) - deallocate(b % parinfo % cellsToRecv) - deallocate(b % parinfo % cellsToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % cellsToCopy) - deallocate(b % parinfo % edgesToSend) - deallocate(b % parinfo % edgesToRecv) - deallocate(b % parinfo % edgesToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % edgesToCopy) - deallocate(b % parinfo % verticesToSend) - deallocate(b % parinfo % verticesToRecv) - deallocate(b % parinfo % verticesToCopy) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToSend) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToRecv) + call mpas_dmpar_destroy_mulithalo_exchange_list(b % parinfo % verticesToCopy) deallocate(b % parinfo) From 30ef8827b80c0c7dfe9739c60bd82b4649a5b077 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 16:43:05 -0400 Subject: [PATCH 12/24] Fix memory leak in mpas_pool_routines.F In `mpas_pool_add_package` subroutine which adds a package to a pool, after the new pool member is allocated, the data subcomponent is allocated, which also allocates the logical value `simple_logical`. That value must be deallocated before the data pointer (dptr) is deallocated in `mpas_pool_destroy_pool`. Direct leak of 20 byte(s) in 5 object(s) allocated from: #0 0x7f85b0cef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000021bfb89 in __mpas_pool_routines_MOD_mpas_pool_add_package /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_pool_routines.F:5207 #2 0x0000024baa91 in __mpas_stream_manager_MOD_mpas_stream_mgr_add_pkg /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:2297 #3 0x0000024d68ba in stream_mgr_add_pkg_c /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_stream_manager.F:6321 #4 0x0000024fbede in xml_stream_parser /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/xml_stream_parser.c:1385 #5 0x00000040c916 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:42 #6 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #7 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #8 0x7f85afc08680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #9 0x7f85afc08797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: f83cdcaffde35587075ea80e05bcb894963a4bdd) #10 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: bec3816af189fbcb144ed6ff0cbaef36c9dc9016) --- src/framework/mpas_pool_routines.F | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/framework/mpas_pool_routines.F b/src/framework/mpas_pool_routines.F index 4f57211deb..ad7df945b3 100644 --- a/src/framework/mpas_pool_routines.F +++ b/src/framework/mpas_pool_routines.F @@ -229,6 +229,14 @@ recursive subroutine mpas_pool_destroy_pool(inPool)!{{{ deallocate(ptr % data % simple_int) end if + else if (ptr % contentsType == MPAS_POOL_PACKAGE) then + + dptr => ptr % data + + if (dptr % contentsType == MPAS_POOL_LOGICAL) then + deallocate(dptr % simple_logical) + end if + else if (ptr % contentsType == MPAS_POOL_CONFIG) then dptr => ptr % data From 613bd0f178ea2bc8a0a52494fd47cae69f480c28 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 16:55:13 -0400 Subject: [PATCH 13/24] Fix memory leak in mpas_block_creator.F In `mpas_block_creator_build_0halo_cell_fields` local array `sendingHaloLayers` must be deallocated before this subrotine returns Direct leak of 4 byte(s) in 1 object(s) allocated from: #0 0x7fb58d2ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000025538dc in __mpas_block_creator_MOD_mpas_block_creator_build_0halo_cell_fields /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_block_creator.F:160 #2 0x000002156b05 in __mpas_bootstrapping_MOD_mpas_bootstrap_framework_phase1 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_bootstrapping.F:259 #3 0x00000040c49c in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:356 #4 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #5 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #6 0x7fb58c008680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #7 0x7fb58c008797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #8 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 62d0976668433575208c249f3d706b6b133aa425) --- src/framework/mpas_block_creator.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework/mpas_block_creator.F b/src/framework/mpas_block_creator.F index e9fea7253f..73b45fe8de 100644 --- a/src/framework/mpas_block_creator.F +++ b/src/framework/mpas_block_creator.F @@ -249,6 +249,8 @@ subroutine mpas_block_creator_build_0halo_cell_fields(nHalos, indexToCellIDBlock call mpas_dmpar_alltoall_field(cellsOnCellBlock, cellsOnCell_0Halo, sendingHaloLayers) call mpas_dmpar_alltoall_field(verticesOnCellBlock, verticesOnCell_0Halo, sendingHaloLayers) call mpas_dmpar_alltoall_field(edgesOnCellBlock, edgesOnCell_0Halo, sendingHaloLayers) + + deallocate(sendingHaloLayers) end subroutine mpas_block_creator_build_0halo_cell_fields!}}} !*********************************************************************** From ae45a181d9ca6657ea0728b5516a635dd96d334a Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 17:35:07 -0400 Subject: [PATCH 14/24] Fix memory leak in block_dimension_routines.inc In `atm_setup_derived_dimensions` subroutine dimension values are allocated locally if they are not already in a dimensionPool, and they must be deleted after being added to the dimensionPool. Since this is a generated routine, the changes must be made in src/tools/registry/gen_inc.c Direct leak of 4 byte(s) in 1 object(s) allocated from: #0 0x7fa3a06ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x0000008e638a in __atm_core_interface_MOD_atm_setup_derived_dimensions /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/src/core_atmosphere/block_dimension_routines.inc:171 #2 0x00000251c432 in __mpas_block_creator_MOD_mpas_block_creator_finalize_block_phase2 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_block_creator.F:1275 #3 0x000002151669 in __mpas_bootstrapping_MOD_mpas_bootstrap_framework_phase2 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_bootstrapping.F:675 #4 0x00000040cdd6 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:389 #5 0x000000402934 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #6 0x0000004029b2 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #7 0x7fa39f608680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #8 0x7fa39f608797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #9 0x000000402844 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x402844) (BuildId: 66c1f6616f19ebcc31fb59115fab767050354512) --- src/tools/registry/gen_inc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/registry/gen_inc.c b/src/tools/registry/gen_inc.c index 5ec8b06d18..078a4d399a 100644 --- a/src/tools/registry/gen_inc.c +++ b/src/tools/registry/gen_inc.c @@ -1086,6 +1086,7 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ } fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); + fortprintf(fd, " deallocate(%s)\n", dimname); fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); // Namelist defined dimension if(strncmp(dimdef, "namelist:", 9) == 0){ @@ -1101,6 +1102,7 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, " allocate(%s)\n", dimname); fortprintf(fd, " %s = MPAS_MISSING_DIM\n", dimname); fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); + fortprintf(fd, " deallocate(%s)\n", dimname); fortprintf(fd, " end if\n\n"); } } From 3bc0c4e3229a9349f4e12f5cec20ffde0eab1882 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 6 May 2026 18:13:30 -0400 Subject: [PATCH 15/24] Fix memory leak in mpas_io.F In `MPAS_io_init` it is not necessary to allocate `ioContext % smiol_context`, it will be allocated in `SMIOLf_init` Direct leak of 28 byte(s) in 1 object(s) allocated from: #0 0x7f6c6e4ef41f in malloc (/lib64/libasan.so.8+0xef41f) (BuildId: 2a7b18228484f7c17d959b022f3ff6aaaf687b3c) #1 0x00000242e24c in __mpas_io_MOD_mpas_io_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_io.F:213 #2 0x00000236bff6 in __mpas_framework_MOD_mpas_framework_init_phase2 /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/framework/mpas_framework.F:155 #3 0x000000408f59 in __mpas_subdriver_MOD_mpas_init /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas_subdriver.F:257 #4 0x000000401fb4 in mpas /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:18 #5 0x000000402032 in main /home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/src/driver/mpas.F:10 #6 0x7f6c6d408680 in __libc_start_call_main (/lib64/libc.so.6+0x3680) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #7 0x7f6c6d408797 in __libc_start_main@GLIBC_2.2.5 (/lib64/libc.so.6+0x3797) (BuildId: 79bea8ccf296ed42137a854a0cb75020bb783545) #8 0x000000401ec4 in _start (/home/dusan/sufs/simple-mpas/src/MPAS-Model-develop/build/bin/mpas_atmosphere+0x401ec4) (BuildId: 4e38fd7d901cf1506c34e3522552d885e5afaebd) --- src/framework/mpas_io.F | 1 - 1 file changed, 1 deletion(-) diff --git a/src/framework/mpas_io.F b/src/framework/mpas_io.F index a96907b7cb..724e71b802 100644 --- a/src/framework/mpas_io.F +++ b/src/framework/mpas_io.F @@ -210,7 +210,6 @@ subroutine MPAS_io_init(ioContext, io_task_count, io_task_stride, io_system, ier #endif #ifdef MPAS_SMIOL_SUPPORT - allocate(ioContext % smiol_context) #ifdef MPAS_USE_MPI_F08 local_ierr = SMIOLf_init(ioContext % dminfo % comm % mpi_val, & #else From 7d0cbee3e56bbf8d5c5ed9ee1674d33ef1020998 Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Thu, 7 May 2026 17:48:50 -0600 Subject: [PATCH 16/24] Provide default initialization for members of the met_data derived type This commit adds default initialization for the members of the met_data derived type in the init_atm_read_met module. Without well-defined values for some members, unpredictable floating-point exceptions were possible, e.g., Program received signal SIGFPE: Floating-point exception - erroneous arithmetic operation. Backtrace for this error: #0 0x15494ec288ff in ??? #1 0x59ddc0 in __init_atm_read_met_MOD_read_next_met_field at src/core_init_atmosphere/mpas_init_atm_read_met.F:388 #2 0x4ccb7a in __init_atm_cases_MOD_init_atm_case_lbc at src/core_init_atmosphere/mpas_init_atm_cases.F:6218 #3 0x58a373 in __init_atm_cases_MOD_init_atm_setup_case at src/core_init_atmosphere/mpas_init_atm_cases.F:327 #4 0x497807 in __init_atm_core_MOD_init_atm_core_run at src/core_init_atmosphere/mpas_init_atm_core.F:92 #5 0x406084 in __mpas_subdriver_MOD_mpas_run at src/driver/mpas_subdriver.F:416 #6 0x4046da in mpas at src/driver/mpas.F:20 #7 0x404745 in main at src/driver/mpas.F:10 By initializing the members of the met_data type, these unpredictable errors can be avoided. --- .../mpas_init_atm_read_met.F | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/core_init_atmosphere/mpas_init_atm_read_met.F b/src/core_init_atmosphere/mpas_init_atm_read_met.F index 69c662766b..7b46c37d49 100644 --- a/src/core_init_atmosphere/mpas_init_atm_read_met.F +++ b/src/core_init_atmosphere/mpas_init_atm_read_met.F @@ -25,17 +25,37 @@ module init_atm_read_met ! Derived types type met_data - integer :: version, nx, ny, iproj - real (kind=real32) :: xfcst, xlvl, startlat, startlon, starti, startj, & - deltalat, deltalon, dx, dy, xlonc, & - truelat1, truelat2, earth_radius - real (kind=real32), pointer, dimension(:,:) :: slab - logical :: is_wind_grid_rel - character (len=9) :: field - character (len=24) :: hdate - character (len=25) :: units - character (len=32) :: map_source - character (len=46) :: desc + integer :: version = 5, & ! Format version (must =5 for WPS format) + nx = 0, & ! First (x) dimension of 2-d array 'slab' + ny = 0, & ! Second (y) dimension of 2-d array 'slab' + iproj = PROJ_LATLON ! Code for projection of data in array + + real (kind=real32) :: xfcst = 0.0_real32, & ! Forecast hour of data + xlvl = 0.0_real32, & ! Vertical level of data in 2-d array 'slab' + startlat = 0.0_real32, & ! Latitude of starting point (degrees) + startlon = 0.0_real32, & ! Longitude of starting point (degrees) + starti = 1.0_real32, & ! Starting x-/i-index + startj = 1.0_real32, & ! Starting y-/j-index + deltalat = 0.0_real32, & ! Grid spacing (degrees) in meridional direction + deltalon = 0.0_real32, & ! Grid spacing (degrees) in zonal direction + dx = 0.0_real32, & ! Grid spacing (km) in x-direction + dy = 0.0_real32, & ! Grid spacing (km) in y-direction + xlonc = 0.0_real32, & ! Standard longitude of projection + truelat1 = 0.0_real32, & ! First true latitude of projection + truelat2 = 0.0_real32, & ! Second true latitude of projection + earth_radius = EARTH_RADIUS_M / 1000.0_real32 ! Earth radius (km) + + real (kind=real32), pointer, dimension(:,:) :: slab => null() ! 2-d array of data + + logical :: is_wind_grid_rel = .false. ! Flag indicating whether winds are + ! relative to source grid (.true.) or + ! relative to earth (.false.) + + character (len=9) :: field = '' ! Name of the field + character (len=24) :: hdate = '' ! Valid date for data YYYY:MM:DD_HH:mm:ss + character (len=25) :: units = '' ! Units of data + character (len=32) :: map_source = '' ! Source model / originating center + character (len=46) :: desc = '' ! Short description of data end type met_data From 4ab5b571a0b2669203fbc0ba86fb2ee4bcef02d6 Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Tue, 26 May 2026 22:26:40 +0000 Subject: [PATCH 17/24] Fix bug in registry-generated code to set up derived dimensions The previous commit (ae45a181) aimed to fix memory leaks that occurred while setting up derived dimensions by deallocating dimensions after they were allocated and added to the dimension pool (since the pool assigns the value of the dimension to its own internal memory, it is up to calling code to free any memory allocated for the dimension argument to mpas_pool_add_dimension). However, any later derived dimensions that depended on a dimension whose value came from a namelist or other registry definiton would generate a segmentation fault after referencing an unassociated pointer to the now deallocated dimension. For example, in the init_atmosphere core, the 'nVertLevels' dimension is set based on the namelist option 'config_nvertlevels', and the 'nVertLevelsP1' dimension is subsequently set to 'nVertLevels+1'. This commit fixes this issue by re-setting pointers to point to pool-owned memory after a dimension has been allocated, added to the dimension pool, and freed. --- src/tools/registry/gen_inc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tools/registry/gen_inc.c b/src/tools/registry/gen_inc.c index 078a4d399a..87fb0d7b41 100644 --- a/src/tools/registry/gen_inc.c +++ b/src/tools/registry/gen_inc.c @@ -1087,6 +1087,8 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " deallocate(%s)\n", dimname); + fortprintf(fd, " ! Now let %s point to pool memory so the dimension can be referenced later in this routine as needed\n", dimname); + fortprintf(fd, " call mpas_pool_get_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); // Namelist defined dimension if(strncmp(dimdef, "namelist:", 9) == 0){ @@ -1103,6 +1105,8 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, " %s = MPAS_MISSING_DIM\n", dimname); fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " deallocate(%s)\n", dimname); + fortprintf(fd, " ! Now let %s point to pool memory so the dimension can be referenced later in this routine as needed\n", dimname); + fortprintf(fd, " call mpas_pool_get_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " end if\n\n"); } } From 4069eb3dc5beb32741f3bc6e37c47a4d065b067c Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Tue, 26 May 2026 22:29:07 +0000 Subject: [PATCH 18/24] Clean up indentation in registry-generated code in block_dimension_routines.inc The parse_dimensions_from_registry C code in the registry is responsible for generating the block_dimension_routines.inc file, and this commit makes changes in parse_dimensions_from_registry to clean up whitespace / indentation issues in the generated Fortran code. The changes in this commit otherwise have no impact on results. --- src/tools/registry/gen_inc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tools/registry/gen_inc.c b/src/tools/registry/gen_inc.c index 87fb0d7b41..ade4b85783 100644 --- a/src/tools/registry/gen_inc.c +++ b/src/tools/registry/gen_inc.c @@ -1063,7 +1063,7 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, "\n"); - fortprintf(fd, "call mpas_log_write('Assigning remaining dimensions from definitions in Registry.xml ...')\n"); + fortprintf(fd, " call mpas_log_write('Assigning remaining dimensions from definitions in Registry.xml ...')\n"); for (dims_xml = ezxml_child(registry, "dims"); dims_xml; dims_xml = dims_xml->next) { for (dim_xml = ezxml_child(dims_xml, "dim"); dim_xml; dim_xml = dim_xml->next) { @@ -1079,17 +1079,17 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ if(strncmp(dimdef, "namelist:", 9) == 0){ snprintf(option_name, 1024, "%s", (dimdef)+9); fortprintf(fd, " %s = %s\n", dimname, option_name); - fortprintf(fd, "call mpas_log_write(' %s = $i (%s)', intArgs=(/%s/))\n", dimname, option_name, option_name); + fortprintf(fd, " call mpas_log_write(' %s = $i (%s)', intArgs=(/%s/))\n", dimname, option_name, option_name); } else { fortprintf(fd, " %s = %s\n", dimname, dimdef); - fortprintf(fd, "call mpas_log_write(' %s = $i', intArgs=(/%s/))\n", dimname, dimdef); + fortprintf(fd, " call mpas_log_write(' %s = $i', intArgs=(/%s/))\n", dimname, dimdef); } fortprintf(fd, " call mpas_pool_add_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); fortprintf(fd, " deallocate(%s)\n", dimname); fortprintf(fd, " ! Now let %s point to pool memory so the dimension can be referenced later in this routine as needed\n", dimname); fortprintf(fd, " call mpas_pool_get_dimension(dimensionPool, '%s', %s)\n", dimname, dimname); - fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); + fortprintf(fd, " else if ( %s == MPAS_MISSING_DIM ) then\n", dimname, dimname); // Namelist defined dimension if(strncmp(dimdef, "namelist:", 9) == 0){ snprintf(option_name, 1024, "%s", (dimdef)+9); @@ -1098,7 +1098,7 @@ int parse_dimensions_from_registry(ezxml_t registry)/*{{{*/ fortprintf(fd, " %s = %s\n", dimname, dimdef); } - fortprintf(fd, " end if\n\n"); + fortprintf(fd, " end if\n\n"); } else { fortprintf(fd, " if ( .not. associated(%s) ) then\n", dimname); fortprintf(fd, " allocate(%s)\n", dimname); From c03f2af25106e0a5c2463ef1df08e49b86689c96 Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Fri, 29 May 2026 13:50:29 -0600 Subject: [PATCH 19/24] Add a dm_info object to the argument list of core mpas_setup_packages_function Adding a dm_info object allows a core's package setup function to make use of MPI when setting up packages. For example, a core may read data from a file only on MPI rank 0 and broadcast the resulting values to other MPI ranks rather than having each MPI rank independently open and read the file. In addition to updating the mpas_setup_packages_function abstract interface in mpas_core_types.inc with an additional 'dminfo' argument, this commit updates the call to domain_ptr % core % setup_packages in mpas_init to include the dminfo argument, and it also updates the *_setup_packages functions in each core's interface module. --- src/core_atmosphere/mpas_atm_core_interface.F | 3 ++- src/core_init_atmosphere/mpas_init_atm_core_interface.F | 3 ++- src/core_landice/mode_forward/mpas_li_core_interface.F | 3 ++- src/core_ocean/driver/mpas_ocn_core_interface.F | 3 ++- src/core_seaice/model_forward/mpas_seaice_core_interface.F | 3 ++- src/core_sw/mpas_sw_core_interface.F | 3 ++- src/core_test/mpas_test_core_interface.F | 3 ++- src/driver/mpas_subdriver.F | 2 +- src/framework/mpas_core_types.inc | 4 +++- 9 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/core_atmosphere/mpas_atm_core_interface.F b/src/core_atmosphere/mpas_atm_core_interface.F index 21c651905e..7c7dadb204 100644 --- a/src/core_atmosphere/mpas_atm_core_interface.F +++ b/src/core_atmosphere/mpas_atm_core_interface.F @@ -99,7 +99,7 @@ end subroutine atm_setup_domain !> not allocated until after this routine has been called. ! !----------------------------------------------------------------------- - function atm_setup_packages(configs, streamInfo, packages, iocontext) result(ierr) + function atm_setup_packages(configs, streamInfo, packages, iocontext, dminfo) result(ierr) use mpas_dmpar use mpas_derived_types, only : mpas_pool_type, mpas_io_context_type, MPAS_streamInfo_type @@ -116,6 +116,7 @@ function atm_setup_packages(configs, streamInfo, packages, iocontext) result(ier type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr logical, pointer :: iauActive diff --git a/src/core_init_atmosphere/mpas_init_atm_core_interface.F b/src/core_init_atmosphere/mpas_init_atm_core_interface.F index f277a4a72f..3d70c3ba5b 100644 --- a/src/core_init_atmosphere/mpas_init_atm_core_interface.F +++ b/src/core_init_atmosphere/mpas_init_atm_core_interface.F @@ -100,7 +100,7 @@ end subroutine init_atm_setup_domain !> not allocated until after this routine has been called. ! !----------------------------------------------------------------------- - function init_atm_setup_packages(configs, streamInfo, packages, iocontext) result(ierr) + function init_atm_setup_packages(configs, streamInfo, packages, iocontext, dminfo) result(ierr) use mpas_derived_types, only : mpas_pool_type, mpas_io_context_type, MPAS_streamInfo_type use mpas_pool_routines, only : mpas_pool_get_config, mpas_pool_get_package @@ -111,6 +111,7 @@ function init_atm_setup_packages(configs, streamInfo, packages, iocontext) resul type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr logical :: lexist diff --git a/src/core_landice/mode_forward/mpas_li_core_interface.F b/src/core_landice/mode_forward/mpas_li_core_interface.F index e003bceb21..a091c4ea07 100644 --- a/src/core_landice/mode_forward/mpas_li_core_interface.F +++ b/src/core_landice/mode_forward/mpas_li_core_interface.F @@ -90,13 +90,14 @@ end subroutine li_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function li_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr) + function li_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr) implicit none type (mpas_pool_type), intent(inout) :: configPool type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ! Local variables diff --git a/src/core_ocean/driver/mpas_ocn_core_interface.F b/src/core_ocean/driver/mpas_ocn_core_interface.F index 28609948d3..593b108c3a 100644 --- a/src/core_ocean/driver/mpas_ocn_core_interface.F +++ b/src/core_ocean/driver/mpas_ocn_core_interface.F @@ -95,7 +95,7 @@ end subroutine ocn_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use ocn_analysis_driver @@ -103,6 +103,7 @@ function ocn_setup_packages(configPool, streamInfo, packagePool, iocontext) resu type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr diff --git a/src/core_seaice/model_forward/mpas_seaice_core_interface.F b/src/core_seaice/model_forward/mpas_seaice_core_interface.F index 82c8c0f2b0..90f2a22478 100644 --- a/src/core_seaice/model_forward/mpas_seaice_core_interface.F +++ b/src/core_seaice/model_forward/mpas_seaice_core_interface.F @@ -88,7 +88,7 @@ end subroutine seaice_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -98,6 +98,7 @@ function seaice_setup_packages(configPool, streamInfo, packagePool, iocontext) r type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr diff --git a/src/core_sw/mpas_sw_core_interface.F b/src/core_sw/mpas_sw_core_interface.F index 04df23f19d..608576c09f 100644 --- a/src/core_sw/mpas_sw_core_interface.F +++ b/src/core_sw/mpas_sw_core_interface.F @@ -89,7 +89,7 @@ end subroutine sw_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function sw_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function sw_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -99,6 +99,7 @@ function sw_setup_packages(configPool, streamInfo, packagePool, iocontext) resul type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ierr = 0 diff --git a/src/core_test/mpas_test_core_interface.F b/src/core_test/mpas_test_core_interface.F index e600824bc4..cbbee6040a 100644 --- a/src/core_test/mpas_test_core_interface.F +++ b/src/core_test/mpas_test_core_interface.F @@ -89,7 +89,7 @@ end subroutine test_setup_domain!}}} !> *not* allocated until after this routine is called. ! !----------------------------------------------------------------------- - function test_setup_packages(configPool, streamInfo, packagePool, iocontext) result(ierr)!{{{ + function test_setup_packages(configPool, streamInfo, packagePool, iocontext, dminfo) result(ierr)!{{{ use mpas_derived_types @@ -99,6 +99,7 @@ function test_setup_packages(configPool, streamInfo, packagePool, iocontext) res type (MPAS_streamInfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packagePool type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: ierr ierr = 0 diff --git a/src/driver/mpas_subdriver.F b/src/driver/mpas_subdriver.F index 68adea9dc3..80be1a4e8f 100644 --- a/src/driver/mpas_subdriver.F +++ b/src/driver/mpas_subdriver.F @@ -275,7 +275,7 @@ end subroutine xml_stream_get_attributes end if ierr = domain_ptr % core % setup_packages(domain_ptr % configs, domain_ptr % streamInfo, domain_ptr % packages, & - domain_ptr % iocontext) + domain_ptr % iocontext, domain_ptr % dminfo) if ( ierr /= 0 ) then call mpas_log_write('Package setup failed for core '//trim(domain_ptr % core % coreName), messageType=MPAS_LOG_CRIT) end if diff --git a/src/framework/mpas_core_types.inc b/src/framework/mpas_core_types.inc index 15a9866ccd..3789b8eb53 100644 --- a/src/framework/mpas_core_types.inc +++ b/src/framework/mpas_core_types.inc @@ -21,15 +21,17 @@ end interface abstract interface - function mpas_setup_packages_function(configs, streamInfo, packages, iocontext) result(iErr) + function mpas_setup_packages_function(configs, streamInfo, packages, iocontext, dminfo) result(iErr) import mpas_pool_type import mpas_io_context_type import mpas_streaminfo_type + import dm_info type (mpas_pool_type), intent(inout) :: configs type (mpas_streaminfo_type), intent(inout) :: streamInfo type (mpas_pool_type), intent(inout) :: packages type (mpas_io_context_type), intent(inout) :: iocontext + type (dm_info), intent(in) :: dminfo integer :: iErr end function mpas_setup_packages_function end interface From 3bc08ca99349d8ff810ee8e0272ad06d3f7c398e Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Fri, 29 May 2026 13:53:03 -0600 Subject: [PATCH 20/24] Add new routine mpas_split_string_new to the init_atm_core_interface module This commit adds a new private subroutine, mpas_split_string_new, to the init_atm_core_interface module. The purpose of this subroutine is essentially the same as the mpas_split_string routine in the mpas_string_utils module, but the maximum length of any substring does not need to be assumed in mpas_split_string_new, whereas the mpas_split_string subroutine requires the caller to declare a pointer to an array of strings of a specific length. Eventually, this routine could be migrated to mpas_string_utils and used as a replacement for mpas_split_string. --- .../mpas_init_atm_core_interface.F | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/core_init_atmosphere/mpas_init_atm_core_interface.F b/src/core_init_atmosphere/mpas_init_atm_core_interface.F index 8af69bc786..0d1da690b5 100644 --- a/src/core_init_atmosphere/mpas_init_atm_core_interface.F +++ b/src/core_init_atmosphere/mpas_init_atm_core_interface.F @@ -14,6 +14,8 @@ module init_atm_core_interface use mpas_io_units use mpas_log, only : mpas_log_write + private :: mpas_split_string_new + contains @@ -523,6 +525,82 @@ function init_atm_setup_block(block) result(ierr) end function init_atm_setup_block + !----------------------------------------------------------------------- + ! routine mpas_split_string_new + ! + !> \brief Splits a string at a specified delimiter, returning an array of strings + !> \author Michael Duda + !> \date 28 May 2026 + !> \details + !> This routine takes as input a string and a delimiter character, and returns an + !> array of sub-strings from the input string that are separated by one or more + !> delimiter characters. + !> + !> If more than one delimiter character appears consecutively, no empty string + !> in the output subStrings array is generated. + !> + !> The length of the strings in the output subStrings argument is equal to the + !> length of the longest substring in the input string. + ! + !----------------------------------------------------------------------- + subroutine mpas_split_string_new(string, delimiter, subStrings) + + implicit none + + ! Arguments + character(len=*), intent(in) :: string + character, intent(in) :: delimiter + character(len=:), dimension(:), allocatable, intent(inout) :: subStrings + + ! Local variables + integer :: i, j, n_strs, strlen, max_strlen + + + i = 1 + n_strs = 0 + strlen = 0 + max_strlen = 1 + PARSE_LOOP: do while (i <= len(string)) + do while (string(i:i) == delimiter) + i = i + 1 + if (i > len(string)) exit PARSE_LOOP + end do + + n_strs = n_strs + 1 + strlen = 0 + do while (string(i:i) /= delimiter) + i = i + 1 + strlen = strlen + 1 + if (i > len(string)) exit + end do + max_strlen = max(strlen, max_strlen) + end do PARSE_LOOP + + if (allocated(subStrings)) deallocate(subStrings) + allocate(character(len=max_strlen) :: subStrings(n_strs)) + + i = 1 + n_strs = 0 + COPY_LOOP: do while (i <= len(string)) + do while (string(i:i) == delimiter) + i = i + 1 + if (i > len(string)) exit COPY_LOOP + end do + + n_strs = n_strs + 1 + j = 1 + do while (string(i:i) /= delimiter) + subStrings(n_strs)(j:j) = string(i:i) + i = i + 1 + j = j + 1 + if (i > len(string)) exit + end do + subStrings(n_strs)(j:max_strlen) = '' + end do COPY_LOOP + + end subroutine mpas_split_string_new + + #include "setup_immutable_streams.inc" #include "block_dimension_routines.inc" From f3b116c47ce395feca42665fd56eb54b770db07e Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Fri, 29 May 2026 14:36:47 -0600 Subject: [PATCH 21/24] Apply packages to individual hydrometeors in the init_atmosphere core This commit defines new packages for hydrometeors, one package per hydrometeor, in the init_atmosphere core, and it provides two means of activating these packages. 1) If the new namelist option config_active_hydrometeors is set to 'detect_automatically', packages will be set up by scanning through the fields in the initial intermediate file, with packages being activated only for those hydrometeors that are present in the file. 2) Alternatively, and to avoid the need to scan through all fields in the initial intermediate file, config_active_hydrometeors may be set to a semicolon-separated list of zero or more hydrometeor names (e.g., 'qc;qr'). Packages for those specified hydrometeors will be active, and the packages for all unspecified hydrometeors will be inactive. A new subroutine, setup_hydrometeor_packages, handles the details of the hydrometeor package setup, and this routine is called from init_atm_setup_packages only if config_init_case is 7 or 9, and if met_stage_out is true. --- src/core_init_atmosphere/Registry.xml | 19 +- .../mpas_init_atm_core_interface.F | 196 +++++++++++++++++- 2 files changed, 210 insertions(+), 5 deletions(-) diff --git a/src/core_init_atmosphere/Registry.xml b/src/core_init_atmosphere/Registry.xml index 8834c402a9..024b2136f9 100644 --- a/src/core_init_atmosphere/Registry.xml +++ b/src/core_init_atmosphere/Registry.xml @@ -217,6 +217,11 @@ description="Whether to use specific-humidity as the first-guess moisture variable. If this option is False, relative humidity will be used." possible_values="true or false"/> + + @@ -387,6 +392,8 @@ + + @@ -1159,10 +1166,12 @@ description="Water vapor mixing ratio"/> + description="Cloud water mixing ratio" + packages="qc"/> + description="Rain water mixing ratio" + packages="qr"/> + description="Cloud water mixing ratio on lateral boundary cells" + packages="qc"/> + description="Rain water mixing ratio on lateral boundary cells" + packages="qr"/> \brief Set up packages for hydrometeors based on contents of intermediate file + !> \author Michael Duda + !> \date 27 May 2026 + !> \details + !> This routine is responsible for setting up packages for hydrometeor species (qc, + !> qr, etc.) based on the value of the 'config_active_hydrometeors' namelist option, + !> and possibly on the contents of the initial intermediate file (i.e., the + !> intermediate file valid at the start time of a simulation; the assumption is + !> that all intermediate files contain the same set of fields). + !> + !> 1) If config_active_hydrometeors is set to 'detect_automatically', packages will + !> be set up by scanning through the fields in the initial intermediate file, + !> with packages being activated only for those hydrometeors that are present in + !> the file. + !> + !> 2) Alternatively, and to avoid the need to scan through all fields in the + !> initial intermediate file, if config_active_hydrometeors is set to a + !> semicolon-separated list of zero or more hydrometeor names (e.g., 'qc;qr'), + !> packages for those specified hydrometeors will be active, and the packages + !> for all unspecified hydrometeors will be inactive. + !> + !> If the setup of all hydrometeor packages is successful, a value of 0 is returned + !> in the ierr output argument; otherwise, a non-zero value is returned. + ! + !----------------------------------------------------------------------- + subroutine setup_hydrometeor_packages(packages, configs, dminfo, ierr) + + use init_atm_read_met, only : read_met_init, read_next_met_field, read_met_close, met_data + use mpas_timer, only : mpas_timer_start, mpas_timer_stop + + implicit none + + type (mpas_pool_type), intent(inout) :: packages + type (mpas_pool_type), intent(in) :: configs + type (dm_info), intent(in) :: dminfo + integer, intent(out) :: ierr + + character(len=StrKIND), pointer :: config_active_hydrometeors + character(len=StrKIND), pointer :: config_met_prefix, config_start_time + integer :: istatus + type (met_data) :: field + + integer :: i + character(len=:), dimension(:), allocatable :: hydrometeors + + logical, pointer :: qc, qr + + + call mpas_timer_start('setup_hydrometeor_packages') + + ierr = 0 + + call mpas_pool_get_config(configs, 'config_active_hydrometeors', config_active_hydrometeors) + + if (.not. associated(config_active_hydrometeors)) then + call mpas_log_write( & + "The namelist option 'config_active_hydrometeors' could not be found when setting up hydrometeor packages.", & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_pool_get_package(packages, 'qcActive', qc) + call mpas_pool_get_package(packages, 'qrActive', qr) + + if (.not. associated(qc) & + .or. .not. associated(qr) & + ) then + + call mpas_log_write('One or more packages could not be found when setting up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + qc = .false. + qr = .false. + + if (trim(config_active_hydrometeors) == 'detect_automatically') then + + call mpas_pool_get_config(configs, 'config_met_prefix', config_met_prefix) + call mpas_pool_get_config(configs, 'config_start_time', config_start_time) + + if (.not. associated(config_met_prefix) & + .or. .not. associated(config_start_time) & + ) then + + call mpas_log_write( & + 'One or more namelist options could not be found when setting up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + + ierr = 1 + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_log_write('Setting up hydrometeor packages from '//trim(config_met_prefix)//':'//config_start_time(1:13)) + + if (dminfo % my_proc_id == IO_NODE) then + call read_met_init(trim(config_met_prefix), .false., config_start_time(1:13), ierr) + + if (ierr == 0) then + call read_next_met_field(field, istatus) + else + call mpas_log_write('Could not open intermediate file ' & + //trim(config_met_prefix)//':'//config_start_time(1:13) & + //' to set up hydrometeor packages.', & + messageType=MPAS_LOG_ERR) + istatus = 1 + end if + + do while (istatus == 0) + if (trim(field % field) == 'QC') then + qc = .true. + else if (trim(field % field) == 'QR') then + qr = .true. + end if + + deallocate(field % slab) + + ! If all packages are true, there is no point in scanning through the rest + ! of the intermediate file, since this loop over fields can only switch packages + ! from false to true. + if (qc .and. qr) exit + + call read_next_met_field(field, istatus) + end do + + call read_met_close() + + call mpas_dmpar_bcast_int(dminfo, ierr) + call mpas_dmpar_bcast_logical(dminfo, qc) + call mpas_dmpar_bcast_logical(dminfo, qr) + else + call mpas_dmpar_bcast_int(dminfo, ierr) + call mpas_dmpar_bcast_logical(dminfo, qc) + call mpas_dmpar_bcast_logical(dminfo, qr) + end if + + else + + call mpas_log_write('Setting up hydrometeor packages from config_active_hydrometeors list: ' & + //trim(config_active_hydrometeors)) + + call mpas_split_string_new(trim(config_active_hydrometeors), ';', hydrometeors) + + do i = 1, size(hydrometeors) + select case (trim(hydrometeors(i))) + case ('qc') + qc = .true. + case ('qr') + qr = .true. + case default + call mpas_log_write('Unrecognized hydrometeor '//trim(hydrometeors(i)) & + //' found in config_active_hydrometeors', & + messageType=MPAS_LOG_WARN) + end select + end do + + deallocate(hydrometeors) + + end if + + if (ierr /= 0) then + call mpas_log_write('Failed to set up hydrometeor packages.', messageType=MPAS_LOG_ERR) + call mpas_timer_stop('setup_hydrometeor_packages') + return + end if + + call mpas_log_write(' QC = $l', logicArgs=[qc]) + call mpas_log_write(' QR = $l', logicArgs=[qr]) + call mpas_log_write('----- done setting up hydrometeor packages -----') + call mpas_log_write('') + + call mpas_timer_stop('setup_hydrometeor_packages') + + end subroutine setup_hydrometeor_packages + + !*********************************************************************** ! ! function init_atm_setup_clock From 59b0ca7bafcb8528b4ef96235645d79d59a3cc5d Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Wed, 20 May 2026 12:00:15 -0600 Subject: [PATCH 22/24] Provide optimized code path in check_regex_match function for non-regex strings When the 'pattern' argument to the check_regex_match function in regex_matching.c is not a regular expression, a simple string comparison is sufficient and the cost of compiling a regular expression can be avoided. This commit adds an optimized code path in the check_regex_match function for this case. Thanks to Yu-Tze Hong (@aspenhong) and Ying-Jhang Wu (@yjwu890355) from the Central Weather Administration (CWA), Taipei, Taiwan, for identifying this opportunity for optimization and for providing an initial version of these code changes. --- src/framework/regex_matching.c | 43 ++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/framework/regex_matching.c b/src/framework/regex_matching.c index d37a23436d..ae62d09a5d 100644 --- a/src/framework/regex_matching.c +++ b/src/framework/regex_matching.c @@ -1,14 +1,54 @@ #include #include #include +#include #define MAX_LEN 1024 -void check_regex_match(const char * pattern, const char * str, int *imatch){ +static const char *BRE_SPECIAL_CHARS = ".[\\*^$"; + +/****************************************************************************** + * + * possible_bre + * + * Determine whether an input string may be a Basic Regular Expression (BRE). + * This function may return false positives (i.e., a return value indicating + * that a string is a BRE, when in fact the string is not a BRE), but it will + * never return a false negative (i.e., a return value indicating that a string + * is not a BRE, when in fact the string is a BRE). + * + * Inputs: + * s - a null-terminated string + * + * Return value: 1 if the input string 's' is possibly a BRE and 0 otherwise. + * + ******************************************************************************/ +static int possible_bre(const char *s) +{ + for (; *s != '\0'; s++) { + if (strchr(BRE_SPECIAL_CHARS, *s) != NULL) { + return 1; + } + } + return 0; +} + + +void check_regex_match(const char * pattern, const char * str, int *imatch) +{ regex_t regex; char bracketed_pattern[MAX_LEN]; int ierr, len; + /* + * If pattern is a simple string and not a basic regular expression, + * a string comparison will suffice + */ + if (!possible_bre(pattern)) { + *imatch = (strcmp(pattern, str) == 0) ? 1 : 0; + return; + } + *imatch = 0; len = snprintf(bracketed_pattern, 1024, "^%s$", pattern); if ( len >= MAX_LEN ) { @@ -34,4 +74,3 @@ void check_regex_match(const char * pattern, const char * str, int *imatch){ *imatch = -1; } } - From 2bc5d33e178c99af3ff6ec9dc7f8df3910a96ca6 Mon Sep 17 00:00:00 2001 From: Michael Duda Date: Mon, 8 Jun 2026 16:58:41 -0600 Subject: [PATCH 23/24] Add documentation for the check_regex_match function in regex_matching.c --- src/framework/regex_matching.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/framework/regex_matching.c b/src/framework/regex_matching.c index ae62d09a5d..17136507a7 100644 --- a/src/framework/regex_matching.c +++ b/src/framework/regex_matching.c @@ -34,6 +34,23 @@ static int possible_bre(const char *s) } +/****************************************************************************** + * + * check_regex_match + * + * Determine whether an input string 'str' matches the Basic Regular Expression + * (BRE) 'pattern'. + * + * Inputs: + * pattern - a null-terminated string that may contain any valid BRE, + * including a simple string + * str - a null-terminated string to be checked against pattern + * + * Return value: 1 if the input string str matches the BRE pattern, + * 0 if the input string does not match the BRE pattern, and + * -1 if an error occurred. + * + ******************************************************************************/ void check_regex_match(const char * pattern, const char * str, int *imatch) { regex_t regex; From eab77c9e6f7780599d1a504bc698e1be971b9bb6 Mon Sep 17 00:00:00 2001 From: cenamiller Date: Wed, 22 Jul 2026 12:35:03 -0600 Subject: [PATCH 24/24] Add CI files to develop branch, and make manual dispatch only --- .github/actions/build-mpas/action.yml | 141 +++++ .github/actions/download-testdata/action.yml | 97 +++ .github/actions/ect-summary/action.yml | 100 +++ .github/actions/mpas-version/action.yml | 47 ++ .github/actions/print-mpas-logs/action.yml | 64 ++ .github/actions/resolve-container/action.yml | 65 ++ .github/actions/run-mpas/action.yml | 195 ++++++ .github/actions/run-perturb-mpas/action.yml | 284 +++++++++ .../actions/run-perturb-mpas/perturb_theta.py | 81 +++ .../actions/run-perturb-mpas/trim_history.py | 112 ++++ .../actions/setup-nsight-systems/action.yml | 56 ++ .github/actions/validate-ect/action.yml | 210 +++++++ .github/ci-config.env | 138 ++++ .github/data/ect_excluded_vars.txt | 35 ++ .github/scripts/compare-bfb-nc.py | 63 ++ .github/scripts/install-nsight-systems-cli.sh | 120 ++++ .github/scripts/resolve-nsys.sh | 52 ++ .github/scripts/run-nsys-profile.sh | 79 +++ .github/workflows/_test-bfb.yml | 427 +++++++++++++ .github/workflows/_test-compiler.yml | 258 ++++++++ .github/workflows/_test-gpu.yml | 272 ++++++++ .github/workflows/bfb-decomp-gpu.yml | 20 + .github/workflows/bfb-decomp.yml | 19 + .github/workflows/bfb-io-gpu.yml | 20 + .github/workflows/bfb-io.yml | 19 + .github/workflows/bfb-nvhpc-cpu-vs-gpu.yml | 24 + .../workflows/compile-nvhpc-cuda-mpich.yml | 58 ++ .github/workflows/coverage.yml | 125 ++++ .github/workflows/ect-ensemble-gen.yml | 590 ++++++++++++++++++ .github/workflows/ect-test.yml | 237 +++++++ .github/workflows/profile-gpu-nsight.yml | 194 ++++++ .github/workflows/test-cross-repo.yml | 80 +++ .github/workflows/test-gcc-mpich.yml | 19 + .github/workflows/test-gcc-openmpi.yml | 15 + .github/workflows/test-gpu-mpich.yml | 15 + .github/workflows/test-gpu-openmpi.yml | 16 + .github/workflows/test-intel-mpich.yml | 19 + .github/workflows/test-intel-openmpi.yml | 15 + .github/workflows/test-nvhpc-mpich.yml | 19 + .github/workflows/test-nvhpc-openmpi.yml | 17 + .github/workflows/unit-tests.yml | 102 +++ tests/CMakeLists.txt | 40 ++ tests/unit/CMakeLists.txt | 13 + tests/unit/test_spline_interpolation.pf | 200 ++++++ 44 files changed, 4772 insertions(+) create mode 100644 .github/actions/build-mpas/action.yml create mode 100644 .github/actions/download-testdata/action.yml create mode 100644 .github/actions/ect-summary/action.yml create mode 100644 .github/actions/mpas-version/action.yml create mode 100644 .github/actions/print-mpas-logs/action.yml create mode 100644 .github/actions/resolve-container/action.yml create mode 100644 .github/actions/run-mpas/action.yml create mode 100644 .github/actions/run-perturb-mpas/action.yml create mode 100644 .github/actions/run-perturb-mpas/perturb_theta.py create mode 100644 .github/actions/run-perturb-mpas/trim_history.py create mode 100644 .github/actions/setup-nsight-systems/action.yml create mode 100644 .github/actions/validate-ect/action.yml create mode 100644 .github/ci-config.env create mode 100644 .github/data/ect_excluded_vars.txt create mode 100644 .github/scripts/compare-bfb-nc.py create mode 100644 .github/scripts/install-nsight-systems-cli.sh create mode 100644 .github/scripts/resolve-nsys.sh create mode 100644 .github/scripts/run-nsys-profile.sh create mode 100644 .github/workflows/_test-bfb.yml create mode 100644 .github/workflows/_test-compiler.yml create mode 100644 .github/workflows/_test-gpu.yml create mode 100644 .github/workflows/bfb-decomp-gpu.yml create mode 100644 .github/workflows/bfb-decomp.yml create mode 100644 .github/workflows/bfb-io-gpu.yml create mode 100644 .github/workflows/bfb-io.yml create mode 100644 .github/workflows/bfb-nvhpc-cpu-vs-gpu.yml create mode 100644 .github/workflows/compile-nvhpc-cuda-mpich.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/ect-ensemble-gen.yml create mode 100644 .github/workflows/ect-test.yml create mode 100644 .github/workflows/profile-gpu-nsight.yml create mode 100644 .github/workflows/test-cross-repo.yml create mode 100644 .github/workflows/test-gcc-mpich.yml create mode 100644 .github/workflows/test-gcc-openmpi.yml create mode 100644 .github/workflows/test-gpu-mpich.yml create mode 100644 .github/workflows/test-gpu-openmpi.yml create mode 100644 .github/workflows/test-intel-mpich.yml create mode 100644 .github/workflows/test-intel-openmpi.yml create mode 100644 .github/workflows/test-nvhpc-mpich.yml create mode 100644 .github/workflows/test-nvhpc-openmpi.yml create mode 100644 .github/workflows/unit-tests.yml create mode 100644 tests/CMakeLists.txt create mode 100644 tests/unit/CMakeLists.txt create mode 100644 tests/unit/test_spline_interpolation.pf diff --git a/.github/actions/build-mpas/action.yml b/.github/actions/build-mpas/action.yml new file mode 100644 index 0000000000..0db8a88847 --- /dev/null +++ b/.github/actions/build-mpas/action.yml @@ -0,0 +1,141 @@ +name: 'Build MPAS' +description: 'Build MPAS-Atmosphere with specified compiler configuration' + +inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi, etc.)' + required: true + use-pio: + description: 'Use PIO library (true/false)' + required: false + default: 'false' + openacc: + description: 'Enable OpenACC (true/false)' + required: false + default: 'false' + precision: + description: 'Floating-point precision (single or double)' + required: false + default: 'single' + build-timeout: + description: 'Build timeout in minutes' + required: false + default: '20' + +outputs: + executable: + description: 'Path to built executable' + value: ${{ steps.build.outputs.executable }} + +runs: + using: 'composite' + steps: + - name: Build MPAS-A + id: build + shell: bash + run: | + # Source container environment if available + # This sets NETCDF, PNETCDF, PIO, and other library paths + if [ -f /container/config_env.sh ]; then + echo "Sourcing container environment from /container/config_env.sh" + source /container/config_env.sh + fi + + # MPAS Makefile uses `cpp` for Registry preprocessing. + # Some containers (e.g. openSUSE Leap) don't ship it separately. + if ! command -v cpp &>/dev/null; then + echo "cpp not found, installing..." + if command -v zypper &>/dev/null; then + zypper install -y --no-recommends cpp + elif command -v dnf &>/dev/null; then + dnf install -y cpp + elif command -v apt-get &>/dev/null; then + apt-get update && apt-get install -y cpp + fi + fi + + # Map compiler input to COMPILER_FAMILY if not already set + # The container should set this, but we provide a fallback + if [ -z "${COMPILER_FAMILY}" ]; then + case "${{ inputs.compiler }}" in + gcc|gfortran|gnu) export COMPILER_FAMILY="gcc" ;; + nvhpc|nvfortran) export COMPILER_FAMILY="nvhpc" ;; + oneapi|intel|ifx) export COMPILER_FAMILY="oneapi" ;; + llvm|flang|clang) export COMPILER_FAMILY="clang" ;; + *) export COMPILER_FAMILY="${{ inputs.compiler }}" ;; + esac + fi + + # Set up I/O configuration + if [ "${{ inputs.use-pio }}" = "true" ]; then + export PIO_ROOT=${PIO_ROOT:-/container/pio} + export USE_PIO2=true + else + unset PIO + export USE_PIO2=false + fi + + # Set up accelerator configuration + if [ "${{ inputs.openacc }}" = "true" ]; then + export OPENACC=true + fi + + echo "Build configuration:" + echo " Compiler input: ${{ inputs.compiler }}" + echo " COMPILER_FAMILY: ${COMPILER_FAMILY}" + echo " NETCDF: ${NETCDF:-not set}" + echo " PNETCDF: ${PNETCDF:-not set}" + echo " USE_PIO2: ${USE_PIO2}" + echo " OPENACC: ${OPENACC:-false}" + echo " PRECISION: ${{ inputs.precision }}" + + # Settings from .github/ci-config.env — edit that file to change targets/workarounds + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + + # Map COMPILER_FAMILY to make target via ci-config.env lookup + VARNAME="MAKE_TARGET_${COMPILER_FAMILY}" + MAKE_TARGET="${!VARNAME}" + if [ -z "${MAKE_TARGET}" ]; then + echo "::error::No make target for COMPILER_FAMILY=${COMPILER_FAMILY}. Add MAKE_TARGET_${COMPILER_FAMILY}= to .github/ci-config.env" + exit 1 + fi + + EXTRA_MAKE_FLAGS="" + if [ "${COMPILER_FAMILY}" = "nvhpc" ]; then + ARCH="${NVHPC_TARGET_ARCH:--tp=px}" + sed -i "/^nvhpc:/,/^[a-z]/ s/\"FFLAGS_OPT = /\"FFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"CFLAGS_OPT = /\"CFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"CXXFLAGS_OPT = /\"CXXFLAGS_OPT = ${ARCH} /" Makefile + sed -i "/^nvhpc:/,/^[a-z]/ s/\"LDFLAGS_OPT = /\"LDFLAGS_OPT = ${ARCH} /" Makefile + EXTRA_MAKE_FLAGS="${NVHPC_EXTRA_MAKE_FLAGS}" + elif [ "${COMPILER_FAMILY}" = "oneapi" ]; then + EXTRA_MAKE_FLAGS="${ONEAPI_EXTRA_MAKE_FLAGS}" + fi + + echo " Make target: ${MAKE_TARGET}" + echo " Extra make flags: ${EXTRA_MAKE_FLAGS:-}" + echo " Parallel jobs: ${MAKE_J_PROCS:-$(nproc)}" + + PRECISION_FLAG="" + if [ "${{ inputs.precision }}" = "double" ]; then + PRECISION_FLAG="PRECISION=double" + fi + + timeout ${{ inputs.build-timeout }}m \ + make ${MAKE_TARGET} CORE=atmosphere ${EXTRA_MAKE_FLAGS} ${PRECISION_FLAG} --jobs ${MAKE_J_PROCS:-$(nproc)} + + # Set output + echo "executable=$(pwd)/atmosphere_model" >> $GITHUB_OUTPUT + + - name: Verify executable + shell: bash + run: | + if [ ! -f atmosphere_model ]; then + echo "ERROR: atmosphere_model not found!" + exit 1 + fi + ls -la atmosphere_model + file atmosphere_model diff --git a/.github/actions/download-testdata/action.yml b/.github/actions/download-testdata/action.yml new file mode 100644 index 0000000000..86d1457943 --- /dev/null +++ b/.github/actions/download-testdata/action.yml @@ -0,0 +1,97 @@ +name: 'Download Test Data' +description: 'Download and extract an MPAS test case archive from GitHub releases' + +inputs: + resolution: + description: 'Test case resolution (e.g., 240km, 120km). Used to look up RELEASE_TESTDATA_{RES} in ci-config.env.' + required: true + dest-dir: + description: 'Destination directory name for the extracted test case' + required: false + default: '' + +outputs: + case-dir: + description: 'Path to the extracted test case directory' + value: ${{ steps.extract.outputs.case-dir }} + +runs: + using: composite + steps: + - name: Resolve release tag and archive + id: resolve + shell: bash + run: | + RESOLUTION="${{ inputs.resolution }}" + ARCHIVE="${RESOLUTION}.tar.gz" + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found at ${CI_CONFIG}" + exit 1 + fi + source "${CI_CONFIG}" + + RES_UPPER=$(echo "${RESOLUTION}" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + TAG_VAR="RELEASE_TESTDATA_${RES_UPPER}" + TAG="${!TAG_VAR}" + + if [ -z "${TAG}" ]; then + echo "::error::No release tag for resolution '${RESOLUTION}'. Add ${TAG_VAR}= to ci-config.env." + exit 1 + fi + + REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + URL="https://github.com/${REPO}/releases/download/${TAG}/${ARCHIVE}" + + echo "release-tag=${TAG}" >> $GITHUB_OUTPUT + echo "archive=${ARCHIVE}" >> $GITHUB_OUTPUT + echo "url=${URL}" >> $GITHUB_OUTPUT + echo "Resolved: ${TAG_VAR}=${TAG} → ${URL}" + + - name: Cache test case archive + id: cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ${{ steps.resolve.outputs.archive }} + key: testdata-${{ steps.resolve.outputs.release-tag }} + + - name: Download test case archive + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + URL="${{ steps.resolve.outputs.url }}" + ARCHIVE="${{ steps.resolve.outputs.archive }}" + echo "Downloading ${ARCHIVE} from ${URL}..." + curl -fsSL --retry 5 --retry-delay 5 "${URL}" -o "${ARCHIVE}" + echo "Downloaded $(du -h "${ARCHIVE}" | cut -f1)" + + - name: Extract test case + id: extract + shell: bash + run: | + ARCHIVE="${{ steps.resolve.outputs.archive }}" + DEST="${{ inputs.dest-dir }}" + + echo "Cache hit: ${{ steps.cache.outputs.cache-hit }}" + echo "Archive: ${ARCHIVE} ($(du -h "${ARCHIVE}" | cut -f1))" + + tar xzf "${ARCHIVE}" + + CASE_DIR=$(tar tzf "${ARCHIVE}" 2>/dev/null | head -1 | cut -d/ -f1 || true) + if [ -z "${CASE_DIR}" ]; then + CASE_DIR=$(ls -td */ 2>/dev/null | head -1 | tr -d '/') + fi + + if [ -z "${CASE_DIR}" ] || [ ! -d "${CASE_DIR}" ]; then + echo "::error::Failed to extract test case from ${ARCHIVE}" + exit 1 + fi + + if [ -n "${DEST}" ] && [ "${DEST}" != "${CASE_DIR}" ]; then + mv "${CASE_DIR}" "${DEST}" + CASE_DIR="${DEST}" + fi + + echo "case-dir=${CASE_DIR}" >> $GITHUB_OUTPUT + echo "Extracted test case to: ${CASE_DIR}" diff --git a/.github/actions/ect-summary/action.yml b/.github/actions/ect-summary/action.yml new file mode 100644 index 0000000000..6175f5a33a --- /dev/null +++ b/.github/actions/ect-summary/action.yml @@ -0,0 +1,100 @@ +name: 'ECT Summary' +description: > + Generate a consolidated Ensemble Consistency Test results table from + enriched result files produced by the validate-ect action. Writes a + Markdown table to $GITHUB_STEP_SUMMARY with auto-discovered columns. + +inputs: + results-path: + description: 'Directory containing downloaded ect-result-* artifact subdirectories' + required: true + +runs: + using: 'composite' + steps: + - name: Generate summary table + shell: bash + run: | + RESULTS_PATH="${{ inputs.results-path }}" + + # Collect all result files + RESULT_FILES=() + for f in "${RESULTS_PATH}"/ect-result-*/ect-result.txt; do + [ -f "$f" ] && RESULT_FILES+=("$f") + done + + if [ ${#RESULT_FILES[@]} -eq 0 ]; then + echo "::warning::No ECT result files found in ${RESULTS_PATH}" + echo "## Ensemble Consistency Test (ECT) Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "No results available." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # Discover column names from the first result file (all keys except "result") + COLUMNS=() + while IFS='=' read -r key value; do + [ -z "$key" ] && continue + if [ "$key" != "result" ]; then + COLUMNS+=("$key") + fi + done < "${RESULT_FILES[0]}" + + # Build header row + HEADER="| " + SEPARATOR="| " + for col in "${COLUMNS[@]}"; do + COL_TITLE=$(echo "$col" | sed 's/.*/\u&/') + HEADER+="${COL_TITLE} | " + SEPARATOR+="--- | " + done + HEADER+="Result |" + SEPARATOR+="--- |" + + # Build data rows + PASS=0 FAIL=0 ERROR=0 SKIP=0 TOTAL=0 + ROWS="" + for f in "${RESULT_FILES[@]}"; do + TOTAL=$((TOTAL + 1)) + + # Parse key=value pairs + declare -A DATA=() + while IFS='=' read -r key value; do + [ -z "$key" ] && continue + DATA["$key"]="$value" + done < "$f" + + RESULT="${DATA[result]}" + case "$RESULT" in + PASSED) ICON="PASSED"; PASS=$((PASS + 1)) ;; + FAILED) ICON="**FAILED**"; FAIL=$((FAIL + 1)) ;; + SKIPPED) ICON="SKIPPED"; SKIP=$((SKIP + 1)) ;; + *) ICON="ERROR"; ERROR=$((ERROR + 1)) ;; + esac + + ROW="| " + for col in "${COLUMNS[@]}"; do + ROW+="${DATA[$col]:-—} | " + done + ROW+="${ICON} |" + ROWS+="${ROW}"$'\n' + + unset DATA + done + + SORTED_ROWS=$(echo "$ROWS" | sort) + + { + echo "## Ensemble Consistency Test (ECT) Results" + echo "" + echo "$HEADER" + echo "$SEPARATOR" + echo "$SORTED_ROWS" + echo "" + echo "**Total: ${TOTAL}** — ${PASS} passed, ${FAIL} failed, ${ERROR} error, ${SKIP} skipped" + + if [ ${FAIL} -gt 0 ] || [ ${ERROR} -gt 0 ]; then + echo "" + echo "> One or more ECT validations failed. Check individual ECT Validate job logs for PyCECT details." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/mpas-version/action.yml b/.github/actions/mpas-version/action.yml new file mode 100644 index 0000000000..73c3b3e47a --- /dev/null +++ b/.github/actions/mpas-version/action.yml @@ -0,0 +1,47 @@ +name: 'Get MPAS Version' +description: > + Read the MPAS version string from src/core_atmosphere/Registry.xml. + Strict — fails the workflow if the file is missing or the version + attribute cannot be parsed (no silent "unknown" fallback). + +inputs: + registry-path: + description: 'Path to Registry.xml relative to GITHUB_WORKSPACE' + required: false + default: 'src/core_atmosphere/Registry.xml' + +outputs: + version: + description: 'MPAS version string (e.g., 8.4.0)' + value: ${{ steps.extract.outputs.version }} + +runs: + using: 'composite' + steps: + - id: extract + shell: bash + run: | + python3 - "${{ inputs.registry-path }}" <<'PYEOF' + import os + import sys + import xml.etree.ElementTree as ET + + path = sys.argv[1] + if not os.path.isfile(path): + print(f"::error::MPAS Registry not found at {path}") + sys.exit(1) + try: + root = ET.parse(path).getroot() + except ET.ParseError as e: + print(f"::error::Could not parse {path}: {e}") + sys.exit(1) + + version = root.attrib.get('version') + if not version: + print(f"::error::No version attribute on in {path}") + sys.exit(1) + + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"version={version}\n") + print(f"MPAS version: {version}") + PYEOF diff --git a/.github/actions/print-mpas-logs/action.yml b/.github/actions/print-mpas-logs/action.yml new file mode 100644 index 0000000000..53ed7e33b1 --- /dev/null +++ b/.github/actions/print-mpas-logs/action.yml @@ -0,0 +1,64 @@ +name: 'Print MPAS Logs' +description: > + Print MPAS per-rank log files (log.atmosphere..out / .err) to the + workflow log inside collapsible ::group:: blocks. Read-only; never fails + on its own (use if: always() at the call site). + +inputs: + log-dir: + description: 'Directory to scan for log files' + required: true + pattern: + description: > + Glob (relative to log-dir) of files to print. Default catches both + .out and .err per rank. + required: false + default: 'log.atmosphere.*' + max-lines: + description: > + If set, print only the last N lines of each file via tail. Empty (default) + prints the whole file. + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Print logs + shell: bash + run: | + LOG_DIR="${{ inputs.log-dir }}" + PATTERN="${{ inputs.pattern }}" + MAX_LINES="${{ inputs.max-lines }}" + + if [ ! -d "${LOG_DIR}" ]; then + echo "print-mpas-logs: directory '${LOG_DIR}' does not exist; nothing to print." + exit 0 + fi + + cd "${LOG_DIR}" + shopt -s nullglob + + # Sort so .out comes before .err per rank (lexical order does this). + FILES=( $(ls -1 ${PATTERN} 2>/dev/null | sort) ) + + if [ ${#FILES[@]} -eq 0 ]; then + echo "print-mpas-logs: no files matching '${PATTERN}' in ${LOG_DIR}." + exit 0 + fi + + echo "print-mpas-logs: ${#FILES[@]} file(s) from ${LOG_DIR}" + # Close the implicit group GitHub opens for the run: script source so + # the per-file groups below render at the step's top level (collapsible + # at normal font size) instead of nested inside the script-source group. + echo "::endgroup::" + for f in "${FILES[@]}"; do + echo "::group::${f}" + if [ -n "${MAX_LINES}" ]; then + echo "(last ${MAX_LINES} lines)" + tail -n "${MAX_LINES}" "${f}" || true + else + cat "${f}" || true + fi + echo "::endgroup::" + done diff --git a/.github/actions/resolve-container/action.yml b/.github/actions/resolve-container/action.yml new file mode 100644 index 0000000000..fccafffd76 --- /dev/null +++ b/.github/actions/resolve-container/action.yml @@ -0,0 +1,65 @@ +name: 'Resolve Container Image' +description: 'Assemble a container image name from ci-config.env templates' + +inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi)' + required: true + mpi: + description: 'MPI implementation (openmpi, mpich)' + required: true + gpu: + description: 'GPU variant (empty string, or "cuda")' + required: false + default: '' + +outputs: + image: + description: 'Fully qualified container image name' + value: ${{ steps.resolve.outputs.image }} + +runs: + using: 'composite' + steps: + - name: Resolve container image + id: resolve + shell: bash + run: | + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found at ${CI_CONFIG}" + exit 1 + fi + source "${CI_CONFIG}" + + COMPILER="${{ inputs.compiler }}" + MPI="${{ inputs.mpi }}" + GPU="${{ inputs.gpu }}" + + # Resolve compiler name (fallback to raw value if no mapping) + comp_var="CONTAINER_COMPILER_${COMPILER}" + COMPILER_NAME="${!comp_var:-$COMPILER}" + + # Resolve MPI name (fallback to raw value if no mapping) + mpi_var="CONTAINER_MPI_${MPI}" + MPI_NAME="${!mpi_var:-$MPI}" + + # Pick template: GPU vs CPU, then check for per-compiler override + if [ -n "${GPU}" ]; then + gpu_override="CONTAINER_IMAGE_GPU_${COMPILER}" + TEMPLATE="${!gpu_override:-$CONTAINER_IMAGE_GPU}" + else + cpu_override="CONTAINER_IMAGE_${COMPILER}" + TEMPLATE="${!cpu_override:-$CONTAINER_IMAGE}" + fi + + # Substitute placeholders + IMAGE="${TEMPLATE//\{compiler\}/$COMPILER_NAME}" + IMAGE="${IMAGE//\{mpi\}/$MPI_NAME}" + + echo "Resolved container image: ${IMAGE}" + echo " Compiler: ${COMPILER} → ${COMPILER_NAME}" + echo " MPI: ${MPI} → ${MPI_NAME}" + echo " GPU: ${GPU:-none}" + + echo "image=${IMAGE}" >> $GITHUB_OUTPUT diff --git a/.github/actions/run-mpas/action.yml b/.github/actions/run-mpas/action.yml new file mode 100644 index 0000000000..063697a90b --- /dev/null +++ b/.github/actions/run-mpas/action.yml @@ -0,0 +1,195 @@ +name: 'Run MPAS' +description: 'Run MPAS-Atmosphere test case' + +inputs: + executable: + description: 'Path to atmosphere_model executable' + required: false + default: './atmosphere_model' + num-procs: + description: 'Number of MPI processes' + required: false + default: '1' + run-duration: + description: 'Run duration (format: D_HH:MM:SS). If empty, uses the namelist default from the test case archive.' + required: false + default: '' + restart-interval: + description: 'Restart output interval (format: D_HH:MM:SS). If empty, uses the streams.atmosphere default.' + required: false + default: '' + resolution: + description: 'Test case resolution name (e.g., 240km). Used to download the archive and name the working directory.' + required: false + default: '240km' + mpi-impl: + description: 'MPI implementation (openmpi, mpich)' + required: false + default: '' + run-timeout: + description: 'Run timeout in minutes' + required: false + default: '20' + working-dir: + description: 'Working directory name for the run' + required: false + default: '' + strict-exit-check: + description: 'Fail on non-zero exit code (default true). Set false when gfortran IEEE warnings produce non-zero exit but model output is valid.' + required: false + default: 'true' + +outputs: + log-dir: + description: 'Directory containing log files' + value: ${{ steps.run.outputs.log-dir }} + status: + description: 'Run status (success/failed)' + value: ${{ steps.run.outputs.status }} + +runs: + using: 'composite' + steps: + - name: Resolve configuration + id: config + shell: bash + run: | + RESOLUTION="${{ inputs.resolution }}" + WORKDIR="${{ inputs.working-dir }}" + if [ -z "${WORKDIR}" ]; then + WORKDIR="run-${RESOLUTION}" + fi + + echo "workdir=${WORKDIR}" >> $GITHUB_OUTPUT + echo "resolution=${RESOLUTION}" >> $GITHUB_OUTPUT + + echo "=== Run configuration ===" + echo " Resolution: ${RESOLUTION}" + echo " Work dir: ${WORKDIR}" + echo " Timeout: ${{ inputs.run-timeout }}m" + if [ -n "${{ inputs.run-duration }}" ]; then + echo " Duration: ${{ inputs.run-duration }} (override)" + else + echo " Duration: (namelist default)" + fi + + - name: Download test case + uses: ./.github/actions/download-testdata + id: download + with: + resolution: ${{ inputs.resolution }} + dest-dir: ${{ steps.config.outputs.workdir }} + + - name: Link executable + shell: bash + run: | + WORKDIR="${{ steps.config.outputs.workdir }}" + chmod +x ${{ inputs.executable }} + ln -sf $(realpath ${{ inputs.executable }}) "${WORKDIR}/atmosphere_model" + + - name: Configure namelist overrides + shell: bash + working-directory: ${{ steps.config.outputs.workdir }} + run: | + if [ -n "${{ inputs.run-duration }}" ]; then + DURATION="${{ inputs.run-duration }}" + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${DURATION}'/" namelist.atmosphere + echo "Overrode config_run_duration = '${DURATION}'" + fi + + if [ -n "${{ inputs.restart-interval }}" ]; then + RESTART="${{ inputs.restart-interval }}" + sed -i '// s/output_interval="[^"]*"/output_interval="'"${RESTART}"'"/' streams.atmosphere + echo "Overrode restart output_interval = '${RESTART}'" + fi + + echo "=== Namelist ===" + grep config_run_duration namelist.atmosphere + + - name: Run MPAS-A + id: run + shell: bash + working-directory: ${{ steps.config.outputs.workdir }} + run: | + TIMEOUT="${{ inputs.run-timeout }}" + + if [ -f /container/config_env.sh ]; then + source /container/config_env.sh + fi + + # Workaround: some containers omit LD_LIBRARY_PATH from config_env.sh + if [ -z "${LD_LIBRARY_PATH}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" + fi + + if [ -n "${{ inputs.mpi-impl }}" ]; then + export MPI_IMPL="${{ inputs.mpi-impl }}" + fi + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + + MPI_FLAGS="" + if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" + fi + + ulimit -s unlimited 2>/dev/null || echo "Warning: Could not set unlimited stack size" + + echo "=== Run configuration ===" + echo " Resolution: ${{ steps.config.outputs.resolution }}" + echo " Processors: ${{ inputs.num-procs }}" + echo " MPI_IMPL: ${MPI_IMPL:-auto}" + echo " MPI_FLAGS: ${MPI_FLAGS}" + echo " Stack limit: $(ulimit -s)" + echo " Available CPUs: $(nproc 2>/dev/null || echo unknown)" + echo " Available RAM: $(free -m 2>/dev/null | awk '/^Mem:/{print $2 "MB"}' || echo unknown)" + echo " LD_LIBRARY_PATH: ${LD_LIBRARY_PATH:-not set}" + + set +e + timeout ${TIMEOUT}m mpirun -n ${{ inputs.num-procs }} ${MPI_FLAGS} ./atmosphere_model + RUN_STATUS=$? + set -e + + echo "log-dir=$(pwd)" >> $GITHUB_OUTPUT + echo "run-exit-code=${RUN_STATUS}" >> $GITHUB_OUTPUT + if [ $RUN_STATUS -eq 0 ]; then + echo "status=success" >> $GITHUB_OUTPUT + elif [ "${{ inputs.strict-exit-check }}" = "false" ]; then + echo "status=success" >> $GITHUB_OUTPUT + echo "::warning::Model exited with status ${RUN_STATUS} (non-strict mode, treating as success)" + else + echo "status=failed" >> $GITHUB_OUTPUT + echo "::warning::Model run exited with status $RUN_STATUS" + fi + + - name: List output files + shell: bash + if: always() + run: | + WORKDIR="${{ steps.config.outputs.workdir }}" + echo "=== Output files ===" + if [ -d "${WORKDIR}" ]; then + cd "${WORKDIR}" + ls -la log.* 2>/dev/null || echo "No log files found" + ls -la *.nc 2>/dev/null || echo "No NetCDF files found" + else + echo "Working directory ${WORKDIR} does not exist" + fi + + - name: Print MPAS logs + if: always() + uses: ./.github/actions/print-mpas-logs + with: + log-dir: ${{ steps.config.outputs.workdir }} + + - name: Check run status + shell: bash + if: always() + run: | + if [ "${{ steps.run.outputs.status }}" = "failed" ]; then + echo "::error::MPAS model run failed with exit code ${{ steps.run.outputs.run-exit-code }}" + exit 1 + fi diff --git a/.github/actions/run-perturb-mpas/action.yml b/.github/actions/run-perturb-mpas/action.yml new file mode 100644 index 0000000000..9af1660708 --- /dev/null +++ b/.github/actions/run-perturb-mpas/action.yml @@ -0,0 +1,284 @@ +name: 'Run Perturbed MPAS' +description: > + Run one or more perturbed MPAS-A ensemble members for ECT. + Handles IC perturbation, namelist/stream configuration, model execution, + and optional history file trimming. Supports both single-member + (ect-test) and batched (ect-ensemble-gen) use cases. + +inputs: + base-dir: + description: 'Path to the base test case directory (copied per member)' + required: true + executable: + description: 'Path to atmosphere_model executable' + required: true + member-start: + description: 'First ensemble member index' + required: true + member-end: + description: 'Last ensemble member index (same as member-start for single member)' + required: true + run-duration: + description: 'Model run duration per member (format: D_HH:MM:SS)' + required: true + run-timeout: + description: 'Per-member timeout in minutes' + required: false + default: '45' + num-ranks: + description: 'Number of MPI ranks per member' + required: false + default: '1' + mpi-impl: + description: 'MPI implementation (openmpi, mpich)' + required: false + default: 'openmpi' + output-dir: + description: 'Directory to collect trimmed history files' + required: false + default: 'history-output' + trim: + description: 'Trim history files after run (true/false)' + required: false + default: 'true' + restart-file: + description: 'Path to a spun-up restart file. When set, each member starts from this restart (with config_do_restart=.true.) instead of init.nc.' + required: false + default: '' + verbose: + description: 'Print detailed diagnostics for each member (namelist dumps, theta stats)' + required: false + default: 'false' + +outputs: + status: + description: 'Overall status (success/partial/failed)' + value: ${{ steps.run-members.outputs.status }} + members-completed: + description: 'Number of members that produced history files' + value: ${{ steps.run-members.outputs.members-completed }} + +runs: + using: 'composite' + steps: + - name: Run perturbed members + id: run-members + shell: bash + run: | + source /container/config_env.sh + + if command -v conda &>/dev/null; then + eval "$(conda shell.bash hook)" 2>/dev/null + conda activate base 2>/dev/null || true + fi + + if ! python3 -c "import netCDF4, numpy" 2>/dev/null; then + if python3 -m pip install --quiet netCDF4 numpy 2>/dev/null; then + echo "Installed netCDF4/numpy via pip" + elif command -v conda &>/dev/null; then + conda install -y -q netCDF4 numpy + echo "Installed netCDF4/numpy via conda" + else + echo "::error::Could not install netCDF4 (no pip, no conda)" + exit 1 + fi + fi + echo "python3: $(which python3) — $(python3 --version)" + + if [ -z "${LD_LIBRARY_PATH}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" + fi + + CI_CONFIG="${GITHUB_WORKSPACE}/.github/ci-config.env" + if [ -f "${CI_CONFIG}" ]; then + source "${CI_CONFIG}" + fi + ulimit -s unlimited 2>/dev/null || true + + BASEDIR="${{ inputs.base-dir }}" + EXE="${{ inputs.executable }}" + MEMBER_START=${{ inputs.member-start }} + MEMBER_END=${{ inputs.member-end }} + NRANKS=${{ inputs.num-ranks }} + MPI_IMPL="${{ inputs.mpi-impl }}" + OUTDIR="${{ inputs.output-dir }}" + TRIM="${{ inputs.trim }}" + RESTART_FILE="${{ inputs.restart-file }}" + VERBOSE="${{ inputs.verbose }}" + RUN_DURATION="${{ inputs.run-duration }}" + RUN_TIMEOUT="${{ inputs.run-timeout }}" + + MPI_FLAGS="" + if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" + fi + + EXCLUDE_FILE="" + if [ -n "${ECT_EXCLUDED_VARS}" ]; then + EXCLUDE_FILE="${GITHUB_WORKSPACE}/${ECT_EXCLUDED_VARS}" + fi + + mkdir -p "${OUTDIR}" + + TOTAL=0 + COMPLETED=0 + + for MEMBER in $(seq ${MEMBER_START} ${MEMBER_END}); do + TOTAL=$((TOTAL + 1)) + MEMBER_ID=$(printf "%04d" ${MEMBER}) + RUNDIR="run-ect-${MEMBER_ID}" + echo "" + echo "==========================================" + echo " Ensemble member ${MEMBER_ID}" + echo "==========================================" + + cp -r "${BASEDIR}" "${RUNDIR}" + chmod +x "${EXE}" + ln -sf $(realpath "${EXE}") "${RUNDIR}/atmosphere_model" + + if [ -n "${RESTART_FILE}" ]; then + RESTART_TIME=$(python3 -c " + import netCDF4, sys + ds = netCDF4.Dataset(sys.argv[1]) + print(ds.variables['xtime'][0].tobytes().decode().strip()) + ds.close() + " "${RESTART_FILE}") + RESTART_FNAME="restart.$(echo ${RESTART_TIME} | tr ':' '.').nc" + cp "${RESTART_FILE}" "${RUNDIR}/${RESTART_FNAME}" + echo "${RESTART_TIME}" > "${RUNDIR}/restart_timestamp" + PERTURB_FILE="${RUNDIR}/${RESTART_FNAME}" + echo "[$(date +%H:%M:%S)] Restart mode: ${RESTART_FNAME} (time=${RESTART_TIME})" + else + PERTURB_FILE="${RUNDIR}/init.nc" + if [ ! -f "${PERTURB_FILE}" ]; then + PERTURB_FILE=$(ls ${RUNDIR}/*.init*.nc 2>/dev/null | head -1) + fi + fi + + if [ -z "${PERTURB_FILE}" ] || [ ! -f "${PERTURB_FILE}" ]; then + echo "::error::Member ${MEMBER_ID}: could not find file to perturb" + rm -rf "${RUNDIR}" + continue + fi + + echo "[$(date +%H:%M:%S)] Perturbing theta (seed=${MEMBER})..." + python3 ${GITHUB_WORKSPACE}/.github/actions/run-perturb-mpas/perturb_theta.py \ + "${PERTURB_FILE}" --seed ${MEMBER} --magnitude ${ECT_PERTURB_MAGNITUDE} + + if [ "${VERBOSE}" = "true" ]; then + echo " Restart MD5: $(md5sum "${PERTURB_FILE}" | cut -d' ' -f1)" + fi + + cd "${RUNDIR}" + + if [ -n "${RESTART_FILE}" ]; then + sed -i "s/config_do_restart.*/config_do_restart = .true./" namelist.atmosphere + sed -i "s/config_start_time.*/config_start_time = 'file'/" namelist.atmosphere + sed -i '/&restart/a\ config_do_DAcycling = .true.' namelist.atmosphere + fi + + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${RUN_DURATION}'/" namelist.atmosphere + sed -i '// s/output_interval="[^"]*"/output_interval="none"/' streams.atmosphere + sed -i '// s/output_interval="[^"]*"/output_interval="'"${RUN_DURATION}"'"/' streams.atmosphere + + echo "[$(date +%H:%M:%S)] Running MPAS-A (${RUN_DURATION}, ${NRANKS} ranks)..." + set +e + timeout ${RUN_TIMEOUT}m mpirun -n ${NRANKS} ${MPI_FLAGS} ./atmosphere_model + RUN_STATUS=$? + set -e + echo "[$(date +%H:%M:%S)] Model finished (exit code ${RUN_STATUS})" + + for LOGFILE in log.atmosphere.*.out log.atmosphere.*.err; do + [ -f "${LOGFILE}" ] || continue + EXT="${LOGFILE##*.}" + BASE="${LOGFILE%.*}" + cp "${LOGFILE}" "../${OUTDIR}/${BASE}.member${MEMBER_ID}.${EXT}" + done + + if [ "${VERBOSE}" = "true" ]; then + echo "[$(date +%H:%M:%S)] === Namelist config ===" + grep -E 'config_do_restart|config_start_time|config_run_duration' namelist.atmosphere || true + echo "[$(date +%H:%M:%S)] === Restart stream ===" + grep -A2 'immutable_stream name="restart"' streams.atmosphere || true + echo "[$(date +%H:%M:%S)] === Output stream ===" + grep -A2 'stream name="output"' streams.atmosphere || true + + RESTART_NC=$(ls restart.*.nc 2>/dev/null | head -1) + if [ -n "${RESTART_NC}" ]; then + echo "[$(date +%H:%M:%S)] === Restart theta check ===" + python3 -c " + import netCDF4 as nc, numpy as np, sys + ds = nc.Dataset(sys.argv[1]) + th = ds.variables['theta'][:] + print(f' File: {sys.argv[1]}') + print(f' theta dtype={th.dtype} shape={th.shape} mean={np.mean(th):.15e}') + ds.close() + " "${RESTART_NC}" + fi + fi + + echo "[$(date +%H:%M:%S)] History files produced:" + ls -la history.*.nc 2>/dev/null || echo " (none)" + HIST_FILE=$(ls -t history.*.nc 2>/dev/null | head -1 || true) + if [ -n "${HIST_FILE}" ]; then + if [ "${VERBOSE}" = "true" ]; then + echo "[$(date +%H:%M:%S)] === History theta check ===" + python3 -c " + import netCDF4 as nc, numpy as np, sys + ds = nc.Dataset(sys.argv[1]) + if 'theta' in ds.variables: + th = ds.variables['theta'] + data = th[:] + print(f' theta dtype={data.dtype} shape={data.shape} mean={np.mean(data):.15e} min={np.min(data):.10e} max={np.max(data):.10e}') + else: + print(' theta NOT in history variables') + print(f' Available: {list(ds.variables.keys())[:15]}...') + ds.close() + " "${HIST_FILE}" + fi + TSLICE=$(python3 -c " + import netCDF4, sys + ds = netCDF4.Dataset(sys.argv[1]) + print(ds.dimensions['Time'].size - 1) + ds.close() + " "${HIST_FILE}") + echo "[$(date +%H:%M:%S)] Using tslice=${TSLICE} (last time slice)" + if [ "${TRIM}" = "true" ] && [ -n "${EXCLUDE_FILE}" ] && [ -f "${EXCLUDE_FILE}" ]; then + python3 ${GITHUB_WORKSPACE}/.github/actions/run-perturb-mpas/trim_history.py \ + "${HIST_FILE}" "../${OUTDIR}/history.${MEMBER_ID}.nc" \ + --tslice ${TSLICE} \ + --exclude-file "${EXCLUDE_FILE}" + echo "[$(date +%H:%M:%S)] Saved trimmed history for member ${MEMBER_ID}" + else + cp "${HIST_FILE}" "../${OUTDIR}/history.${MEMBER_ID}.nc" + echo "[$(date +%H:%M:%S)] Saved history for member ${MEMBER_ID}" + fi + COMPLETED=$((COMPLETED + 1)) + else + echo "::warning::Member ${MEMBER_ID} failed (exit ${RUN_STATUS}, no history file)" + fi + + cd .. + rm -rf "${RUNDIR}" + done + + echo "" + echo "=== Completed ${COMPLETED}/${TOTAL} members ===" + ls -la "${OUTDIR}/" + + echo "members-completed=${COMPLETED}" >> $GITHUB_OUTPUT + if [ ${COMPLETED} -eq ${TOTAL} ]; then + echo "status=success" >> $GITHUB_OUTPUT + elif [ ${COMPLETED} -gt 0 ]; then + echo "status=partial" >> $GITHUB_OUTPUT + else + echo "status=failed" >> $GITHUB_OUTPUT + echo "::error::No ensemble members produced history files" + exit 1 + fi + + - name: Print MPAS logs + if: always() + uses: ./.github/actions/print-mpas-logs + with: + log-dir: ${{ inputs.output-dir }} diff --git a/.github/actions/run-perturb-mpas/perturb_theta.py b/.github/actions/run-perturb-mpas/perturb_theta.py new file mode 100644 index 0000000000..62ab4520f7 --- /dev/null +++ b/.github/actions/run-perturb-mpas/perturb_theta.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Apply O(10^-14) perturbation to the theta (potential temperature) field +in an MPAS initial conditions file to generate ensemble diversity. + +This is a placeholder until native perturbation support is added to MPAS-A. +Each ensemble member uses a unique seed for reproducible perturbations. +""" + +import argparse +import sys + +import numpy as np + +try: + from netCDF4 import Dataset +except ImportError: + print("ERROR: netCDF4 is required. Install with: pip install netCDF4") + sys.exit(1) + + +def perturb_theta(ic_file, seed, magnitude=1e-14): + rng = np.random.default_rng(seed) + + with Dataset(ic_file, "r+") as ds: + if "theta" not in ds.variables: + print(f"ERROR: 'theta' variable not found in {ic_file}") + print(f" Available variables: {list(ds.variables.keys())[:20]}...") + sys.exit(1) + + theta = ds.variables["theta"] + data = theta[:] + original_mean = float(np.mean(data)) + + perturbation = rng.uniform(-magnitude, magnitude, size=data.shape) + theta[:] = data * (1.0 + perturbation) + + actual_max = np.max(np.abs(perturbation)) + print(f"Applied perturbation to theta field:") + print(f" File: {ic_file}") + print(f" Format: {ds.data_model}") + print(f" Seed: {seed}") + print(f" Magnitude: +/- {magnitude:.0e}") + print(f" Max |eps|: {actual_max:.2e}") + print(f" Shape: {data.shape}") + print(f" Var dtype: {theta.dtype}") + print(f" Original mean: {original_mean:.15e}") + + # Read-back verification: reopen and confirm perturbation persisted + with Dataset(ic_file, "r") as ds: + verify = ds.variables["theta"][:] + verify_mean = float(np.mean(verify)) + diff = verify.astype(np.float64) - data.astype(np.float64) + n_changed = int(np.count_nonzero(diff)) + max_diff = float(np.max(np.abs(diff))) + print(f" Verify mean: {verify_mean:.15e}") + print(f" Changed cells: {n_changed}/{diff.size}") + print(f" Max |diff|: {max_diff:.6e}") + if n_changed == 0: + print(f"ERROR: Perturbation did NOT persist in file!") + print(f" On-disk dtype: {ds.variables['theta'].dtype}") + print(f" If dtype is float32, perturbations below ~1.2e-7 will be rounded away.") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Perturb MPAS theta field for ensemble generation" + ) + parser.add_argument("ic_file", help="Path to MPAS initial conditions NetCDF file") + parser.add_argument("--seed", type=int, required=True, + help="Random seed for reproducible perturbation") + parser.add_argument("--magnitude", type=float, default=1e-14, + help="Perturbation magnitude (default: 1e-14)") + args = parser.parse_args() + + perturb_theta(args.ic_file, args.seed, args.magnitude) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/run-perturb-mpas/trim_history.py b/.github/actions/run-perturb-mpas/trim_history.py new file mode 100644 index 0000000000..706b4268ef --- /dev/null +++ b/.github/actions/run-perturb-mpas/trim_history.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Trim MPAS history files for ECT ensemble processing. + +Extracts a single time slice, removes excluded variables, strips most +static mesh geometry, and applies lossless NetCDF4 deflation (zlib). + +PyCECT requires three area-weighting variables (areaCell, dvEdge, +areaTriangle) to compute global means. These static variables are +preserved even though they lack a Time dimension. + +The ECT exclusion list path is configured via ECT_EXCLUDED_VARS in ci-config.env +and passed via --exclude-file. + +Usage: + python3 trim_history.py input.nc output.nc --tslice -1 --exclude-file excluded_vars.txt +""" + +import argparse +import os +import sys + +import netCDF4 as nc + +# Static variables PyCECT reads for area-weighted global means +# (see pyEnsLib.py generate_global_mean_for_summary_MPAS, lines 745-758) +PYCECT_REQUIRED_STATIC = {'areaCell', 'dvEdge', 'areaTriangle'} + + +def trim_history(infile, outfile, tslice, exclude_vars=None): + exclude = set(exclude_vars or []) + + with nc.Dataset(infile, 'r') as src, nc.Dataset(outfile, 'w', format='NETCDF4') as dst: + ntime = src.dimensions['Time'].size + if tslice < 0: + tslice = ntime + tslice + if tslice < 0 or tslice >= ntime: + print(f"ERROR: tslice={tslice} out of range for {ntime} time slice(s) in {infile}") + sys.exit(1) + dst.setncatts({k: src.getncattr(k) for k in src.ncattrs()}) + + # Copy ALL dimensions — PyCECT checks nCells/nEdges/nVertices + for dname, dim in src.dimensions.items(): + if dname == 'Time': + dst.createDimension(dname, 1) + else: + dst.createDimension(dname, len(dim)) + + # Identify variables to keep: + # 1. Time-varying variables not in the exclude list + # 2. Static variables required by PyCECT for area weighting + keep_dynamic = {} + keep_static = {} + for name, var in src.variables.items(): + if name in exclude: + continue + if 'Time' in var.dimensions: + keep_dynamic[name] = var + elif name in PYCECT_REQUIRED_STATIC: + keep_static[name] = var + + kept = 0 + + # Write static variables (no time slicing needed) + for name, var in keep_static.items(): + outvar = dst.createVariable(name, var.dtype, var.dimensions) + outvar.setncatts({k: var.getncattr(k) for k in var.ncattrs()}) + outvar[:] = var[:] + kept += 1 + + # Write time-varying variables (extract single time slice, compress) + for name, var in keep_dynamic.items(): + dims = var.dimensions + use_zlib = var.size > 1000 + outvar = dst.createVariable( + name, var.dtype, dims, + zlib=use_zlib, complevel=1) + outvar.setncatts({k: var.getncattr(k) for k in var.ncattrs()}) + + tidx = dims.index('Time') + slices = [slice(None)] * len(dims) + slices[tidx] = slice(tslice, tslice + 1) + outvar[:] = var[tuple(slices)] + kept += 1 + + skipped = len(src.variables) - kept + + in_size = os.path.getsize(infile) / 1048576 + out_size = os.path.getsize(outfile) / 1048576 + print(f"Kept {kept} variables ({len(keep_dynamic)} dynamic + " + f"{len(keep_static)} static), dropped {skipped}, " + f"tslice={tslice}, {in_size:.0f}MB -> {out_size:.0f}MB") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Trim MPAS history files for ECT') + parser.add_argument('input', help='Input history file') + parser.add_argument('output', help='Output trimmed file') + parser.add_argument('--tslice', type=int, default=-1, + help='Time slice index to extract (negative counts from end, e.g. -1 = last)') + parser.add_argument('--exclude-file', + help='File listing variable names to exclude') + args = parser.parse_args() + + exclude = [] + if args.exclude_file: + with open(args.exclude_file) as f: + exclude = [line.strip() for line in f + if line.strip() and not line.startswith('#')] + + trim_history(args.input, args.output, args.tslice, exclude) diff --git a/.github/actions/setup-nsight-systems/action.yml b/.github/actions/setup-nsight-systems/action.yml new file mode 100644 index 0000000000..9d28e73d33 --- /dev/null +++ b/.github/actions/setup-nsight-systems/action.yml @@ -0,0 +1,56 @@ +name: 'Setup Nsight Systems CLI' +description: | + Ensure a working `nsys` is available for GPU profiling. If the container image + does not provide Nsight Systems (or only an NVHPC stub), installs the + `nsight-systems-cli` RPM set from NVIDIA's devtools repository (RHEL/Alma/Rocky). + Caches downloaded RPMs under `.cache/nsight-systems-rpms` for faster reruns. + +runs: + using: composite + steps: + - id: nsys_ver + shell: bash + run: | + V=1 + if [ -f .github/ci-config.env ]; then + L=$(grep '^NSYS_CLI_CACHE_VERSION=' .github/ci-config.env | head -1 || true) + if [ -n "${L}" ]; then + V="${L#*=}" + V="${V%%#*}" + V="${V%%[[:space:]]*}" + fi + fi + echo "version=${V}" >> "${GITHUB_OUTPUT}" + echo "NSYS_CLI_CACHE_VERSION=${V}" >> "${GITHUB_ENV}" + + - name: Cache nsight-systems-cli RPMs + id: cache-nsys + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}/.cache/nsight-systems-rpms + # Prefix is what you search for under repo Settings → Actions → Caches (not "nsys build"; we cache .rpm files) + key: nsight-systems-cli-rpms-v${{ steps.nsys_ver.outputs.version }}-rhel9-${{ runner.arch }}-${{ hashFiles('.github/scripts/install-nsight-systems-cli.sh', '.github/scripts/resolve-nsys.sh') }} + + - name: Install nsight-systems-cli + shell: bash + env: + NSYS_RPM_CACHE_DIR: ${{ github.workspace }}/.cache/nsight-systems-rpms + run: | + chmod +x .github/scripts/install-nsight-systems-cli.sh + bash .github/scripts/install-nsight-systems-cli.sh + + - name: Nsight RPM cache (for Actions cache UI) + shell: bash + run: | + { + echo "### Nsight Systems CLI — RPM cache" + echo "" + echo "This caches **downloaded NVIDIA RPMs** in \`.cache/nsight-systems-rpms/\`, not a standalone \`nsys\` build. After \`dnf install\`, \`nsys\` is on the container filesystem." + echo "" + echo "In **Settings → Actions → Caches**, search for keys starting with \`nsight-systems-cli-rpms-\`." + echo "" + echo "- **cache-hit** (this run): \`${{ steps.cache-nsys.outputs.cache-hit }}\`" + } >> "${GITHUB_STEP_SUMMARY}" + if [ -d .cache/nsight-systems-rpms ]; then + echo "- **RPM dir size:** \`$(du -sh .cache/nsight-systems-rpms 2>/dev/null || echo unknown)\`" >> "${GITHUB_STEP_SUMMARY}" + fi diff --git a/.github/actions/validate-ect/action.yml b/.github/actions/validate-ect/action.yml new file mode 100644 index 0000000000..c6037eaa89 --- /dev/null +++ b/.github/actions/validate-ect/action.yml @@ -0,0 +1,210 @@ +name: 'Validate ECT' +description: > + Run PyCECT against an ensemble summary file to validate history output. + Installs PyCECT at the pinned version from ci-config.env, downloads the + ensemble summary from a GitHub release, runs pyCECT, and writes an + enriched result file with dimension metadata for the summary action. + +inputs: + history-dir: + description: 'Path to directory containing history .nc files to validate' + required: true + label: + description: 'Human-readable label for log annotations (e.g. gcc/mpich3/smiol/4proc)' + required: true + mpas-version: + description: 'MPAS version string (used to build the ect-v{version} release tag)' + required: true + dimensions: + description: 'Multi-line key=value pairs describing this test combination (written into result file)' + required: false + default: '' + +outputs: + result: + description: 'ECT result: PASSED, FAILED, SKIPPED, or ERROR' + value: ${{ steps.outcome.outputs.result }} + available: + description: 'Whether the ensemble summary file was found (true/false)' + value: ${{ steps.summary.outputs.available }} + +runs: + using: 'composite' + steps: + - name: Load ECT configuration + id: config + shell: bash + run: | + CI_CONFIG=".github/ci-config.env" + if [ ! -f "${CI_CONFIG}" ]; then + echo "::error::ci-config.env not found" + exit 1 + fi + source "${CI_CONFIG}" + + # ECT release tag is derived from the MPAS version (passed in), + # not stored in ci-config.env. See .github/actions/mpas-version. + echo "summary-file=${ECT_SUMMARY_FILE}" >> $GITHUB_OUTPUT + echo "pycect-tag=${PYCECT_TAG}" >> $GITHUB_OUTPUT + echo "pycect-commit=${PYCECT_COMMIT}" >> $GITHUB_OUTPUT + echo "release-tag=ect-v${{ inputs.mpas-version }}" >> $GITHUB_OUTPUT + + - name: Install PyCECT dependencies + shell: bash + run: pip install "numpy<2" scipy netCDF4 + + - name: Clone PyCECT + shell: bash + run: | + TAG="${{ steps.config.outputs.pycect-tag }}" + COMMIT="${{ steps.config.outputs.pycect-commit }}" + echo "Cloning PyCECT at tag ${TAG} (commit ${COMMIT})..." + git clone --branch "${TAG}" https://github.com/NCAR/PyCECT.git pycect + cd pycect + if [ -n "${COMMIT}" ]; then + ACTUAL=$(git rev-parse HEAD) + if [ "${ACTUAL}" != "${COMMIT}" ]; then + echo "::error::PyCECT commit mismatch: expected ${COMMIT}, got ${ACTUAL}" + exit 1 + fi + fi + + - name: Download ensemble summary file + id: summary + shell: bash + run: | + source .github/ci-config.env + SUMMARY="${{ steps.config.outputs.summary-file }}" + TAG="${{ steps.config.outputs.release-tag }}" + REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + URL="https://github.com/${REPO}/releases/download/${TAG}/${SUMMARY}" + + echo "Downloading ${SUMMARY} from release ${TAG}..." + HTTP_CODE=$(curl --retry 5 --retry-delay 5 -sL -w "%{http_code}" \ + "${URL}" -o "${SUMMARY}") + if [ "${HTTP_CODE}" != "200" ]; then + echo "::warning::Ensemble summary not available at ${URL} (HTTP ${HTTP_CODE})" + echo "available=false" >> $GITHUB_OUTPUT + exit 0 + fi + echo "Downloaded summary file: $(du -h ${SUMMARY})" + echo "available=true" >> $GITHUB_OUTPUT + echo "summary-file=${SUMMARY}" >> $GITHUB_OUTPUT + + - name: List test files + if: steps.summary.outputs.available == 'true' + shell: bash + run: | + echo "=== ECT test history files ===" + ls -la ${{ inputs.history-dir }}/ 2>/dev/null || echo "No history files found" + + - name: Run PyCECT + id: pycect + if: steps.summary.outputs.available == 'true' + shell: bash + run: | + LABEL="${{ inputs.label }}" + # --tslice 0 is invariant: history files are pre-trimmed to a single + # time slice by run-perturb-mpas (trim_history.py creates Time=1). + set +e + python pycect/pyCECT.py \ + --sumfile ${{ steps.summary.outputs.summary-file }} \ + --indir ${{ inputs.history-dir }} \ + --tslice 0 \ + --mpas \ + --verbose \ + --printStdMean \ + 2>&1 | tee pycect_output.txt + PYCECT_STATUS=$? + set -e + + if [ $PYCECT_STATUS -ne 0 ]; then + echo "::error::pyCECT crashed with exit code ${PYCECT_STATUS}" + echo "result=ERROR" >> $GITHUB_OUTPUT + exit 1 + fi + + if grep -qF '****PASSED****' pycect_output.txt; then + echo "::notice::ECT (${LABEL}): PASSED" + echo "result=PASSED" >> $GITHUB_OUTPUT + elif grep -qF '****FAILED****' pycect_output.txt; then + echo "::error::ECT (${LABEL}): FAILED" + echo "result=FAILED" >> $GITHUB_OUTPUT + exit 1 + else + echo "::warning::Could not determine ECT result from pyCECT output" + echo "result=ERROR" >> $GITHUB_OUTPUT + exit 1 + fi + + - name: ECT Results + if: steps.pycect.outputs.result != '' + shell: bash + run: | + LABEL="${{ inputs.label }}" + RESULT="${{ steps.pycect.outputs.result }}" + + if [ "${RESULT}" = "PASSED" ]; then + ICON=":white_check_mark:" + else + ICON=":x:" + fi + + { + echo "## ECT Result: ${ICON} ${RESULT}" + echo "" + echo "**Configuration:** \`${LABEL}\`" + echo "" + echo '```' + grep -E '(PASSED|FAILED|global|regional|Overall)' pycect_output.txt || true + echo '```' + if grep -q 'standardized mean' pycect_output.txt; then + echo "" + echo "
Standardized Mean Summary" + echo "" + echo '```' + grep 'standardized mean' pycect_output.txt || true + echo '```' + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" + + echo "================================" + echo " ECT Result: ${RESULT}" + echo " Label: ${LABEL}" + echo "================================" + + - name: Fail if summary unavailable + if: steps.summary.outputs.available != 'true' + shell: bash + run: | + { + echo "## ECT Result: :warning: SKIPPED" + echo "" + echo "Ensemble summary file not found. Run \`ect-ensemble-gen.yml\` first." + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::ECT validation cannot run — ensemble summary file not found. Run ect-ensemble-gen.yml first." + exit 1 + + - name: Write result file + id: outcome + if: always() + shell: bash + run: | + RESULT="${{ steps.pycect.outputs.result }}" + if [ -z "${RESULT}" ]; then + if [ "${{ steps.summary.outputs.available }}" != "true" ]; then + RESULT="SKIPPED" + else + RESULT="NO_DATA" + fi + fi + echo "result=${RESULT}" >> $GITHUB_OUTPUT + + { + echo "result=${RESULT}" + DIMS="${{ inputs.dimensions }}" + if [ -n "${DIMS}" ]; then + echo "${DIMS}" + fi + } > ect-result.txt diff --git a/.github/ci-config.env b/.github/ci-config.env new file mode 100644 index 0000000000..d7a13a6a07 --- /dev/null +++ b/.github/ci-config.env @@ -0,0 +1,138 @@ +# .github/ci-config.env — Central CI configuration +# ────────────────────────────────────────────────── +# Workflows and composite actions source this file. +# Edit here to change containers, compiler targets, MPI settings, +# test data versions, and test-specific parameters. + + +# ── Container images ────────────────────────────── +# CI jobs run inside Docker containers from ncarcisl/hpcdev with +# pre-installed compilers, MPI libraries, and I/O libraries. +# +# Image names are assembled from the templates below, replacing +# {compiler} and {mpi} placeholders at workflow start. + +# CPU container template. +CONTAINER_IMAGE="docker.io/ncarcisl/hpcdev-x86_64:almalinux9-{compiler}-{mpi}-26.02" + +# GPU (CUDA) container template. +CONTAINER_IMAGE_GPU="docker.io/ncarcisl/hpcdev-x86_64:almalinux9-{compiler}-{mpi}-cuda-26.02" + +# Per-compiler image overrides (optional). +# Format: CONTAINER_IMAGE_{compiler}="template" + +# ── Compiler / MPI name mappings ───────────────── +# Workflows use short family names (gcc, nvhpc, oneapi) for the +# {compiler} placeholder. If the Docker Hub image tag uses a different +# string, add a mapping here. Otherwise the family name is used as-is. +# +# Check available tags at: +# https://hub.docker.com/r/ncarcisl/hpcdev-x86_64/tags +# +# Example: workflow says "gcc" but the image tag is +# almalinux9-gcc14-mpich-26.02 (not almalinux9-gcc-mpich-26.02) +# so we need: +CONTAINER_COMPILER_gcc=gcc14 + +# Uncomment if an MPI mapping is ever needed: +# CONTAINER_MPI_mpich=mpich3 + + +# ── Compiler → Makefile target mapping ──────────── +# MPAS Makefile targets don't always match CI compiler names. +# Format: MAKE_TARGET_{compiler}={makefile_target} + +MAKE_TARGET_gcc=gfortran +MAKE_TARGET_nvhpc=nvhpc +MAKE_TARGET_oneapi=intel +MAKE_TARGET_clang=llvm + + +# ── Compiler workarounds ───────────────────────── + +# NVHPC: portable target architecture (CI builds and runs on different hosts). +NVHPC_TARGET_ARCH=-tp=px + +# NVHPC: disable MPI Fortran 2008 bindings (broken with MPICH4 CFI support). +# https://github.com/pmodels/mpich/issues/6505 +NVHPC_EXTRA_MAKE_FLAGS="MPAS_MPI_F08=0" + +# Intel/OneAPI: same F08 binding issue with hpcdev MPI libraries. +ONEAPI_EXTRA_MAKE_FLAGS="MPAS_MPI_F08=0" + + +# ── MPI runtime flags ──────────────────────────── +# Required because CI runs inside containers as root. + +# OpenMPI: allow root execution and oversubscribe cores. +# Consumed by .github/actions/run-mpas, .github/actions/run-perturb-mpas, +# and .github/scripts/run-nsys-profile.sh (add a parallel `MPICH_RUN_FLAGS` +# in those same files if MPICH ever needs runtime flags). +OPENMPI_RUN_FLAGS="--allow-run-as-root --oversubscribe" + + +# ── Test data releases ──────────────────────────── +# Test data is stored as GitHub release assets on this repository. +# Each asset is independently versioned. The download-testdata action +# resolves release tags from these variables. +# +# Adding a new test case: +# 1. Create a test case archive (namelist + init.nc + streams, etc.) +# 2. gh release create testdata-{resolution}-v1 {resolution}.tar.gz \ +# --repo NCAR/MPAS-Model-CI +# 3. Add RELEASE_TESTDATA_{RESOLUTION}=testdata-{resolution}-v1 below + +# Repository that hosts release assets (test data, ECT summaries, restarts). +# Forks: leave this as-is to pull data from the upstream NCAR repo. +DATA_REPOSITORY=NCAR/MPAS-Model-CI + +RELEASE_TESTDATA_240KM=testdata-240km-v3 +RELEASE_TESTDATA_120KM=testdata-120km-v2 + +# ── ECT release tag ────────────────────────────── +# The ECT release tag (ect-v{MPAS_VERSION}) is derived at runtime from +# src/core_atmosphere/Registry.xml via the .github/actions/mpas-version +# composite action. There is no manual tag here — that ensures readers +# (validate-ect, _test-compiler, _test-gpu, ect-test) and the writer +# (ect-ensemble-gen) cannot drift out of sync. +# +# To publish ECT data for a new MPAS version, run ect-ensemble-gen.yml. + + +# ── ECT configuration ──────────────────────────── +# Parameters for the Ensemble Consistency Test (PyCECT). +# These are CI-specific settings not carried in the model namelist. +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +ECT_RESOLUTION=120km +ECT_ENSEMBLE_SIZE=200 +ECT_PERTURB_MAGNITUDE=1e-14 +ECT_PERTURB_VARIABLE=theta +ECT_SUMMARY_FILE=mpas_ect_summary_120km.nc +ECT_RESTART_FILE=120km-spinup-restart.nc +ECT_EXCLUDED_VARS=.github/data/ect_excluded_vars.txt +PYCECT_TAG=3.3.1 +PYCECT_COMMIT=b3c36a9d72ee211f396d1bc7078f6d5466916b0a + + +# ── BFB test configuration ─────────────────────── +# Used by BFB workflows (feature-ci-bfb branch, _test-bfb.yml). +# Bit-for-bit reproducibility tests using the 240km single-precision +# test case with 3 timesteps (config_dt=1200s). + +BFB_RESOLUTION=240km +BFB_PRECISION=single +BFB_RUN_DURATION=0_01:00:00 +BFB_RUN_TIMEOUT=10 + + +# ── Nsight Systems CLI (profile-gpu-nsight workflow) ─ +# Bump NSYS_CLI_CACHE_VERSION to invalidate GitHub Actions cache of downloaded RPMs +# when NVIDIA updates packages in the devtools repo. +NSYS_CLI_CACHE_VERSION=1 + + +# ── KNOWN ISSUES ───────────────────────────────── +# NVHPC+OpenMPI ECT may fail on GitHub-hosted runners (SIGABRT); subset +# workflows use continue-on-error for that combination. +# NVHPC and OneAPI builds set MPAS_MPI_F08=0 (see ONEAPI/NVHPC extra flags above). diff --git a/.github/data/ect_excluded_vars.txt b/.github/data/ect_excluded_vars.txt new file mode 100644 index 0000000000..478c6e786a --- /dev/null +++ b/.github/data/ect_excluded_vars.txt @@ -0,0 +1,35 @@ +# Variables excluded from ECT history files to reduce artifact size. +# These are either not analyzed by PyCECT or are expensive research +# diagnostics not needed for consistency testing. +# +# Ertel PV diagnostics (3D cell fields, ~12MB each on 120km mesh) +ertel_pv +u_pv +v_pv +theta_pv +vort_pv +iLev_DT +# +# PV tendency terms (3D cell fields, only present with physics enabled) +depv_dt_lw +depv_dt_sw +depv_dt_bl +depv_dt_cu +depv_dt_mix +dtheta_dt_mp +depv_dt_mp +depv_dt_diab +depv_dt_fric +depv_dt_diab_pv +depv_dt_fric_pv +# +# Edge velocity — PyCECT recommends uReconstructZonal/Meridional instead +u +# +# Integer variables — PyCECT excludes these automatically +i_rainnc +i_rainc +kpbl +# +# Time metadata string +xtime diff --git a/.github/scripts/compare-bfb-nc.py b/.github/scripts/compare-bfb-nc.py new file mode 100644 index 0000000000..b7f56a9cef --- /dev/null +++ b/.github/scripts/compare-bfb-nc.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Compare two NetCDF files for bitwise-identical variable *data*. + +NetCDF files from PIO vs SMIOL or different MPI layouts often differ in headers, +attributes, or chunking while variable arrays remain identical — `cmp` is too strict. +""" +from __future__ import annotations + +import sys + +import netCDF4 as nc +import numpy as np + + +def as_array(x): + if np.ma.isMaskedArray(x): + return np.ma.filled(x) + return np.asarray(x) + + +def main() -> int: + if len(sys.argv) != 3: + print("usage: compare-bfb-nc.py ", file=sys.stderr) + return 2 + path_ref, path_test = sys.argv[1], sys.argv[2] + diffs: list[str] = [] + with nc.Dataset(path_ref) as ds_ref, nc.Dataset(path_test) as ds_test: + vr = set(ds_ref.variables) + vt = set(ds_test.variables) + if vr != vt: + only_r = sorted(vr - vt) + only_t = sorted(vt - vr) + print( + "FAIL\t" + f"variable set mismatch: only in reference {only_r}, only in candidate {only_t}" + ) + return 1 + for name in sorted(vr): + a = as_array(ds_ref.variables[name][:]) + b = as_array(ds_test.variables[name][:]) + if a.shape != b.shape: + diffs.append(f"{name}: shape {a.shape} vs {b.shape}") + continue + if not np.array_equal(a, b): + if np.issubdtype(a.dtype, np.floating) or np.issubdtype( + a.dtype, np.complexfloating + ): + maxdiff = float(np.max(np.abs(a - b))) + diffs.append(f"{name} (max|diff|={maxdiff:.6e})") + else: + diffs.append(f"{name} (data differs)") + if diffs: + msg = "; ".join(diffs[:25]) + print(f"FAIL\t{msg}") + if len(diffs) > 25: + print(f"... and {len(diffs) - 25} more", file=sys.stderr) + return 1 + print("OK\tall variables bitwise-identical (NetCDF container/metadata may differ)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/install-nsight-systems-cli.sh b/.github/scripts/install-nsight-systems-cli.sh new file mode 100644 index 0000000000..a6f58e582f --- /dev/null +++ b/.github/scripts/install-nsight-systems-cli.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Install NVIDIA Nsight Systems CLI (nsys) on RHEL/Alma/Rocky via the devtools repo. +# Idempotent: skips if resolve-nsys.sh already finds a working nsys. +# Optional: NSYS_RPM_CACHE_DIR — directory to store .rpm files for actions/cache. +# +# shellcheck shell=bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/resolve-nsys.sh" + +_emit_github_env() { + if [ -n "${GITHUB_ENV:-}" ] && [ -n "${NSYS_BIN:-}" ]; then + { + echo "NSYS_BIN=${NSYS_BIN}" + echo "PATH=${PATH}" + } >> "${GITHUB_ENV}" + fi +} + +if resolve_nsys; then + echo "=== nsys already usable: ${NSYS_BIN} ===" + "${NSYS_BIN}" --version + _emit_github_env + exit 0 +fi + +echo "=== Installing nsight-systems-cli (NVIDIA devtools repo) ===" + +if ! command -v dnf &>/dev/null; then + echo "::error::dnf not found; this installer supports RHEL-family GPU images only." + exit 1 +fi + +# GPG key used by NVIDIA CUDA / devtools RPM repos (RHEL 9) +if [ -f /etc/os-release ]; then + # shellcheck source=/dev/null + source /etc/os-release +else + echo "::error::Cannot read /etc/os-release" + exit 1 +fi + +RHEL_VER="${VERSION_ID%%.*}" +ARCH_DIR="$(rpm --eval '%{_arch}' | sed 's/aarch/arm/')" +REPO_BASE="https://developer.download.nvidia.com/devtools/repos/rhel${RHEL_VER}/${ARCH_DIR}/" + +for key in \ + "https://developer.download.nvidia.com/compute/cuda/repos/rhel${RHEL_VER}/x86_64/D42D0685.pub" \ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub" +do + if rpm --import "${key}" 2>/dev/null; then + break + fi +done + +# Same repo layout as Nsight Systems Installation Guide (RHEL / Alma / Rocky). +cat > /etc/yum.repos.d/nvidia-devtools-ci.repo </dev/null; then + while IFS= read -r cand; do + if [ -x "${cand}" ] && "${cand}" --version &>/dev/null; then + export NSYS_BIN="${cand}" + export PATH="$(dirname "${cand}"):${PATH}" + echo "=== Pinned NSYS_BIN to RPM path: ${NSYS_BIN} ===" + break + fi + done < <(rpm -ql nsight-systems-cli 2>/dev/null | grep -E '/nsys$' || true) +fi + +if ! resolve_nsys; then + echo "::error::nsight-systems-cli did not yield a working nsys. Check NVIDIA repo and image OS version." + exit 1 +fi + +echo "=== nsys ready: ${NSYS_BIN} ===" +"${NSYS_BIN}" --version +_emit_github_env diff --git a/.github/scripts/resolve-nsys.sh b/.github/scripts/resolve-nsys.sh new file mode 100644 index 0000000000..188b36f7b4 --- /dev/null +++ b/.github/scripts/resolve-nsys.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Pick a working `nsys` binary. NVHPC may put a stub first on PATH that fails with +# "nsys-Error-Version ... Nsight_Systems/bin is not available in this installation"; +# we prefer /opt/nvidia/nsight-systems or CUDA toolkit paths and test with `nsys --version`. +# +# Usage: source this file from bash, then call resolve_nsys. +# On success: exports NSYS_BIN to the chosen executable. +# shellcheck shell=bash + +resolve_nsys() { + if [ -n "${NSYS_BIN:-}" ] && [ -x "${NSYS_BIN}" ] && "${NSYS_BIN}" --version &>/dev/null; then + export NSYS_BIN + return 0 + fi + + local prepend="" d dir + local -a dirs + + shopt -s nullglob + for d in /opt/nvidia/nsight-systems/*/bin /usr/local/cuda/bin /usr/local/cuda-*/bin; do + [ -d "$d" ] && prepend="${prepend}${d}:" + done + shopt -u nullglob + + export PATH="${prepend}${PATH}" + + IFS=':' read -ra dirs <<< "${PATH}" + for dir in "${dirs[@]}"; do + [ -z "$dir" ] && continue + [ -x "${dir}/nsys" ] || continue + if "${dir}/nsys" --version &>/dev/null; then + NSYS_BIN="${dir}/nsys" + export NSYS_BIN + return 0 + fi + done + + # Explicit paths (some images omit standard entries from PATH) + local c + shopt -s nullglob + for c in /opt/nvidia/nsight-systems/*/bin/nsys /usr/local/cuda/bin/nsys /usr/local/cuda-*/bin/nsys; do + [ -x "$c" ] || continue + if "$c" --version &>/dev/null; then + NSYS_BIN="$c" + export NSYS_BIN + shopt -u nullglob + return 0 + fi + done + shopt -u nullglob + return 1 +} diff --git a/.github/scripts/run-nsys-profile.sh b/.github/scripts/run-nsys-profile.sh new file mode 100644 index 0000000000..ce20625455 --- /dev/null +++ b/.github/scripts/run-nsys-profile.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Run MPAS-A under Nsight Systems (nsys profile). Intended for CIRRUS GPU CI. +# Usage: run-nsys-profile.sh +set -euo pipefail + +WORKDIR="${1:?workdir required}" +NUM_PROCS="${2:?num-procs required}" +MPI_IMPL="${3:?mpi-impl required}" +TIMEOUT="${4:?timeout minutes required}" +NSYS_BASENAME="${5:?nsys output basename required}" + +if [ -f /container/config_env.sh ]; then + # shellcheck source=/dev/null + source /container/config_env.sh +fi + +if [ -z "${LD_LIBRARY_PATH:-}" ]; then + export LD_LIBRARY_PATH="/usr/lib64:/usr/lib" +fi + +REPO_ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" +CI_CONFIG="${REPO_ROOT}/.github/ci-config.env" +if [ -f "${CI_CONFIG}" ]; then + # shellcheck source=/dev/null + source "${CI_CONFIG}" +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/resolve-nsys.sh" + +if ! resolve_nsys; then + echo "::error::No working nsys found. NVHPC may expose a stub that fails with Nsight version errors;" + echo "::error::install a full Nsight Systems build under /opt/nvidia/nsight-systems or ensure CUDA toolkit nsys is on PATH." + exit 1 +fi + +echo "=== nsys (${NSYS_BIN}) ===" +"${NSYS_BIN}" --version + +if [ -n "${GITHUB_ENV:-}" ]; then + { + echo "NSYS_BIN=${NSYS_BIN}" + } >> "${GITHUB_ENV}" +fi + +MPI_FLAGS="" +if [ "${MPI_IMPL}" = "openmpi" ]; then + MPI_FLAGS="${OPENMPI_RUN_FLAGS:---allow-run-as-root --oversubscribe}" +fi + +ulimit -s unlimited 2>/dev/null || true + +cd "${WORKDIR}" + +OUT_ABS="${PWD}/${NSYS_BASENAME}" +echo "=== Nsight profile ===" +echo " workdir: ${WORKDIR}" +echo " ranks: ${NUM_PROCS}" +echo " mpi: ${MPI_IMPL}" +echo " output: ${OUT_ABS}" +echo " timeout: ${TIMEOUT}m" + +set +e +timeout "${TIMEOUT}"m "${NSYS_BIN}" profile \ + --trace=cuda,nvtx,osrt \ + --stats=true \ + -o "${OUT_ABS}" \ + mpirun -n "${NUM_PROCS}" ${MPI_FLAGS} ./atmosphere_model +RUN_STATUS=$? +set -e + +if [ "${RUN_STATUS}" -ne 0 ]; then + echo "::warning::Profiled run exited with status ${RUN_STATUS}" + exit "${RUN_STATUS}" +fi + +echo "=== nsys profile finished ===" +ls -la "${NSYS_BASENAME}".* 2>/dev/null || ls -la ./*.nsys-rep 2>/dev/null || true diff --git a/.github/workflows/_test-bfb.yml b/.github/workflows/_test-bfb.yml new file mode 100644 index 0000000000..aa5ff3732c --- /dev/null +++ b/.github/workflows/_test-bfb.yml @@ -0,0 +1,427 @@ +# Reusable workflow: bit-for-bit reproducibility tests. +# +# Callers pass a JSON array of *variants*: independent run configurations that must +# produce bitwise-identical history variable data. The first variant (or +# reference_index) is the reference; every other variant is compared against it. +# +# Optional input `gpu: 'true'`: all variants use CUDA image + OpenACC + CIRRUS (nvhpc only). +# Alternatively set `gpu: 'false'` and add per-variant `"openacc": true|false` to mix +# CPU (GitHub-hosted) and GPU (CIRRUS) NVHPC builds in one comparison. +# +# MPI rank count and PIO vs SMIOL are common dimensions but not special — add any +# new scenario by appending another variant object (and a small caller workflow). + +name: _test-bfb + +permissions: + contents: read + actions: write + +on: + workflow_call: + inputs: + compiler: + description: 'Compiler family for the container image (gcc, nvhpc, oneapi)' + required: true + type: string + mpi: + description: 'MPI implementation for the container image (mpich, openmpi)' + required: true + type: string + variants: + description: > + JSON array of at least two variant objects. Required per variant: id (unique slug), + ranks (MPI processes). Optional: use_pio (default false), openacc (default follows gpu input), + label (summary text), resolution, run_duration (override workflow defaults for that variant only). + required: true + type: string + resolution: + description: 'Default test case resolution when a variant omits resolution' + required: false + type: string + default: '240km' + precision: + description: 'Floating-point precision for all builds (single or double)' + required: false + type: string + default: 'single' + run-duration: + description: 'Default model run duration (D_HH:MM:SS) when a variant omits run_duration' + required: false + type: string + default: '0_01:00:00' + run-timeout: + description: 'Run step timeout in minutes' + required: false + type: string + default: '10' + reference_index: + description: 'Which variant (0-based) is the reference for comparison' + required: false + type: number + default: 0 + gpu: + description: 'If true, CUDA image + OpenACC build + CIRRUS GPU runners (requires compiler nvhpc). Same security model as _test-gpu — use workflow_dispatch only.' + required: false + type: string + default: 'false' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + build_matrix: ${{ steps.matrices.outputs.build_matrix }} + run_variants: ${{ steps.matrices.outputs.run_variants }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - name: Validate GPU BFB inputs + if: ${{ inputs.gpu == 'true' }} + shell: bash + run: | + set -euo pipefail + if [ "${{ inputs.compiler }}" != "nvhpc" ]; then + echo "::error::gpu=true requires compiler=nvhpc (got ${{ inputs.compiler }})" + exit 1 + fi + + - name: Parse variants + id: parse + shell: bash + env: + VARIANTS_JSON: ${{ inputs.variants }} + REF_IDX: ${{ inputs.reference_index }} + run: | + set -euo pipefail + RI="${REF_IDX:-0}" + echo "${VARIANTS_JSON}" | jq -e 'type == "array" and length >= 2' >/dev/null \ + || { echo "::error::variants must be a JSON array with at least two entries"; exit 1; } + echo "${VARIANTS_JSON}" | jq -e 'all(.[]; has("id") and has("ranks"))' >/dev/null \ + || { echo "::error::each variant must include id and ranks"; exit 1; } + echo "${VARIANTS_JSON}" | jq -e 'all(.[]; .id | test("^[a-zA-Z0-9._-]+$"))' >/dev/null \ + || { echo "::error::variant id must match ^[a-zA-Z0-9._-]+$"; exit 1; } + N=$(echo "${VARIANTS_JSON}" | jq 'length') + if [ "${RI}" -lt 0 ] || [ "${RI}" -ge "${N}" ]; then + echo "::error::reference_index must be between 0 and $((N - 1))" + exit 1 + fi + + GDEF_JSON='false' + [ "${{ inputs.gpu }}" = "true" ] && GDEF_JSON='true' + NEED_CUDA=$(echo "${VARIANTS_JSON}" | jq -r --argjson gdef "${GDEF_JSON}" 'any(.[]; (.openacc // $gdef))') + echo "need_cuda=${NEED_CUDA}" >> "$GITHUB_OUTPUT" + + - name: Require nvhpc for OpenACC variants + if: ${{ steps.parse.outputs.need_cuda == 'true' }} + shell: bash + run: | + set -euo pipefail + if [ "${{ inputs.compiler }}" != "nvhpc" ]; then + echo "::error::OpenACC variants require compiler=nvhpc (got ${{ inputs.compiler }})" + exit 1 + fi + + - uses: ./.github/actions/resolve-container + id: container_cpu + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + gpu: '' + + - uses: ./.github/actions/resolve-container + id: container_cuda + if: ${{ steps.parse.outputs.need_cuda == 'true' }} + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + gpu: cuda + + - name: Compute build and run matrices + id: matrices + shell: bash + env: + VARIANTS_JSON: ${{ inputs.variants }} + DEF_RES: ${{ inputs.resolution }} + DEF_DUR: ${{ inputs.run-duration }} + CPU_IMG: ${{ steps.container_cpu.outputs.image }} + CUDA_IMG: ${{ steps.container_cuda.outputs.image }} + run: | + set -euo pipefail + GDEF_JSON='false' + [ "${{ inputs.gpu }}" = "true" ] && GDEF_JSON='true' + + if [ "${{ steps.parse.outputs.need_cuda }}" = "true" ] && [ -z "${CUDA_IMG}" ]; then + echo "::error::CUDA container image missing but OpenACC variants are present" + exit 1 + fi + + MIXED=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + '([.[] | (.openacc // $gdef)] | unique | length) > 1') + + BUILD_MATRIX=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + --argjson mixed "${MIXED}" --arg cpuimg "${CPU_IMG}" --arg cudaimg "${CUDA_IMG}" ' + [.[] | { + build_tag: (if (.use_pio // false) then "pio" else "smiol" end), + openacc: (.openacc // $gdef) + }] + | unique_by("\(.build_tag)-\(.openacc)") + | map(. + { + image: (if .openacc then $cudaimg else $cpuimg end), + artifact_name: ( + if $mixed then + ("exe-bfb-" + .build_tag + "-" + (if .openacc then "openacc" else "cpu" end)) + else + ("exe-bfb-" + .build_tag) + end + ) + }) + ') + + RUN_VARIANTS=$(echo "${VARIANTS_JSON}" | jq -c --argjson gdef "${GDEF_JSON}" \ + --argjson mixed "${MIXED}" --arg cpuimg "${CPU_IMG}" --arg cudaimg "${CUDA_IMG}" \ + --arg defres "${DEF_RES}" --arg defdur "${DEF_DUR}" \ + '[.[] | + (if (.use_pio // false) then "pio" else "smiol" end) as $bt | + (.openacc // $gdef) as $oa | + { + id, + build_tag: $bt, + openacc: $oa, + ranks: (.ranks | tonumber), + resolution: (.resolution // $defres), + run_duration: (.run_duration // $defdur), + label: (.label // .id), + image: (if $oa then $cudaimg else $cpuimg end), + artifact_name: ( + if $mixed then + ("exe-bfb-" + $bt + "-" + (if $oa then "openacc" else "cpu" end)) + else + ("exe-bfb-" + $bt) + end + ) + } + ]') + + echo "build_matrix=${BUILD_MATRIX}" >> "$GITHUB_OUTPUT" + echo "run_variants=${RUN_VARIANTS}" >> "$GITHUB_OUTPUT" + echo "mixed_openacc_modes=${MIXED}" >> "$GITHUB_OUTPUT" + echo "Build matrix: ${BUILD_MATRIX}" + echo "Run matrix: ${RUN_VARIANTS}" + + build: + needs: config + strategy: + fail-fast: true + matrix: + include: ${{ fromJSON(needs.config.outputs.build_matrix) }} + name: Build (${{ matrix.build_tag }}, ${{ matrix.openacc && 'openacc' || 'cpu' }}) + runs-on: ${{ matrix.openacc && fromJSON('{"group":"CIRRUS-4x8-gpu"}') || 'ubuntu-latest' }} + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler }} + use-pio: ${{ matrix.build_tag == 'pio' && 'true' || 'false' }} + openacc: ${{ matrix.openacc && 'true' || 'false' }} + precision: ${{ inputs.precision }} + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: ${{ matrix.artifact_name }} + path: atmosphere_model + retention-days: 1 + + run: + needs: [config, build] + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.config.outputs.run_variants) }} + name: Run (${{ matrix.id }}) + runs-on: ${{ matrix.openacc && fromJSON('{"group":"CIRRUS-4x8-gpu"}') || 'ubuntu-latest' }} + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: ${{ matrix.artifact_name }} + + - name: Check GPU availability + if: ${{ matrix.openacc }} + run: | + echo "=== GPU Information ===" + nvidia-smi || echo "WARNING: nvidia-smi failed" + + - name: Run MPAS-A + uses: ./.github/actions/run-mpas + with: + executable: ./atmosphere_model + num-procs: '${{ matrix.ranks }}' + resolution: ${{ matrix.resolution }} + run-duration: ${{ matrix.run_duration }} + run-timeout: ${{ inputs.run-timeout }} + mpi-impl: ${{ inputs.mpi }} + working-dir: run-${{ matrix.id }} + strict-exit-check: 'false' + + - name: Upload history output + uses: actions/upload-artifact@v6 + with: + name: bfb-history-${{ matrix.id }} + path: run-${{ matrix.id }}/history.*.nc + retention-days: 1 + + compare: + needs: [config, run] + if: ${{ !cancelled() && needs.run.result == 'success' }} + name: Compare BFB + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - name: Download all history files + uses: actions/download-artifact@v7 + with: + pattern: bfb-history-* + path: bfb-results + merge-multiple: false + + - name: Compare outputs + env: + VARIANTS_JSON: ${{ inputs.variants }} + REFERENCE_INDEX: ${{ inputs.reference_index }} + shell: bash + run: | + pip install --quiet netCDF4 numpy + + RI="${REFERENCE_INDEX:-0}" + + MERGED=$(echo "${VARIANTS_JSON}" | jq -c \ + --arg defres "${{ inputs.resolution }}" \ + --arg defdur "${{ inputs.run-duration }}" \ + '[.[] | { + id, + build_tag: (if (.use_pio // false) then "pio" else "smiol" end), + ranks: (.ranks | tonumber), + resolution: (.resolution // $defres), + run_duration: (.run_duration // $defdur), + label: (.label // .id) + }]') + + REF_ID=$(echo "${MERGED}" | jq -r --argjson idx "${RI}" '.[$idx].id') + REF_LABEL=$(echo "${MERGED}" | jq -r --argjson idx "${RI}" '.[$idx].label') + + echo "## Bit-for-Bit Comparison" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Reference variant: **${REF_LABEL}** (\`${REF_ID}\`)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + + PASS=0 + FAIL=0 + ERRORS="" + + REF_DIR="bfb-results/bfb-history-${REF_ID}" + REF_FILE=$(ls ${REF_DIR}/history.*.nc 2>/dev/null | head -1) + if [ -z "${REF_FILE}" ]; then + echo "::error::Reference history file not found in ${REF_DIR}" + find bfb-results -name '*.nc' -type f + exit 1 + fi + echo "Reference file: ${REF_FILE}" + + compare_files() { + local label="$1" + local test_file="$2" + local ref_file="$3" + + if [ ! -f "${test_file}" ]; then + echo "::error::${label}: history file missing" + ERRORS="${ERRORS}\n- **${label}**: :x: history file missing" + FAIL=$((FAIL + 1)) + return + fi + + if cmp -s "${ref_file}" "${test_file}"; then + echo "${label}: byte-identical" + echo "- **${label}**: :white_check_mark: byte-identical" >> "$GITHUB_STEP_SUMMARY" + PASS=$((PASS + 1)) + return + fi + + # compare-bfb-nc.py exits 1 on mismatch; with `set -e`, command substitution + # would abort before STATUS/DETAIL are handled — temporarily disable -e. + set +e + RESULT=$(python3 "${GITHUB_WORKSPACE}/.github/scripts/compare-bfb-nc.py" "${ref_file}" "${test_file}") + STATUS=$? + set -e + IFS=$'\t' read -r CODE DETAIL <<< "${RESULT}" + + if [ "${STATUS}" -eq 0 ] && [ "${CODE}" = "OK" ]; then + echo "${label}: data-identical (${DETAIL})" + echo "- **${label}**: :white_check_mark: data-identical (${DETAIL})" >> "$GITHUB_STEP_SUMMARY" + PASS=$((PASS + 1)) + else + echo "${label}: DIFFER — ${DETAIL}" + FAIL=$((FAIL + 1)) + ERRORS="${ERRORS}\n- **${label}**: :x: ${DETAIL}" + fi + } + + N=$(echo "${MERGED}" | jq 'length') + for ((i = 0; i < N; i++)); do + VID=$(echo "${MERGED}" | jq -r --argjson idx "$i" '.[$idx].id') + if [ "${VID}" = "${REF_ID}" ]; then + continue + fi + VLABEL=$(echo "${MERGED}" | jq -r --argjson idx "$i" '.[$idx].label') + DIR="bfb-results/bfb-history-${VID}" + TEST_FILE=$(ls ${DIR}/history.*.nc 2>/dev/null | head -1) + compare_files "${VLABEL} (\`${VID}\`)" "${TEST_FILE}" "${REF_FILE}" + done + + echo "" >> "$GITHUB_STEP_SUMMARY" + if [ -n "${ERRORS}" ]; then + echo -e "${ERRORS}" >> "$GITHUB_STEP_SUMMARY" + fi + + echo "" >> "$GITHUB_STEP_SUMMARY" + TOTAL=$((PASS + FAIL)) + echo "**Result: ${PASS}/${TOTAL} comparisons identical**" >> "$GITHUB_STEP_SUMMARY" + + if [ ${FAIL} -gt 0 ]; then + echo "::error::BFB comparison failed: ${FAIL} of ${TOTAL} comparisons differ" + exit 1 + fi + echo "All ${TOTAL} comparisons passed (byte-identical files or bitwise-identical variable data)" + + cleanup: + needs: [run, compare] + if: always() + runs-on: ubuntu-latest + name: Cleanup + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-bfb-") or startswith("bfb-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/_test-compiler.yml b/.github/workflows/_test-compiler.yml new file mode 100644 index 0000000000..bd132eb139 --- /dev/null +++ b/.github/workflows/_test-compiler.yml @@ -0,0 +1,258 @@ +# Reusable workflow: build and validate MPAS-A for a single compiler+MPI using ECT. +# Called by per-compiler per-MPI subset workflows (e.g. test-gcc-mpich.yml). +# +# Runs ECT with 4 MPI ranks. Validation uses the Ensemble Consistency Test +# (PyCECT) — perturbed runs compared against a pre-built ensemble summary — +# instead of bit-for-bit log comparison. + +name: _test-compiler + +permissions: + contents: read + +on: + workflow_call: + inputs: + compiler: + description: 'Compiler family (gcc, nvhpc, oneapi)' + required: true + type: string + mpi: + description: 'MPI implementation (mpich, openmpi)' + required: true + type: string + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Empty = this repo.' + required: false + type: string + default: '' + mpas-ref: + description: 'Git ref in the MPAS source repo (branch, tag, SHA)' + required: false + type: string + default: '' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + persist-credentials: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: ${{ inputs.compiler }} + mpi: ${{ inputs.mpi }} + - uses: ./.github/actions/mpas-version + id: version + + build: + needs: config + name: Build (${{ inputs.compiler }}, ${{ inputs.mpi }}, smiol) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + persist-credentials: false + + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + persist-credentials: false + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: | + cp -r _ci/.github . && rm -rf _ci + echo "## Source: ${{ inputs.mpas-repository }}@${{ inputs.mpas-ref }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler }} + use-pio: 'false' + precision: double + + - name: Upload executable + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: exe-${{ inputs.compiler }}-${{ inputs.mpi }}-smiol + path: atmosphere_model + retention-days: 1 + + ect-run: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} (${{ inputs.compiler }}, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + + - name: Download executable + id: download + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + continue-on-error: true + with: + name: exe-${{ inputs.compiler }}-${{ inputs.mpi }}-smiol + + - name: Download test case + if: steps.download.outcome == 'success' + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + if: steps.download.outcome == 'success' + id: cache-restart + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + if: steps.download.outcome == 'success' + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + DATA_REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + echo "Downloading ${RESTART}.gz from ${DATA_REPO} release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 5 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${DATA_REPO}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A (member ${{ matrix.member }}) + if: steps.download.outcome == 'success' + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: '${{ matrix.member }}' + member-end: '${{ matrix.member }}' + num-ranks: '4' + mpi-impl: ${{ inputs.mpi }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: always() && steps.download.outcome == 'success' + with: + name: ect-history-${{ inputs.compiler }}-${{ inputs.mpi }}-member${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + ect-validate: + needs: [config, ect-run] + if: ${{ needs.ect-run.result == 'success' }} + name: ECT Validate (${{ inputs.compiler }}, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + sparse-checkout: .github + persist-credentials: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.11' + + - name: Download history files + id: download + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + pattern: ect-history-${{ inputs.compiler }}-${{ inputs.mpi }}-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + if: steps.download.outcome == 'success' + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: ${{ inputs.compiler }}/${{ inputs.mpi }}/smiol + mpas-version: ${{ needs.config.outputs.mpas-version }} + dimensions: | + compiler=${{ inputs.compiler }} + mpi=${{ inputs.mpi }} + io=smiol + + - name: Upload result + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: ect-result-${{ inputs.compiler }}-${{ inputs.mpi }} + path: ect-result.txt + retention-days: 1 + + cleanup: + needs: [ect-run, ect-validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + permissions: + actions: write + + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-") or startswith("ect-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/_test-gpu.yml b/.github/workflows/_test-gpu.yml new file mode 100644 index 0000000000..42735c0bfb --- /dev/null +++ b/.github/workflows/_test-gpu.yml @@ -0,0 +1,272 @@ +# Reusable workflow: GPU (CUDA) validation via ECT (NVHPC). +# Called by test-gpu-mpich.yml and test-gpu-openmpi.yml. +# +# Builds MPAS-A with OpenACC/CUDA on CIRRUS self-hosted runners, then +# validates via ECT (PyCECT) — the same statistical test used for CPU +# subsets. GPU results are not expected to be bit-for-bit identical to CPU. +# +# Security: Runs on self-hosted CIRRUS runners. Callers must use workflow_dispatch +# only — never push or pull_request triggers (fork / unreviewed code risk on CIRRUS). + +name: _test-gpu + +on: + workflow_call: + inputs: + mpi: + description: 'MPI implementation (mpich, openmpi)' + required: true + type: string + # Cross-repo inputs disabled until NCAR security review of running + # external code on self-hosted CIRRUS runners is complete. + # mpas-repository: + # description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Empty = this repo.' + # required: false + # type: string + # default: '' + # mpas-ref: + # description: 'Git ref in the MPAS source repo (branch, tag, SHA)' + # required: false + # type: string + # default: '' + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + image-gpu: ${{ steps.gpu-container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + - uses: ./.github/actions/resolve-container + id: gpu-container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + gpu: cuda + - uses: ./.github/actions/mpas-version + id: version + + build: + needs: config + name: Build (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + # Cross-repo checkout disabled until NCAR security review is complete. + # To re-enable, restore mpas-repository/mpas-ref inputs above and + # replace the checkout step with: + # + # - uses: actions/checkout@v5 + # with: + # repository: ${{ inputs.mpas-repository || github.repository }} + # ref: ${{ inputs.mpas-ref || '' }} + # submodules: 'true' + # + # - uses: actions/checkout@v5 + # if: ${{ inputs.mpas-repository != '' }} + # with: + # path: _ci + # sparse-checkout: .github + # + # - name: Overlay CI infrastructure + # if: ${{ inputs.mpas-repository != '' }} + # shell: bash + # run: | + # cp -r _ci/.github . && rm -rf _ci + # echo "## Source: ${{ inputs.mpas-repository }}@${{ inputs.mpas-ref }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-gpu-cuda-${{ inputs.mpi }} + path: atmosphere_model + retention-days: 1 + + ect-run: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + + - name: Check GPU availability + run: | + echo "=== GPU Information ===" + nvidia-smi || echo "WARNING: nvidia-smi failed" + + - name: Download executable + id: download + uses: actions/download-artifact@v7 + continue-on-error: true + with: + name: exe-gpu-cuda-${{ inputs.mpi }} + + - name: Download test case + if: steps.download.outcome == 'success' + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + if: steps.download.outcome == 'success' + id: cache-restart + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + if: steps.download.outcome == 'success' + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + DATA_REPO="${DATA_REPOSITORY:-${GITHUB_REPOSITORY}}" + echo "Downloading ${RESTART}.gz from ${DATA_REPO} release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 5 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${DATA_REPO}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A (member ${{ matrix.member }}) + if: steps.download.outcome == 'success' + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: '${{ matrix.member }}' + member-end: '${{ matrix.member }}' + num-ranks: '4' + mpi-impl: ${{ inputs.mpi }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@v6 + if: always() && steps.download.outcome == 'success' + with: + name: ect-history-gpu-cuda-${{ inputs.mpi }}-member${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + ect-validate: + needs: [config, ect-run] + if: ${{ needs.ect-run.result == 'success' }} + name: ECT Validate (cuda, ${{ inputs.mpi }}) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Download history files + id: download + continue-on-error: true + uses: actions/download-artifact@v7 + with: + pattern: ect-history-gpu-cuda-${{ inputs.mpi }}-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + if: steps.download.outcome == 'success' + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: nvhpc/${{ inputs.mpi }}/cuda/smiol + mpas-version: ${{ needs.config.outputs.mpas-version }} + dimensions: | + compiler=nvhpc + mpi=${{ inputs.mpi }} + gpu=cuda + io=smiol + + - name: Upload result + if: always() + uses: actions/upload-artifact@v6 + with: + name: ect-result-gpu-cuda-${{ inputs.mpi }} + path: ect-result.txt + retention-days: 1 + + cleanup: + needs: [ect-run, ect-validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete temporary artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name | startswith("exe-") or startswith("ect-history-")) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/bfb-decomp-gpu.yml b/.github/workflows/bfb-decomp-gpu.yml new file mode 100644 index 0000000000..a3bd6e45d7 --- /dev/null +++ b/.github/workflows/bfb-decomp-gpu.yml @@ -0,0 +1,20 @@ +# BFB on GPU: same OpenACC/CUDA build, different MPI rank counts (decomposition). +# CIRRUS self-hosted runners — trigger via workflow_dispatch only (see _test-gpu.yml). + +name: "BFB: Decomposition GPU (1 vs 4 ranks)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'true' + precision: single + run-timeout: '20' + variants: >- + [{"id":"r1","ranks":1,"label":"1 rank"}, + {"id":"r4","ranks":4,"label":"4 ranks"}] diff --git a/.github/workflows/bfb-decomp.yml b/.github/workflows/bfb-decomp.yml new file mode 100644 index 0000000000..438dea26d5 --- /dev/null +++ b/.github/workflows/bfb-decomp.yml @@ -0,0 +1,19 @@ +# Example BFB caller: same I/O build, different MPI rank counts (decomposition). +# Add new scenarios by copying this file and editing `variants` (see `_test-bfb.yml` and `.github/ci-config.env` BFB_*). + +name: "BFB: Decomposition (1 vs 4 ranks)" + +on: + workflow_dispatch: + push: + branches: [hackathon, 'hackathon/**', 'hackathon-*', feature-ci-bfb] + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: gcc + mpi: mpich + variants: >- + [{"id":"r1","ranks":1,"label":"1 rank"}, + {"id":"r4","ranks":4,"label":"4 ranks"}] diff --git a/.github/workflows/bfb-io-gpu.yml b/.github/workflows/bfb-io-gpu.yml new file mode 100644 index 0000000000..34fe2967cb --- /dev/null +++ b/.github/workflows/bfb-io-gpu.yml @@ -0,0 +1,20 @@ +# BFB on GPU: two I/O builds (SMIOL vs PIO), same MPI rank count. +# CIRRUS self-hosted runners — trigger via workflow_dispatch only (see _test-gpu.yml). + +name: "BFB: I/O GPU (SMIOL vs PIO)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'true' + precision: single + run-timeout: '20' + variants: >- + [{"id":"smiol-r4","ranks":4,"label":"SMIOL, 4 ranks"}, + {"id":"pio-r4","use_pio":true,"ranks":4,"label":"PIO, 4 ranks"}] diff --git a/.github/workflows/bfb-io.yml b/.github/workflows/bfb-io.yml new file mode 100644 index 0000000000..9eb2242f5a --- /dev/null +++ b/.github/workflows/bfb-io.yml @@ -0,0 +1,19 @@ +# Example BFB caller: two I/O builds (SMIOL vs PIO), same rank count. +# Add new scenarios by copying this file and editing `variants` (see `_test-bfb.yml` and `.github/ci-config.env` BFB_*). + +name: "BFB: I/O (SMIOL vs PIO)" + +on: + workflow_dispatch: + push: + branches: [hackathon, 'hackathon/**', 'hackathon-*', feature-ci-bfb] + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: gcc + mpi: mpich + variants: >- + [{"id":"smiol-r4","ranks":4,"label":"SMIOL, 4 ranks"}, + {"id":"pio-r4","use_pio":true,"ranks":4,"label":"PIO, 4 ranks"}] diff --git a/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml b/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml new file mode 100644 index 0000000000..78d477a7e4 --- /dev/null +++ b/.github/workflows/bfb-nvhpc-cpu-vs-gpu.yml @@ -0,0 +1,24 @@ +# BFB: NVHPC without OpenACC (GitHub-hosted) vs NVHPC + OpenACC (CIRRUS GPU). +# Same MPI, rank count, resolution, and single precision — compares history variable +# data bitwise. CPU vs GPU OpenACC often differs; this workflow makes that visible. +# +# workflow_dispatch for routine use (after this file is on the default branch). +# GPU/OpenACC comparisons remain dispatch-only so PR code cannot run on CIRRUS. + +name: "BFB: NVHPC CPU vs GPU (OpenACC)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-bfb.yml + with: + compiler: nvhpc + mpi: mpich + gpu: 'false' + precision: single + run-timeout: '20' + variants: >- + [{"id":"cpu","ranks":4,"openacc":false,"label":"NVHPC CPU (no OpenACC)"}, + {"id":"gpu","ranks":4,"openacc":true,"label":"NVHPC GPU (OpenACC)"}] diff --git a/.github/workflows/compile-nvhpc-cuda-mpich.yml b/.github/workflows/compile-nvhpc-cuda-mpich.yml new file mode 100644 index 0000000000..6bdffda53b --- /dev/null +++ b/.github/workflows/compile-nvhpc-cuda-mpich.yml @@ -0,0 +1,58 @@ +# NVHPC + MPICH + CUDA: compile-only on GitHub-hosted runners. +# Validates the OpenACC/CUDA toolchain (no GPU present; no model run). + +name: "NVHPC+MPICH+CUDA (compile-only)" + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + config: + name: Resolve CUDA container + runs-on: ubuntu-latest + outputs: + image: ${{ steps.gpu.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: gpu + with: + compiler: nvhpc + mpi: mpich + gpu: cuda + + compile: + needs: config + name: Compile (OpenACC, MPICH) + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + build-timeout: '45' + + - name: Summary + shell: bash + run: | + echo "## NVHPC + MPICH + CUDA compile-only" >> "$GITHUB_STEP_SUMMARY" + echo "Built \`atmosphere_model\` with OpenACC in ${{ needs.config.outputs.image }}." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..0acd7c9981 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,125 @@ +# Code Coverage workflow +# Builds MPAS-A with GCC coverage instrumentation, runs the 240km test case, +# generates an lcov report, and uploads to Codecov. +# +name: Code Coverage + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + push: + branches: [master] + +jobs: + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: gcc + mpi: mpich + + coverage: + name: Build, Run, and Generate Coverage + needs: config + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Install lcov + run: | + dnf install -y epel-release + dnf install -y lcov + + - name: Build MPAS-A with coverage flags + run: | + source /container/config_env.sh + + # Patch gfortran target: swap -O3 for -O0 --coverage -g + sed -i 's/"FFLAGS_OPT = -O3 -ffree-line-length-none/"FFLAGS_OPT = -O0 --coverage -g -ffree-line-length-none/' Makefile + sed -i 's/"CFLAGS_OPT = -O3"/"CFLAGS_OPT = -O0 --coverage -g"/' Makefile + sed -i 's/"CXXFLAGS_OPT = -O3"/"CXXFLAGS_OPT = -O0 --coverage -g"/' Makefile + sed -i 's/"LDFLAGS_OPT = -O3"/"LDFLAGS_OPT = --coverage"/' Makefile + + make gfortran CORE=atmosphere --jobs $(nproc) + + - name: Download test data + uses: ./.github/actions/download-testdata + with: + resolution: 240km + + - name: Run 240km test case + run: | + source /container/config_env.sh + cd 240km + ln -sf ../atmosphere_model . + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '0_00:30:00'/" namelist.atmosphere + ulimit -s unlimited 2>/dev/null || true + mpirun -n 1 ./atmosphere_model + + - name: Generate coverage report + run: | + source /container/config_env.sh + + lcov --capture \ + --directory src \ + --output-file coverage.info \ + --ignore-errors source,gcov + + # Remove external/system files from coverage + lcov --remove coverage.info \ + '/usr/*' \ + '*/external/*' \ + --output-file coverage.info \ + --ignore-errors unused + + - name: Upload to Codecov + uses: codecov/codecov-action@v6 + with: + files: coverage.info + flags: fortran + name: mpas-atmosphere + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage artifact + uses: actions/upload-artifact@v6 + if: always() + with: + name: coverage-report + path: coverage.info + retention-days: 30 diff --git a/.github/workflows/ect-ensemble-gen.yml b/.github/workflows/ect-ensemble-gen.yml new file mode 100644 index 0000000000..0b7cc41e92 --- /dev/null +++ b/.github/workflows/ect-ensemble-gen.yml @@ -0,0 +1,590 @@ +# Ensemble Consistency Test — Ensemble Generation +# Generates 200 perturbed MPAS-A simulations on the 120km mesh and produces +# an ensemble summary file using pyEnsSumMPAS for use with PyCECT. +# +# This is expensive (~200 model runs) and should only be triggered manually +# when there are major version or science changes requiring a new reference ensemble. +# +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +name: ECT Ensemble Generation + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + ensemble-size: + description: 'Number of ensemble members to generate' + type: number + required: false + default: 200 + members-per-job: + description: 'Number of ensemble members to run per job (batch size)' + type: number + required: false + default: 10 + num-ranks: + description: 'Number of MPI ranks per ensemble member' + type: number + required: false + default: 4 + compiler: + description: 'Compiler suite (gcc or nvhpc)' + type: choice + options: [gcc, nvhpc] + required: false + default: 'nvhpc' + mpi-impl: + description: 'MPI implementation (mpich, openmpi)' + type: choice + options: [mpich, openmpi] + required: false + default: 'mpich' + +jobs: + #=========================================================================== + # PREPARE: Compute batch matrix + #=========================================================================== + prepare: + runs-on: ubuntu-latest + outputs: + batches: ${{ steps.compute.outputs.batches }} + ensemble-size: ${{ steps.compute.outputs.ensemble-size }} + container-image: ${{ steps.container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: ${{ inputs.compiler || 'nvhpc' }} + mpi: ${{ inputs.mpi-impl || 'mpich' }} + + - name: Compute batch ranges + id: compute + run: | + ESIZE=${{ inputs.ensemble-size || 200 }} + BATCH=${{ inputs.members-per-job || 10 }} + echo "ensemble-size=${ESIZE}" >> $GITHUB_OUTPUT + + BATCHES="[" + for (( start=0; start> $GITHUB_OUTPUT + echo "Generated batches: ${BATCHES}" + + #=========================================================================== + # BUILD + #=========================================================================== + build: + needs: prepare + name: Build MPAS-A (${{ inputs.compiler || 'nvhpc' }}/${{ inputs.mpi-impl || 'mpich' }}) + runs-on: ubuntu-latest + outputs: + mpas-version: ${{ steps.version.outputs.mpas-version }} + mpas-commit: ${{ steps.version.outputs.mpas-commit }} + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Extract MPAS version + id: mpas_version + uses: ./.github/actions/mpas-version + + - name: Resolve commit and re-export version + id: version + shell: bash + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + COMMIT=$(git rev-parse --short=8 HEAD) + echo "mpas-version=${{ steps.mpas_version.outputs.version }}" >> $GITHUB_OUTPUT + echo "mpas-commit=${COMMIT}" >> $GITHUB_OUTPUT + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: ${{ inputs.compiler || 'nvhpc' }} + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-ect-ensemble + path: atmosphere_model + retention-days: 1 + + #=========================================================================== + # SPIN-UP: 24-hour unperturbed run to generate spun-up restart file + # Hydrometeor fields (rain, snow, cloud water) are zero in cold-start + # init.nc and need time to develop realistic values. + # See Price-Broncucia et al. (2025), Section 3.2. + #=========================================================================== + spinup: + needs: [prepare, build] + name: 24h Spin-up + runs-on: ubuntu-latest + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + + - name: Restore cached restart + id: cache + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download executable + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v7 + with: + name: exe-ect-ensemble + + - name: Make executable + if: steps.cache.outputs.cache-hit != 'true' + run: chmod +x atmosphere_model + + - name: Run 24h spin-up + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/run-mpas + with: + executable: ./atmosphere_model + resolution: 120km + num-procs: '4' + mpi-impl: ${{ inputs.mpi-impl || 'mpich' }} + run-duration: '1_00:00:00' + restart-interval: '1_00:00:00' + run-timeout: '350' + strict-exit-check: 'false' + + - name: Verify restart file + if: steps.cache.outputs.cache-hit != 'true' + run: | + RESTART=$(ls run-120km/restart.*.nc 2>/dev/null | head -1) + if [ -z "${RESTART}" ]; then + echo "::error::Spin-up did not produce a restart file" + ls -la run-120km/*.nc 2>/dev/null || true + exit 1 + fi + echo "Spin-up restart: ${RESTART} ($(du -h ${RESTART} | cut -f1))" + cp "${RESTART}" spinup-restart.nc + + - name: Cache restart file + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Upload restart artifact + uses: actions/upload-artifact@v6 + with: + name: spinup-restart + path: spinup-restart.nc + retention-days: 1 + + #=========================================================================== + # RUN: Batched ensemble members + #=========================================================================== + run-ensemble: + needs: [prepare, build, spinup] + strategy: + fail-fast: false + matrix: + batch: ${{ fromJson(needs.prepare.outputs.batches) }} + + name: Members ${{ matrix.batch.start }}-${{ matrix.batch.end }} + runs-on: ubuntu-latest + container: + image: ${{ needs.prepare.outputs.container-image }} + + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-ect-ensemble + + - name: Make executable + run: chmod +x atmosphere_model + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Download spin-up restart + uses: actions/download-artifact@v7 + with: + name: spinup-restart + + - name: Run ensemble members ${{ matrix.batch.start }}-${{ matrix.batch.end }} + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: ${{ matrix.batch.start }} + member-end: ${{ matrix.batch.end }} + num-ranks: ${{ inputs.num-ranks || 4 }} + mpi-impl: ${{ inputs.mpi-impl || 'mpich' }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: spinup-restart.nc + + - name: Upload history files + uses: actions/upload-artifact@v6 + with: + name: ect-ensemble-${{ matrix.batch.start }}-${{ matrix.batch.end }} + path: history-output/history.*.nc + retention-days: 1 + + #=========================================================================== + # SUMMARY: Generate ensemble summary file with pyEnsSumMPAS + # Security: This job only checks out THIS repo (for config/tooling). + # The MPAS version and commit come from the build job outputs, so no + # external repo code is checked out or executed here. + #=========================================================================== + generate-summary: + needs: [prepare, build, run-ensemble] + if: always() && needs.run-ensemble.result != 'cancelled' + name: Generate Summary + runs-on: ubuntu-latest + outputs: + summary-name: ${{ steps.meta.outputs.summary-name }} + summary-prefix: ${{ steps.meta.outputs.summary-prefix }} + mpas-version: ${{ steps.meta.outputs.mpas-version }} + mpas-repo: ${{ steps.meta.outputs.mpas-repo }} + branch: ${{ steps.meta.outputs.branch }} + commit: ${{ steps.meta.outputs.commit }} + date: ${{ steps.meta.outputs.date }} + resolution: ${{ steps.meta.outputs.resolution }} + ensemble-size: ${{ steps.meta.outputs.ensemble-size }} + compiler: ${{ steps.meta.outputs.compiler }} + mpi-impl: ${{ steps.meta.outputs.mpi-impl }} + precision: ${{ steps.meta.outputs.precision }} + run-duration: ${{ steps.meta.outputs.run-duration }} + perturb-magnitude: ${{ steps.meta.outputs.perturb-magnitude }} + perturb-variable: ${{ steps.meta.outputs.perturb-variable }} + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Collect metadata + id: meta + run: | + source .github/ci-config.env + ECT_SUMMARY_PREFIX="${ECT_SUMMARY_FILE%.nc}" + RUN_DURATION=0_02:36:00 + VERSION="${{ needs.build.outputs.mpas-version }}" + DATE=$(date -u +%Y-%m-%d) + SUMMARY_NAME="${ECT_SUMMARY_PREFIX}_v${VERSION}_${DATE//-/}.nc" + + MPAS_REPO="${{ inputs.mpas-repository || github.repository }}" + MPAS_BRANCH="${{ inputs.mpas-ref || github.ref_name }}" + MPAS_COMMIT="${{ needs.build.outputs.mpas-commit }}" + + echo "mpas-version=${VERSION}" >> $GITHUB_OUTPUT + echo "mpas-repo=${MPAS_REPO}" >> $GITHUB_OUTPUT + echo "branch=${MPAS_BRANCH}" >> $GITHUB_OUTPUT + echo "commit=${MPAS_COMMIT}" >> $GITHUB_OUTPUT + echo "date=${DATE}" >> $GITHUB_OUTPUT + echo "resolution=${ECT_RESOLUTION}" >> $GITHUB_OUTPUT + echo "ensemble-size=${{ needs.prepare.outputs.ensemble-size }}" >> $GITHUB_OUTPUT + echo "summary-prefix=${ECT_SUMMARY_PREFIX}" >> $GITHUB_OUTPUT + echo "summary-name=${SUMMARY_NAME}" >> $GITHUB_OUTPUT + echo "compiler=${{ inputs.compiler || 'nvhpc' }}" >> $GITHUB_OUTPUT + echo "mpi-impl=${{ inputs.mpi-impl || 'mpich' }}" >> $GITHUB_OUTPUT + echo "precision=double" >> $GITHUB_OUTPUT + echo "run-duration=${RUN_DURATION}" >> $GITHUB_OUTPUT + echo "perturb-magnitude=${ECT_PERTURB_MAGNITUDE}" >> $GITHUB_OUTPUT + echo "perturb-variable=${ECT_PERTURB_VARIABLE}" >> $GITHUB_OUTPUT + + echo "=== ECT Metadata ===" + echo " MPAS repo: ${MPAS_REPO}" + echo " MPAS version: ${VERSION}" + echo " Branch: ${MPAS_BRANCH}" + echo " Commit: ${MPAS_COMMIT}" + echo " Resolution: ${ECT_RESOLUTION}" + echo " Compiler: ${{ inputs.compiler || 'nvhpc' }}" + echo " MPI: ${{ inputs.mpi-impl || 'mpich' }}" + echo " Precision: double" + echo " Run duration: ${RUN_DURATION}" + echo " Perturbation: ${ECT_PERTURB_MAGNITUDE} (${ECT_PERTURB_VARIABLE})" + echo " Ensemble: ${{ needs.prepare.outputs.ensemble-size }} members" + echo " Summary: ${SUMMARY_NAME}" + + - name: Install dependencies + run: pip install "numpy<2" scipy netCDF4 + + - name: Clone PyCECT + run: | + source .github/ci-config.env + git clone --depth 1 --branch "${PYCECT_TAG}" https://github.com/NCAR/PyCECT.git pycect + + - name: Download all history files + uses: actions/download-artifact@v7 + with: + pattern: ect-ensemble-* + path: ensemble-files + merge-multiple: true + + - name: Validate ensemble files + run: | + echo "=== Validating ensemble history files ===" + GOOD=0 + BAD=0 + for f in ensemble-files/history.*.nc; do + if python3 -c "import netCDF4; netCDF4.Dataset('$f').close()" 2>/dev/null; then + GOOD=$((GOOD + 1)) + else + echo "::warning::Removing corrupted file: $f" + rm -f "$f" + BAD=$((BAD + 1)) + fi + done + echo "Valid: ${GOOD}, Corrupted: ${BAD}" + if [ "${GOOD}" -lt 48 ]; then + echo "::error::Too few valid ensemble members (${GOOD}). PyCECT needs more members than output variables (~47 after trimming). Recommended: 200." + exit 1 + fi + + - name: Generate ensemble summary + run: | + SUMMARY_NAME="${{ steps.meta.outputs.summary-name }}" + ESIZE=${{ needs.prepare.outputs.ensemble-size }} + NFILES=$(ls ensemble-files/history.*.nc 2>/dev/null | wc -l) + if [ "${NFILES}" -lt "${ESIZE}" ]; then + echo "::warning::Only ${NFILES} of ${ESIZE} requested members available (some may have been removed due to corruption)" + ESIZE=${NFILES} + fi + + JSONFILE=ect_pycect_exclude.json + echo '{"ExcludedVar": []}' > "${JSONFILE}" + + # --tslice 0 is invariant: ensemble files are pre-trimmed to a single + # time slice by run-perturb-mpas (trim_history.py creates Time=1). + python pycect/pyEnsSumMPAS.py \ + --esize ${ESIZE} \ + --indir ensemble-files \ + --sumfile "${SUMMARY_NAME}" \ + --tslice 0 \ + --tag $(date +%Y%m%d) \ + --model mpas \ + --mach github-actions \ + --verbose \ + --jsonfile ${JSONFILE} \ + --mpi_disable + + if [ ! -f "${SUMMARY_NAME}" ]; then + echo "::error::pyEnsSumMPAS did not produce ${SUMMARY_NAME}." + echo " PyCECT requires ensemble size >= number of output variables (~47 after trimming)." + echo " Current ensemble size: ${ESIZE}. Increase to at least 50 (recommended: 200)." + exit 1 + fi + + - name: Upload summary artifact + uses: actions/upload-artifact@v6 + if: always() + with: + name: mpas-ect-summary + path: ${{ steps.meta.outputs.summary-name }} + retention-days: 90 + if-no-files-found: warn + + - name: Summary + if: always() + run: | + SUMMARY_NAME="${{ steps.meta.outputs.summary-name }}" + echo "==========================================" + echo " ECT Ensemble Summary Generated" + echo "==========================================" + echo "" + echo " MPAS repo: ${{ steps.meta.outputs.mpas-repo }}" + echo " MPAS version: ${{ steps.meta.outputs.mpas-version }}" + echo " Branch: ${{ steps.meta.outputs.branch }}" + echo " Commit: ${{ steps.meta.outputs.commit }}" + echo " Resolution: ${{ steps.meta.outputs.resolution }}" + echo " Compiler: ${{ steps.meta.outputs.compiler }}/${{ steps.meta.outputs.mpi-impl }}" + echo " Precision: ${{ steps.meta.outputs.precision }}" + echo " Duration: ${{ steps.meta.outputs.run-duration }}" + echo " Perturbation: ${{ steps.meta.outputs.perturb-magnitude }} (${{ steps.meta.outputs.perturb-variable }})" + echo " Ensemble size: ${{ steps.meta.outputs.ensemble-size }}" + echo " Date: ${{ steps.meta.outputs.date }}" + echo "" + if [ -f "${SUMMARY_NAME}" ]; then + echo " File: ${SUMMARY_NAME}" + echo " Size: $(du -h "${SUMMARY_NAME}" | cut -f1)" + fi + + #=========================================================================== + # PUBLISH RESTART: Upload spin-up restart to ect-v{MPAS_VERSION} + # Runs after spinup regardless of ensemble/summary outcome. + # Auto-creates the release (tagged by MPAS version) if it doesn't exist. + #=========================================================================== + publish-restart: + needs: [build, spinup] + name: Publish Restart + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github/ci-config.env + sparse-checkout-cone-mode: false + + - name: Download restart artifact + uses: actions/download-artifact@v7 + with: + name: spinup-restart + + - name: Upload to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + source .github/ci-config.env + VERSION="${{ needs.build.outputs.mpas-version }}" + TAG="ect-v${VERSION}" + + # Create release if it doesn't exist + if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "Creating release ${TAG}..." + gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --title "ECT data (MPAS v${VERSION})" \ + --notes "ECT ensemble summary and spin-up restart for MPAS v${VERSION}. Updated automatically by the ect-ensemble-gen workflow." + fi + + RESTART_NAME="${ECT_RESTART_FILE}" + mv spinup-restart.nc "${RESTART_NAME}" + + echo "Compressing ${RESTART_NAME} ($(du -h "${RESTART_NAME}" | cut -f1))..." + gzip -1 "${RESTART_NAME}" + + gh release upload "${TAG}" "${RESTART_NAME}.gz" \ + --repo "${GITHUB_REPOSITORY}" --clobber + + echo "=== Published restart to release ${TAG} ===" + echo " File: ${RESTART_NAME}.gz ($(du -h "${RESTART_NAME}.gz" | cut -f1))" + + #=========================================================================== + # PUBLISH SUMMARY: Upload ensemble summary to ect-v{MPAS_VERSION} + # Only runs when summary generation succeeds. + # Auto-creates the release (tagged by MPAS version) if it doesn't exist. + #=========================================================================== + publish-summary: + needs: [build, generate-summary] + if: needs.generate-summary.result == 'success' + name: Publish Summary + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github/ci-config.env + sparse-checkout-cone-mode: false + + - name: Download summary artifact + uses: actions/download-artifact@v7 + with: + name: mpas-ect-summary + + - name: Upload to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + source .github/ci-config.env + VERSION="${{ needs.build.outputs.mpas-version }}" + TAG="ect-v${VERSION}" + + # Create release if it doesn't exist + if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "Creating release ${TAG}..." + gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --title "ECT data (MPAS v${VERSION})" \ + --notes "ECT ensemble summary and spin-up restart for MPAS v${VERSION}. Updated automatically by the ect-ensemble-gen workflow." + fi + + SUMMARY_NAME="${{ needs.generate-summary.outputs.summary-name }}" + SUMMARY_PREFIX="${{ needs.generate-summary.outputs.summary-prefix }}" + CURRENT_SUMMARY="${SUMMARY_PREFIX}.nc" + cp "${SUMMARY_NAME}" "${CURRENT_SUMMARY}" + + gh release upload "${TAG}" "${CURRENT_SUMMARY}" \ + --repo "${GITHUB_REPOSITORY}" --clobber + + echo "=== Published summary to release ${TAG} ===" + echo " File: ${CURRENT_SUMMARY}" + + #=========================================================================== + # CLEANUP + # Only runs on a fully successful workflow. On any failure (batch, summary, + # or publish), every artifact is retained until retention-days expires so + # `gh run rerun --failed` can find the executable, spin-up restart, and + # per-batch history files it needs to recover. + #=========================================================================== + cleanup: + needs: [run-ensemble, generate-summary, publish-restart, publish-summary] + if: needs.publish-summary.result == 'success' + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete build and ensemble artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name == "exe-ect-ensemble" or .name == "spinup-restart" or (.name | startswith("ect-ensemble-"))) | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/ect-test.yml b/.github/workflows/ect-test.yml new file mode 100644 index 0000000000..fa0b325f2b --- /dev/null +++ b/.github/workflows/ect-test.yml @@ -0,0 +1,237 @@ +# Ensemble Consistency Test (ECT) +# Runs 3 perturbed MPAS-A simulations on the 120km mesh and compares +# against a pre-built ensemble summary file using PyCECT. +# +# This test detects statistically meaningful changes in model output +# that go beyond internal variability (non-BFB but scientifically equivalent +# changes will pass). Run this on code changes NOT expected to change science. +# +# Reference: Price-Broncucia et al. (2025), doi:10.5194/gmd-18-2349-2025 + +name: Ensemble Consistency Test (ECT) + +on: + push: + branches: [master] + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model). Leave empty to use this repo.' + required: false + default: '' + mpas-ref: + description: 'Git ref (branch, tag, or SHA) in the MPAS source repo' + required: false + default: '' + +jobs: + #=========================================================================== + # CONFIG: Resolve container image from ci-config.env + #=========================================================================== + config: + name: Resolve Config + runs-on: ubuntu-latest + outputs: + image: ${{ steps.container.outputs.image }} + mpas-version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: | + .github + src/core_atmosphere/Registry.xml + sparse-checkout-cone-mode: false + - uses: ./.github/actions/resolve-container + id: container + with: + compiler: gcc + mpi: openmpi + - uses: ./.github/actions/mpas-version + id: version + + #=========================================================================== + # BUILD + #=========================================================================== + build: + needs: config + name: Build MPAS-A + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.mpas-repository || github.repository }} + ref: ${{ inputs.mpas-ref || '' }} + submodules: 'true' + + - uses: actions/checkout@v5 + if: ${{ inputs.mpas-repository != '' }} + with: + path: _ci + sparse-checkout: .github + + - name: Overlay CI infrastructure + if: ${{ inputs.mpas-repository != '' }} + shell: bash + run: cp -r _ci/.github . && rm -rf _ci + + - name: Build MPAS-A (double precision) + uses: ./.github/actions/build-mpas + with: + compiler: gcc + precision: double + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-ect + path: atmosphere_model + retention-days: 1 + + #=========================================================================== + # RUN: 3 perturbed ensemble members + #=========================================================================== + run: + needs: [config, build] + strategy: + fail-fast: false + matrix: + member: [0, 1, 2] + + name: ECT member ${{ matrix.member }} + runs-on: ubuntu-latest + container: + image: ${{ needs.config.outputs.image }} + + steps: + - uses: actions/checkout@v5 + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-ect + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: 120km + dest-dir: base-case + + - name: Restore cached restart + id: cache-restart + uses: actions/cache/restore@v5 + with: + path: spinup-restart.nc + key: ect-spinup-restart-${{ hashFiles('.github/ci-config.env') }} + + - name: Download spin-up restart + id: restart + shell: bash + run: | + source .github/ci-config.env + RESTART="${ECT_RESTART_FILE}" + RELEASE_TAG="ect-v${{ needs.config.outputs.mpas-version }}" + + if [ -f "spinup-restart.nc" ]; then + echo "Using cached restart file" + mv spinup-restart.nc "${RESTART}" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Downloading ${RESTART}.gz from release ${RELEASE_TAG}..." + HTTP_CODE=$(curl -sL --retry 3 --retry-delay 5 -w "%{http_code}" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/${RESTART}.gz" \ + -o "${RESTART}.gz") + if [ "${HTTP_CODE}" = "200" ]; then + gunzip "${RESTART}.gz" + echo "Downloaded restart: $(du -h ${RESTART})" + echo "available=true" >> $GITHUB_OUTPUT + echo "file=${RESTART}" >> $GITHUB_OUTPUT + else + echo "::warning::Spin-up restart not available (HTTP ${HTTP_CODE}), running from cold-start init.nc" + echo "available=false" >> $GITHUB_OUTPUT + fi + + - name: Run perturbed MPAS-A + uses: ./.github/actions/run-perturb-mpas + with: + base-dir: base-case + executable: ./atmosphere_model + member-start: ${{ matrix.member }} + member-end: ${{ matrix.member }} + run-duration: '0_02:36:00' + run-timeout: '45' + restart-file: ${{ steps.restart.outputs.available == 'true' && steps.restart.outputs.file || '' }} + + - name: Upload history file + uses: actions/upload-artifact@v6 + if: always() + with: + name: ect-history-${{ matrix.member }} + path: history-output/history.*.nc + retention-days: 1 + + - name: Upload log files + uses: actions/upload-artifact@v6 + if: always() + with: + name: ect-logs-${{ matrix.member }} + path: history-output/log.*.out + retention-days: 5 + + #=========================================================================== + # VALIDATE: Run PyCECT against ensemble summary + #=========================================================================== + validate: + needs: [config, run] + if: always() + name: PyCECT Validation + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Download history files + uses: actions/download-artifact@v7 + with: + pattern: ect-history-* + path: ect-test-files + merge-multiple: true + + - name: Run ECT validation + uses: ./.github/actions/validate-ect + with: + history-dir: ect-test-files + label: gcc/openmpi/ect-standalone + mpas-version: ${{ needs.config.outputs.mpas-version }} + + #=========================================================================== + # CLEANUP + #=========================================================================== + cleanup: + needs: [run, validate] + if: always() + runs-on: ubuntu-latest + name: Cleanup + + steps: + - name: Delete build artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ + --paginate --jq '.artifacts[] | select(.name == "exe-ect") | .id' \ + | while read id; do + gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/$id || true + done || true diff --git a/.github/workflows/profile-gpu-nsight.yml b/.github/workflows/profile-gpu-nsight.yml new file mode 100644 index 0000000000..92ed3b7969 --- /dev/null +++ b/.github/workflows/profile-gpu-nsight.yml @@ -0,0 +1,194 @@ +# Nsight Systems GPU profiling on CIRRUS (keep to very short runs) + +name: GPU Nsight (profile) + +permissions: + contents: read + actions: write + +on: + workflow_dispatch: + inputs: + mpi: + description: MPI implementation + required: true + default: mpich + type: choice + options: + - mpich + - openmpi + resolution: + description: Test case resolution (see RELEASE_TESTDATA_* in ci-config.env) + required: false + default: '240km' + type: string + num_procs: + description: MPI rank count + required: false + default: '4' + type: string + run_duration: + description: config_run_duration + required: false + default: '0_01:00:00' + type: string + run_timeout_minutes: + description: Wall-clock timeout for mpirun + nsys + required: false + default: '40' + type: string + +jobs: + config: + name: Resolve CUDA container + runs-on: ubuntu-latest + outputs: + image-gpu: ${{ steps.gpu-container.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + sparse-checkout: .github + - uses: ./.github/actions/resolve-container + id: gpu-container + with: + compiler: nvhpc + mpi: ${{ inputs.mpi }} + gpu: cuda + + build: + needs: config + name: Build (cuda, ${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + with: + submodules: 'true' + + - name: Build MPAS-A (double precision, OpenACC) + uses: ./.github/actions/build-mpas + with: + compiler: nvhpc + use-pio: 'false' + openacc: 'true' + precision: double + build-timeout: '45' + + - name: Upload executable + uses: actions/upload-artifact@v6 + with: + name: exe-profile-nsight-${{ inputs.mpi }} + path: atmosphere_model + retention-days: 1 + + profile: + needs: [config, build] + if: ${{ needs.build.result == 'success' }} + name: Nsight profile (${{ inputs.mpi }}) + runs-on: + group: CIRRUS-4x8-gpu + container: + image: ${{ needs.config.outputs.image-gpu }} + + steps: + - uses: actions/checkout@v5 + + - name: GPU check + shell: bash + run: | + echo "## GPU" >> "$GITHUB_STEP_SUMMARY" + nvidia-smi || echo "::warning::nvidia-smi failed" + nvidia-smi || true + + - name: Setup Nsight Systems CLI (install + RPM cache) + uses: ./.github/actions/setup-nsight-systems + + - name: Download executable + uses: actions/download-artifact@v7 + with: + name: exe-profile-nsight-${{ inputs.mpi }} + + - name: Download test case + uses: ./.github/actions/download-testdata + with: + resolution: ${{ inputs.resolution }} + dest-dir: nsight-case + + - name: Link executable and config_run_duration (dt unchanged) + shell: bash + run: | + chmod +x atmosphere_model + ln -sf "$(pwd)/atmosphere_model" nsight-case/atmosphere_model + DURATION="${{ inputs.run_duration }}" + sed -i "s/config_run_duration = '[^']*'/config_run_duration = '${DURATION}'/" nsight-case/namelist.atmosphere + echo "config_run_duration -> ${DURATION} (config_dt left as in test case)" + grep -E 'config_dt|config_run_duration' nsight-case/namelist.atmosphere | head -n 5 || true + + - name: Run nsys profile + shell: bash + run: | + chmod +x .github/scripts/run-nsys-profile.sh + bash .github/scripts/run-nsys-profile.sh \ + nsight-case \ + "${{ inputs.num_procs }}" \ + "${{ inputs.mpi }}" \ + "${{ inputs.run_timeout_minutes }}" \ + nsys-profile + + - name: nsys stats (text) + shell: bash + working-directory: nsight-case + run: | + if [ -f /container/config_env.sh ]; then + source /container/config_env.sh + fi + source "${GITHUB_WORKSPACE}/.github/scripts/resolve-nsys.sh" + if ! resolve_nsys; then + echo "::warning::No working nsys; skipping nsys stats generation" + exit 0 + fi + echo "Using nsys: ${NSYS_BIN}" + shopt -s nullglob + REP="" + for f in nsys-profile.nsys-rep *.nsys-rep *.qdrep; do + if [ -f "$f" ]; then + REP="$f" + break + fi + done + if [ -z "${REP}" ]; then + echo "::warning::No Nsight session file found for nsys stats" + ls -la + exit 0 + fi + echo "=== nsys stats (full log → nsys-profile-stats.txt; first 200 lines below) ===" + "${NSYS_BIN}" stats "${REP}" > "nsys-profile-stats.txt" 2>&1 || true + head -n 200 nsys-profile-stats.txt + + - name: Job summary + if: always() + shell: bash + run: | + { + echo "## Nsight profile" + echo "- MPI: ${{ inputs.mpi }}" + echo "- Resolution: ${{ inputs.resolution }}" + echo "- Ranks: ${{ inputs.num_procs }}" + echo "- config_run_duration: ${{ inputs.run_duration }}" + echo "" + echo "Session files and \`nsys stats\` text are uploaded as artifacts (short retention)." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Nsight artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: nsys-profile-${{ github.run_id }}-${{ inputs.mpi }} + path: | + nsight-case/nsys-profile.* + nsight-case/nsys-profile-stats.txt + retention-days: 3 + if-no-files-found: warn diff --git a/.github/workflows/test-cross-repo.yml b/.github/workflows/test-cross-repo.yml new file mode 100644 index 0000000000..7cc5e5ab32 --- /dev/null +++ b/.github/workflows/test-cross-repo.yml @@ -0,0 +1,80 @@ +# Cross-repo testing: run CI against an external MPAS source repo. +# Dispatch-only — does not affect badge status on master. + +name: "Cross-Repo Test" + +on: + workflow_dispatch: + inputs: + mpas-repository: + description: 'MPAS source repo (e.g. MPAS-Dev/MPAS-Model)' + required: true + mpas-ref: + description: 'Git ref (branch, tag, or SHA)' + required: true + compilers: + description: 'Compilers to test (comma-separated: gcc,nvhpc,oneapi)' + required: false + default: 'gcc,nvhpc,oneapi' + mpi: + description: 'MPI implementation' + required: false + default: 'mpich' + # GPU cross-repo testing disabled until NCAR security review of running + # external code on self-hosted CIRRUS runners is complete. + # include-gpu: + # description: 'Include GPU (CUDA) test' + # required: false + # type: boolean + # default: false + +jobs: + config: + runs-on: ubuntu-latest + outputs: + compilers: ${{ steps.set.outputs.compilers }} + mpi: ${{ steps.set.outputs.mpi }} + repo: ${{ steps.set.outputs.repo }} + ref: ${{ steps.set.outputs.ref }} + steps: + - id: set + shell: bash + run: | + COMPILERS="${{ inputs.compilers }}" + REPO="${{ inputs.mpas-repository || 'MPAS-Dev/MPAS-Model' }}" + REF="${{ inputs.mpas-ref || 'develop' }}" + MPI="${{ inputs.mpi || 'mpich' }}" + + echo "compilers=$(printf '%s' "${COMPILERS:-gcc}" | jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$";""))')" >> "$GITHUB_OUTPUT" + echo "mpi=${MPI}" >> "$GITHUB_OUTPUT" + echo "repo=${REPO}" >> "$GITHUB_OUTPUT" + echo "ref=${REF}" >> "$GITHUB_OUTPUT" + + echo "## Cross-Repo Test" >> "$GITHUB_STEP_SUMMARY" + echo "| | |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---|" >> "$GITHUB_STEP_SUMMARY" + echo "| **Source** | [\`${REPO}@${REF}\`](https://github.com/${REPO}/tree/${REF}) |" >> "$GITHUB_STEP_SUMMARY" + echo "| **Compilers** | \`${COMPILERS:-gcc}\` |" >> "$GITHUB_STEP_SUMMARY" + echo "| **MPI** | \`${MPI}\` |" >> "$GITHUB_STEP_SUMMARY" + cpu-matrix: + needs: config + strategy: + fail-fast: false + matrix: + compiler: ${{ fromJSON(needs.config.outputs.compilers) }} + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: ${{ matrix.compiler }} + mpi: ${{ needs.config.outputs.mpi }} + mpas-repository: ${{ needs.config.outputs.repo }} + mpas-ref: ${{ needs.config.outputs.ref }} + + # GPU cross-repo testing disabled until NCAR security review is complete. + # gpu: + # needs: config + # if: ${{ inputs.include-gpu }} + # uses: ./.github/workflows/_test-gpu.yml + # with: + # mpi: ${{ needs.config.outputs.mpi }} + # mpas-repository: ${{ needs.config.outputs.repo }} + # mpas-ref: ${{ needs.config.outputs.ref }} diff --git a/.github/workflows/test-gcc-mpich.yml b/.github/workflows/test-gcc-mpich.yml new file mode 100644 index 0000000000..fcefd8db1e --- /dev/null +++ b/.github/workflows/test-gcc-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: GNU + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "GNU+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: gcc + mpi: mpich + diff --git a/.github/workflows/test-gcc-openmpi.yml b/.github/workflows/test-gcc-openmpi.yml new file mode 100644 index 0000000000..ca7965a80e --- /dev/null +++ b/.github/workflows/test-gcc-openmpi.yml @@ -0,0 +1,15 @@ +# Subset CI: GNU + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "GNU+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: gcc + mpi: openmpi + diff --git a/.github/workflows/test-gpu-mpich.yml b/.github/workflows/test-gpu-mpich.yml new file mode 100644 index 0000000000..efd1097fa6 --- /dev/null +++ b/.github/workflows/test-gpu-mpich.yml @@ -0,0 +1,15 @@ +# Subset CI: NVHPC + MPICH (GPU vs CPU) +# Runs on CIRRUS self-hosted runners with GPU access. +# workflow_dispatch only — no push/pull_request (self-hosted security). + +name: "NVHPC+MPICH (GPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-gpu.yml + with: + mpi: mpich + diff --git a/.github/workflows/test-gpu-openmpi.yml b/.github/workflows/test-gpu-openmpi.yml new file mode 100644 index 0000000000..fff0259dcf --- /dev/null +++ b/.github/workflows/test-gpu-openmpi.yml @@ -0,0 +1,16 @@ +# Subset CI: NVHPC + OpenMPI (GPU vs CPU) +# Runs on CIRRUS self-hosted runners with GPU access. +# Known issue: NVHPC+OpenMPI SIGABRT. +# workflow_dispatch only — no push/pull_request (self-hosted security). + +name: "NVHPC+OpenMPI (GPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-gpu.yml + with: + mpi: openmpi + diff --git a/.github/workflows/test-intel-mpich.yml b/.github/workflows/test-intel-mpich.yml new file mode 100644 index 0000000000..a73195efd3 --- /dev/null +++ b/.github/workflows/test-intel-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: Intel + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "Intel+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: oneapi + mpi: mpich + diff --git a/.github/workflows/test-intel-openmpi.yml b/.github/workflows/test-intel-openmpi.yml new file mode 100644 index 0000000000..3050baf238 --- /dev/null +++ b/.github/workflows/test-intel-openmpi.yml @@ -0,0 +1,15 @@ +# Subset CI: Intel + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "Intel+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: oneapi + mpi: openmpi + diff --git a/.github/workflows/test-nvhpc-mpich.yml b/.github/workflows/test-nvhpc-mpich.yml new file mode 100644 index 0000000000..ac898e9822 --- /dev/null +++ b/.github/workflows/test-nvhpc-mpich.yml @@ -0,0 +1,19 @@ +# Subset CI: NVHPC + MPICH (CPU) +# Quick ECT validation with 4 MPI ranks. + +name: "NVHPC+MPICH (CPU)" + +on: + workflow_dispatch: + push: + branches: [master, 'hackathon-*'] + pull_request: + branches: [master, 'hackathon-*'] + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: nvhpc + mpi: mpich + diff --git a/.github/workflows/test-nvhpc-openmpi.yml b/.github/workflows/test-nvhpc-openmpi.yml new file mode 100644 index 0000000000..31297a0ede --- /dev/null +++ b/.github/workflows/test-nvhpc-openmpi.yml @@ -0,0 +1,17 @@ +# Subset CI: NVHPC + OpenMPI (CPU) +# Quick ECT validation with 4 MPI ranks. +# Known issue: model exits 134 (SIGABRT) on GA runners with 4 ranks. +# MPICH works. + +name: "NVHPC+OpenMPI (CPU)" + +on: + workflow_dispatch: + +jobs: + test: + uses: ./.github/workflows/_test-compiler.yml + with: + compiler: nvhpc + mpi: openmpi + diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..10c2dc8426 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +# Unit Tests +# Builds and runs pFUnit-based unit tests for standalone MPAS procedures. +# Tests mathematical/computational routines that don't require the full +# MPAS framework (interpolation, geometry, sorting, etc.). +# +# Adapted from ESCOMP/CAM-SIMA MPAS dynamical core CI. + +name: Unit Tests + +permissions: + contents: read + +on: + pull_request: + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/unit-tests.yml' + push: + branches: [master] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/unit-tests.yml' + workflow_dispatch: + +jobs: + unit-tests: + name: pFUnit (GCC ${{ matrix.gcc-version }}) + timeout-minutes: 15 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + gcc-version: ['12', '13', '14'] + + env: + CC: gcc-${{ matrix.gcc-version }} + CXX: g++-${{ matrix.gcc-version }} + FC: gfortran-${{ matrix.gcc-version }} + PFUNIT_VERSION: 'v4.13.0' + PFUNIT_PATH: ${{ github.workspace }}/pFUnit + + steps: + - name: Install GCC toolchain + run: | + sudo apt-get update + sudo apt-get install -y gcc-${{ matrix.gcc-version }} g++-${{ matrix.gcc-version }} gfortran-${{ matrix.gcc-version }} + + - name: Checkout + uses: actions/checkout@v5 + + - name: Cache pFUnit + id: cache-pfunit + uses: actions/cache@v5 + with: + key: pfunit-${{ env.PFUNIT_VERSION }}-gcc-${{ matrix.gcc-version }} + path: ${{ env.PFUNIT_PATH }}/install + + - name: Build pFUnit + if: steps.cache-pfunit.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch ${PFUNIT_VERSION} \ + https://github.com/Goddard-Fortran-Ecosystem/pFUnit.git ${PFUNIT_PATH} + + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${PFUNIT_PATH}/install \ + -B ${PFUNIT_PATH}/build \ + -S ${PFUNIT_PATH} + + cmake --build ${PFUNIT_PATH}/build --parallel $(nproc) + cmake --install ${PFUNIT_PATH}/build + + - name: Build tests + run: | + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_PREFIX_PATH=${PFUNIT_PATH}/install \ + -B tests/build \ + -S tests + + cmake --build tests/build --parallel $(nproc) + + - name: Run tests + run: | + ctest \ + --build-config Debug \ + --output-on-failure \ + --test-dir tests/build \ + --verbose \ + --output-log tests/build/unit-tests.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v6 + with: + name: unit-tests-log-gcc-${{ matrix.gcc-version }} + path: tests/build/unit-tests.log + retention-days: 14 + if-no-files-found: ignore diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000..3e8f2a9fee --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.21) + +project(mpas_unit_tests + VERSION 0.1.0 + DESCRIPTION "Unit tests for standalone MPAS-Model procedures" + LANGUAGES Fortran +) + +# Build a small library from MPAS source files that have testable, +# standalone procedures with minimal dependencies. +# Add source files here as more tests are developed. +set(MPAS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../src) + +add_library(mpas_testable_procedures) +target_sources(mpas_testable_procedures + PRIVATE + ${MPAS_SRC}/framework/mpas_kind_types.F + ${MPAS_SRC}/operators/mpas_spline_interpolation.F +) +target_include_directories(mpas_testable_procedures + INTERFACE ${CMAKE_CURRENT_BINARY_DIR} +) + +# MPAS .F files are free-form Fortran with C preprocessor directives. +# gfortran treats .F as fixed-form by default, so we must override. +target_compile_options(mpas_testable_procedures + PRIVATE + $<$:-cpp -ffree-form -fbacktrace -ffree-line-length-none> + $<$:-fpp -free -traceback> + $<$:-fpp -free -traceback> +) + +if(PROJECT_IS_TOP_LEVEL) + include(CTest) + enable_testing() +endif() + +if(BUILD_TESTING) + add_subdirectory(unit) +endif() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt new file mode 100644 index 0000000000..68777daffb --- /dev/null +++ b/tests/unit/CMakeLists.txt @@ -0,0 +1,13 @@ +find_package(PFUNIT REQUIRED) + +# Spline interpolation tests +add_pfunit_ctest(test_spline_interpolation + TEST_SOURCES test_spline_interpolation.pf + LINK_LIBRARIES mpas_testable_procedures +) +target_compile_options(test_spline_interpolation + PRIVATE + $<$:-fbacktrace -ffree-line-length-none> + $<$:-traceback> + $<$:-traceback> +) diff --git a/tests/unit/test_spline_interpolation.pf b/tests/unit/test_spline_interpolation.pf new file mode 100644 index 0000000000..ae61d26479 --- /dev/null +++ b/tests/unit/test_spline_interpolation.pf @@ -0,0 +1,200 @@ +!----------------------------------------------------------------------- +! Unit tests for mpas_spline_interpolation module +! +! Tests cubic spline and linear interpolation routines against +! known analytic functions. Adapted from the testing patterns in +! ESCOMP/CAM-SIMA MPAS dynamical core CI. +!----------------------------------------------------------------------- + +module test_spline_interpolation + + use funit + use mpas_kind_types, only: RKIND + use mpas_spline_interpolation, only: & + mpas_cubic_spline_coefficients, & + mpas_interpolate_cubic_spline, & + mpas_integrate_cubic_spline, & + mpas_interpolate_linear + + implicit none + + real(kind=RKIND), parameter :: tol = 1.0e-10_RKIND + +contains + + !--------------------------------------------------------------------- + ! Linear interpolation tests + !--------------------------------------------------------------------- + + @test + subroutine test_linear_interp_identity() + ! Interpolating at the same nodes should return the original values. + ! mpas_interpolate_linear uses strict < on x(kIn+1), so it cannot + ! interpolate at x(n). We test n-1 interior+left-boundary nodes. + integer, parameter :: n = 5 + real(kind=RKIND) :: x(n), y(n), xOut(n-1), yOut(n-1) + integer :: i + + x = [1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + y = [10.0_RKIND, 20.0_RKIND, 30.0_RKIND, 40.0_RKIND, 50.0_RKIND] + xOut = x(1:n-1) + + call mpas_interpolate_linear(x, y, n, xOut, yOut, n-1) + + do i = 1, n-1 + @assertEqual(y(i), yOut(i), tol) + end do + end subroutine test_linear_interp_identity + + @test + subroutine test_linear_interp_midpoints() + ! Midpoint interpolation of a linear function should be exact + integer, parameter :: n = 3, nOut = 2 + real(kind=RKIND) :: x(n), y(n), xOut(nOut), yOut(nOut) + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND] + y = [0.0_RKIND, 2.0_RKIND, 4.0_RKIND] + xOut = [0.5_RKIND, 1.5_RKIND] + + call mpas_interpolate_linear(x, y, n, xOut, yOut, nOut) + + @assertEqual(1.0_RKIND, yOut(1), tol) + @assertEqual(3.0_RKIND, yOut(2), tol) + end subroutine test_linear_interp_midpoints + + @test + subroutine test_linear_interp_quadratic_function() + ! Linear interpolation of y = x^2 at midpoints + integer, parameter :: n = 5, nOut = 4 + real(kind=RKIND) :: x(n), y(n), xOut(nOut), yOut(nOut) + real(kind=RKIND) :: expected + integer :: i + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND] + y = [0.0_RKIND, 1.0_RKIND, 4.0_RKIND, 9.0_RKIND, 16.0_RKIND] + xOut = [0.5_RKIND, 1.5_RKIND, 2.5_RKIND, 3.5_RKIND] + + call mpas_interpolate_linear(x, y, n, xOut, yOut, nOut) + + ! Linear interpolation of x^2 at midpoints: (x_i^2 + x_{i+1}^2) / 2 + do i = 1, nOut + expected = (x(i)**2 + x(i+1)**2) / 2.0_RKIND + @assertEqual(expected, yOut(i), tol) + end do + end subroutine test_linear_interp_quadratic_function + + !--------------------------------------------------------------------- + ! Cubic spline tests + !--------------------------------------------------------------------- + + @test + subroutine test_cubic_spline_linear_function() + ! Cubic spline of a linear function should have zero second derivatives + integer, parameter :: n = 5 + real(kind=RKIND) :: x(n), y(n), y2(n) + + x = [1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + y = [2.0_RKIND, 4.0_RKIND, 6.0_RKIND, 8.0_RKIND, 10.0_RKIND] + + call mpas_cubic_spline_coefficients(x, y, n, y2) + + @assertEqual(0.0_RKIND, y2(1), tol) + @assertEqual(0.0_RKIND, y2(n), tol) + end subroutine test_cubic_spline_linear_function + + @test + subroutine test_cubic_spline_reproduces_nodes() + ! Interpolating at the original nodes should return the original values. + ! mpas_interpolate_cubic_spline uses strict < on x(kIn+1), so it cannot + ! interpolate at x(n). We test n-1 interior+left-boundary nodes. + integer, parameter :: n = 6 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: xOut(n-1), yOut(n-1) + integer :: i + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND, 4.0_RKIND, 5.0_RKIND] + do i = 1, n + y(i) = sin(x(i)) + end do + xOut = x(1:n-1) + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_interpolate_cubic_spline(x, y, y2, n, xOut, yOut, n-1) + + do i = 1, n-1 + @assertEqual(y(i), yOut(i), tol) + end do + end subroutine test_cubic_spline_reproduces_nodes + + @test + subroutine test_cubic_spline_accuracy_sine() + ! Cubic spline of sin(x) should be accurate to ~O(h^4) + integer, parameter :: n = 11, nOut = 5 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: xOut(nOut), yOut(nOut) + real(kind=RKIND) :: h, maxErr + integer :: i + + ! Nodes from 0 to pi with h = pi/10 + h = acos(-1.0_RKIND) / real(n - 1, RKIND) + do i = 1, n + x(i) = real(i - 1, RKIND) * h + y(i) = sin(x(i)) + end do + + ! Test at midpoints between nodes 3-7 (away from boundaries) + do i = 1, nOut + xOut(i) = x(i + 3) + 0.5_RKIND * h + end do + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_interpolate_cubic_spline(x, y, y2, n, xOut, yOut, nOut) + + maxErr = 0.0_RKIND + do i = 1, nOut + maxErr = max(maxErr, abs(yOut(i) - sin(xOut(i)))) + end do + + ! With h ~ 0.31, cubic spline error should be well under 1e-4 + @assertTrue(maxErr < 1.0e-4_RKIND, 'Cubic spline of sin(x) error too large') + end subroutine test_cubic_spline_accuracy_sine + + @test + subroutine test_cubic_spline_integrate_linear() + ! Integral of a linear function f(x) = 2x from 0 to 3 should be 9 + integer, parameter :: n = 4 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: integral + + x = [0.0_RKIND, 1.0_RKIND, 2.0_RKIND, 3.0_RKIND] + y = [0.0_RKIND, 2.0_RKIND, 4.0_RKIND, 6.0_RKIND] + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_integrate_cubic_spline(x, y, y2, n, 0.0_RKIND, 3.0_RKIND, integral) + + @assertEqual(9.0_RKIND, integral, tol) + end subroutine test_cubic_spline_integrate_linear + + @test + subroutine test_cubic_spline_integrate_sine() + ! Integral of sin(x) from 0 to pi should be 2.0 + integer, parameter :: n = 21 + real(kind=RKIND) :: x(n), y(n), y2(n) + real(kind=RKIND) :: integral, pi_val + integer :: i + + pi_val = acos(-1.0_RKIND) + do i = 1, n + x(i) = real(i - 1, RKIND) * pi_val / real(n - 1, RKIND) + y(i) = sin(x(i)) + end do + + call mpas_cubic_spline_coefficients(x, y, n, y2) + call mpas_integrate_cubic_spline(x, y, y2, n, 0.0_RKIND, pi_val, integral) + + ! Analytic integral of sin(x) from 0 to pi = 2.0 + ! With 21 nodes (h~0.157), spline integration error is ~1.7e-6 + @assertEqual(2.0_RKIND, integral, 1.0e-5_RKIND) + end subroutine test_cubic_spline_integrate_sine + +end module test_spline_interpolation