Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import os
import sys
import traceback
import typing

import numpy as np
from mpi4py import MPI
Expand All @@ -19,6 +20,13 @@ def in_exception_context() -> bool:
return False


def cast_err_msg_to_str(err_msg: str | typing.Any):
"""If err_msg is not a string, cast it to string and add a note flagging the type provided."""
if not isinstance(err_msg, str):
err_msg = f"Expected instance of str for err_msg, got type: {type(err_msg)}: {err_msg}"
return err_msg


def err_out_screen(err_msg: str, exc: BaseException | None = None):
"""Print an error message to the screen and exit the program gracefully.

Expand All @@ -31,6 +39,7 @@ def err_out_screen(err_msg: str, exc: BaseException | None = None):

Logan Karsten - National Center for Atmospheric Research, karsten@ucar.edu
"""
err_msg = cast_err_msg_to_str(err_msg)
if exc is not None:
err_msg += f" - {exc}"
err_msg_out = "ERROR: " + err_msg
Expand Down Expand Up @@ -65,6 +74,7 @@ def err_out_screen_para(err_msg: str, MpiConfig, exc: BaseException | None = Non
:param exc: Optional exception object to append to the error message.
:return: None
"""
err_msg = cast_err_msg_to_str(err_msg)
if exc is not None:
err_msg += f" - {exc}"
err_msg_out = f"ERROR: RANK - {MpiConfig.rank} : {err_msg}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ def __init__(self, config_options: ConfigOptions):
self.log_debug = partial(err_handler.log_msg, self.config_options, self, True)
self.log_info = partial(err_handler.log_msg, self.config_options, self, False)
self.log_warning = partial(err_handler.log_warning, self.config_options, self)

self.original_excepthook = None # Will be set to the original sys.excepthook when __register_exit_handlers is called
self.__register_exit_handlers()

def initialize_comm(self, comm=None):
Expand Down Expand Up @@ -145,7 +147,8 @@ def __register_exit_handlers(self) -> None:
against potential deadlock conditions to be sure (would need to confirm that a non-0 rank initiating an abort would cause
rank 0 break out of a collective call if it happens to be waiting at one)."""
# Exceptions
sys.exepthook = self.__excepthook
self.original_excepthook = sys.excepthook
sys.excepthook = self.__excepthook
# Regular exits
atexit.register(self._cleanup)
# Signals
Expand All @@ -154,13 +157,23 @@ def __register_exit_handlers(self) -> None:

def __excepthook(self, ex_type, value, tb) -> None:
"""Custom excepthook which follows these steps:
1. Call Python's built-in excepthook.
1. Call original excepthook which had been saved earlier.
2. Log .errMsg as CRITICAL (unless it is None).
3. Cleanup.
4. MPI Abort.

To apply, set `sys.excepthook` to this method."""
sys.__excepthook__(ex_type, value, tb)
To apply:
Set `self.original_excepthook = sys.excepthook`,
Then set `sys.excepthook = {this method}`.
"""
# Call original excepthook stored earlier (not raw sys.__excepthook__)
if self.original_excepthook is None:
raise RuntimeError(
"In custom __excepthook, but self.original_excepthook does not contain the saved original hook as expected."
)
self.original_excepthook(ex_type, value, tb)

# Perform additional actions
if self.config_options.errMsg is not None:
err_handler.log_critical(
self.config_options,
Expand Down