Ms endorse#195
Conversation
| res = np.array(res) | ||
| fine_coarse_res = res[:, 1] | ||
|
|
||
| result_type = np.dtype((np.float, np.array(fine_coarse_res[0]).shape)) |
There was a problem hiding this comment.
Reason: float seems to be incompatible with new Numpy versions
| for i_level in range(len(self._hdf_object.level_parameters)): | ||
| self._level_groups.append(self._hdf_object.add_level_group(str(i_level))) | ||
|
|
||
| def _hdf_result_format(self, locations, times): |
There was a problem hiding this comment.
Simplified, moved into hdf5.py as it is implementation detail how the format is stored.
There was a problem hiding this comment.
Moereover the format saving was wrong since it same format has been assumed for all dict elements.
| Load result format | ||
| """ | ||
| results_format = self._hdf_object.load_result_format() | ||
| quantities = [] |
There was a problem hiding this comment.
Same thing, but using constructing function with named parameters.
| for level, samples in successful_samples.items(): | ||
| if len(samples) > 0: | ||
| self._level_groups[level].append_successful(np.array(samples)) | ||
| ids, sample_values = zip(*samples) |
There was a problem hiding this comment.
need to make pairs here and convert to numpy arrays just before storage.
| except Exception: | ||
| str_list = traceback.format_exception(*sys.exc_info()) | ||
| err_msg = "".join(str_list) | ||
| print(err_msg) |
There was a problem hiding this comment.
TODO: replace by suitable logging? Only in case of debugging turned on.
Problem is that the exceptions may not be propagated to the Sampler if pool somehow fail. So it would be great to log the exception within the sampling job process.
There was a problem hiding this comment.
We do not have support for worker job logging. So keep it.
| if not os.path.isdir(sample_dir): | ||
| os.makedirs(sample_dir, mode=0o775, exist_ok=True) | ||
|
|
||
| # We have observed possible problems with directory creation on the |
There was a problem hiding this comment.
I guessed that there is some problem with NFS, however in fact there was probably problem with various threads writing to the same directory, randomly cleaning it after it was created.
I suggest to keep the code commented, it allows to better check that we are able to write into the directory. There could be problem with rights anyway.
| self._save_running_time(level_sim._level_id, running_time) | ||
|
|
||
| if not err_msg: | ||
| self._queues.setdefault(level_sim._level_id, queue.Queue()).put((sample_id, (result[0], result[1]))) |
There was a problem hiding this comment.
just to improve readibility
| Execute pbs script | ||
| :return: None | ||
| """ | ||
| if len(self._scheduled) > 0: |
There was a problem hiding this comment.
Decrease indention level
| pbs_process.write_pbs_id(pbs_id) | ||
|
|
||
| process = subprocess.run(['qsub', job_file], stderr=subprocess.PIPE, stdout=subprocess.PIPE) | ||
| try: |
There was a problem hiding this comment.
I quite didn't get reason for this try block. What is reason for PIPEs? I propose to just let subproces.run finish and take resulting stdout and stderr.
| import time | ||
| import logging | ||
|
|
||
| class FileSafe(h5py.File): |
There was a problem hiding this comment.
Context manager to open the file with some timeout.
- yet need to check few places marked with TODO
| @@ -33,36 +33,14 @@ def __init__(self, file_path): | |||
| for i_level in range(len(self._hdf_object.level_parameters)): | |||
There was a problem hiding this comment.
Changes in StorageHDF need to be made in StorageMemory as well.
| return "result_format" | ||
|
|
||
| def save_result_format(self, result_format, res_dtype): | ||
| def single_format(self, spec:"QuantitySpec"): |
There was a problem hiding this comment.
Moved from sample_storage_hdf5 as QunatitySpec serialization.
|
|
||
| # Ensure dataset exists (resizable) | ||
| with h5py.File(self.file_name, 'a') as hdf_file: | ||
| # format item in main group |
There was a problem hiding this comment.
Switch to hdf5 group in order to support heterogeneous items in the result.
| moms = self.eval_moments(value) | ||
| power = -np.sum(moms * self.multipliers / self._moment_errs, axis=1) | ||
| power = np.minimum(np.maximum(power, -200), 200) | ||
| return np.exp(power) |
There was a problem hiding this comment.
Fix of error caused by more strict new Numpy versions that disallow automatic conversion of (1,)shape to scalar.
| except Exception: | ||
| str_list = traceback.format_exception(*sys.exc_info()) | ||
| err_msg = "".join(str_list) | ||
| print(err_msg) |
There was a problem hiding this comment.
We do not have support for worker job logging. So keep it.
| # commands; qsub failures are handled by returncode below, while command | ||
| # execution errors still propagate as environment errors. | ||
| # try: | ||
| if process.returncode != 0: |
There was a problem hiding this comment.
Try block removed.
PbsCommands returns a decoded CommandOutput for completed
commands; qsub failures are handled by returncode below, while the command
execution errors still propagate as environment errors.
2 participants
Various changes made during application to the Endorse software.