From 5ffa2b4a6eda63ad2b1ba666662956680f6caa9f Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sat, 12 Sep 2026 11:39:41 +0200 Subject: [PATCH 01/72] ignore output logs and inference tomls used for testing --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 2aedede6b..7fdfa33a1 100644 --- a/.gitignore +++ b/.gitignore @@ -335,3 +335,10 @@ src/proteus/_version.py # Local scratch directories (not part of the project) /.playwright-mcp/ /platon/ + +# output logs +*.out + +# test inference tomls +input/test_infer_proteus/ +input/inference/test_infer From ccb4d337a060e50e0f99db4a3a75e99ea55c48ba Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 12:12:39 +0200 Subject: [PATCH 02/72] validate proteus config in inference runs --- src/proteus/inference/inference.py | 153 +++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 10 deletions(-) diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index 7f0c5c017..8af4710d7 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -7,7 +7,9 @@ from __future__ import annotations # system libraries +import copy import logging +import math import os import shutil import time @@ -17,13 +19,25 @@ import proteus.inference.plot as plotBO +# proteus libraries +from proteus.config import ( + UnknownConfigKeyError, + find_key_problems, + format_orphan_message, + read_config, + structure_config, +) + # bayesopt source files from proteus.inference.async_BO import checkpoint, parallel_process from proteus.inference.gen_D_init import create_init -from proteus.inference.objective import prot_builder, set_child_timeout +from proteus.inference.objective import ( + WORKER_CONFIG_OVERRIDES, + apply_nested_updates, + prot_builder, + set_child_timeout, +) from proteus.inference.utils import print_results, str_time - -# proteus libraries from proteus.utils.coupler import get_proteus_directories from proteus.utils.helper import safe_rm from proteus.utils.logs import setup_logger @@ -33,6 +47,120 @@ log = logging.getLogger('fwl.' + __name__) +# Stand-in for the per-run output folder when validating. Only the shape of the +# value matters here; the real path carries the worker and iteration indices. +_VALIDATION_OUT_PATH = 'workers/w_0/i_0' + + +def _reject_bad_config(raw: dict, label: str) -> None: + """Apply the PROTEUS config checks to a raw dict, naming its source in errors. + + Parameters + ---------- + - raw (dict): Raw TOML dict to check against the PROTEUS config schema. + - label (str): Source description quoted back in any error message. + + Returns + ---------- + - None + + Raises: + UnknownConfigKeyError: If the dict carries keys outside the schema. + ValueError: If a value fails schema validation. + """ + orphans, mistyped = find_key_problems(raw) + if orphans or mistyped: + raise UnknownConfigKeyError(format_orphan_message(orphans, label, mistyped)) + structure_config(raw, label) + + +def parameter_bounds(parameters: dict) -> dict[str, tuple[float, float]]: + """Return the swept-parameter ranges as ordered float pairs. + + Parameters + ---------- + - parameters (dict): Mapping of dot-separated config keys to [min, max]. + + Returns + ---------- + - dict[str, tuple[float, float]]: Same keys, bounds as (min, max) floats. + + Raises: + ValueError: If a range is not a pair of numbers, or does not increase. + """ + bounds: dict[str, tuple[float, float]] = {} + for key, value in parameters.items(): + numeric = ( + isinstance(value, (list, tuple)) + and len(value) == 2 + and all(isinstance(v, (int, float)) for v in value) + ) + if not numeric: + raise ValueError( + f"Bounds for inference parameter '{key}' must be a pair of numbers " + f'[min, max], got {value!r}' + ) + low, high = float(value[0]), float(value[1]) + # TOML admits `inf` and `nan`. An infinite bound passes the schema's + # own range checks and then makes every unnormalised sample infinite. + if not (math.isfinite(low) and math.isfinite(high)): + raise ValueError( + f"Bounds for inference parameter '{key}' must be finite, got {value!r}" + ) + if low >= high: + raise ValueError( + f"Bounds for inference parameter '{key}' must increase, got [{low:g}, {high:g}]" + ) + bounds[key] = (low, high) + return bounds + + +def validate_reference_config(ref_config: str, parameters: dict) -> None: + """Reject a reference config the workers could not run, before any run starts. + + The file is checked exactly as PROTEUS checks its own input, and then again + with every swept parameter set to each end of its range. A mistyped + parameter name shows up as an unrecognised key, and a bound outside what + the schema accepts shows up as a validation failure, both reported here + rather than as a worker crash part-way through the study. + + Only the two ends of the range are checked, with every parameter moved + together, so this is a screen rather than a proof. A range whose interior + holds an invalid combination still passes. Conversely, a schema rule that + couples two swept parameters can make one of the two variants invalid even + though most of the space is fine: sweeping both `params.dt.minimum` and + `params.dt.maximum` can put the minimum above the maximum at one end, and + the study is refused. Sweep one side of such a pair, or widen the other. + + Parameters + ---------- + - ref_config (str): Path to the reference PROTEUS config file. + - parameters (dict): Mapping of dot-separated config keys to [min, max]. + + Returns + ---------- + - None + + Raises: + UnknownConfigKeyError: If any variant carries keys outside the schema. + ValueError: If a bound is malformed, or a variant fails validation. + """ + bounds = parameter_bounds(parameters) + raw = read_config(ref_config) + + # The file as the user wrote it. + _reject_bad_config(raw, str(ref_config)) + + # The file as a worker will run it. Bounds are cast to float to match what + # the optimiser writes back into each worker's config. + for label, index in (('lower', 0), ('upper', 1)): + updates = {key: pair[index] for key, pair in bounds.items()} + updates.update(WORKER_CONFIG_OVERRIDES) + updates['params.out.path'] = _VALIDATION_OUT_PATH + candidate = apply_nested_updates(copy.deepcopy(raw), updates) + _reject_bad_config(candidate, f'{ref_config} (parameters at their {label} bounds)') + + # Entry point for inference scheme, providing infererence-config dict def run_inference(config): """Run the full asynchronous Bayesian inference workflow. @@ -53,6 +181,18 @@ def run_inference(config): # dictionary of directories dirs = get_proteus_directories(config['output']) + # Everything that can be rejected from the config alone is rejected here, + # because the next step empties the output folder and a study re-run after + # a typo would otherwise destroy the previous study's results. + if config['n_workers'] >= os.cpu_count(): + raise RuntimeError(f'Not enough CPU cores for {config["n_workers"]} workers') + + config['ref_config'] = os.path.join(dirs['proteus'], config['ref_config']) + if not os.path.isfile(config['ref_config']): + raise FileNotFoundError('Cannot find reference config: ' + config['ref_config']) + + validate_reference_config(config['ref_config'], config['parameters']) + # Create output directory safe_rm(dirs['output']) os.makedirs(dirs['output']) @@ -81,15 +221,8 @@ def run_inference(config): # Default for configs that pre-date this field config.setdefault('failure_codes', []) - # Ensure there are enough CPU cores for the specified number of workers - if config['n_workers'] >= os.cpu_count(): - raise RuntimeError(f'Not enough CPU cores for {config["n_workers"]} workers') - # Check path to reference config - config['ref_config'] = os.path.join(dirs['proteus'], config['ref_config']) log.info(f'Reference config: {config["ref_config"]}') - if not os.path.isfile(config['ref_config']): - raise FileNotFoundError('Cannot find reference config: ' + config['ref_config']) # Update ref_config path to point to a copy, in case user removes the original file copy_config = os.path.join(os.path.join(dirs['output'], 'ref_config.toml')) From b819b0ff2fa28053a6fb4adecbb8bb79fe9c4450 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 14:58:17 +0200 Subject: [PATCH 03/72] small refactor for config check --- src/proteus/inference/objective.py | 44 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 7a7c57e1f..e39b6aae5 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -29,6 +29,13 @@ DEFAULT_CHILD_TIMEOUT_S = 6 * 3600.0 _CHILD_TIMEOUT_ENV = 'PROTEUS_INFERENCE_CHILD_TIMEOUT_S' +# Config entries every worker overwrites in the reference config, regardless of +# which parameters are being swept. Shared with the startup validation so the +# configuration that is checked is the configuration that is run. +WORKER_CONFIG_OVERRIDES = { + 'params.out.plot_mod': 'none', + 'params.out.logging': 'WARNING', +} def set_child_timeout(seconds: float | None = None) -> None: """Record the per-child PROTEUS timeout for inference worker processes. @@ -57,6 +64,33 @@ def child_timeout_s() -> float | None: return DEFAULT_CHILD_TIMEOUT_S return val if val > 0 else None +def apply_nested_updates(config: dict, updates: dict) -> dict: + """Set dot-separated keys in a nested config dict, in place. + + Parameters + ---------- + - config (dict): Nested configuration dictionary, modified in place. + - updates (dict): Mapping of dot-separated key paths to new values. + + Returns + ---------- + - dict: The same dictionary that was passed in. + + Raises: + ValueError: If a key path descends through an entry that holds a value + rather than a table. + """ + for key, value in updates.items(): + parts = key.split('.') + d = config + for i, part in enumerate(parts[:-1]): + d = d.setdefault(part, {}) + if not isinstance(d, dict): + prefix = '.'.join(parts[: i + 1]) + raise ValueError(f"Cannot set '{key}': '{prefix}' holds a value, not a section") + d[parts[-1]] = value + return config + def update_toml(config_file: str, updates: dict, output_file: str) -> None: """Update values in a TOML configuration file. @@ -83,12 +117,7 @@ def update_toml(config_file: str, updates: dict, output_file: str) -> None: config = toml.load(f) # Apply nested updates - for key, value in updates.items(): - parts = key.split('.') - d = config - for part in parts[:-1]: - d = d.setdefault(part, {}) - d[parts[-1]] = value + apply_nested_updates(config, updates) # Ensure destination directory exists output_path.parent.mkdir(parents=True, exist_ok=True) @@ -139,8 +168,7 @@ def run_proteus( parameters['params.out.path'] = str(out_dir) # Don't allow workers to make plots or logs - parameters['params.out.plot_mod'] = 'none' - parameters['params.out.logging'] = 'WARNING' + parameters.update(WORKER_CONFIG_OVERRIDES) # Generate config update_toml(ref_config, parameters, str(out_cfg)) From 05b929fe6e059752600c7b1376f2e73449454c66 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 15:00:05 +0200 Subject: [PATCH 04/72] add number of functions for robust error handling in proteus workers --- src/proteus/inference/objective.py | 441 ++++++++++++++++++++++++++--- 1 file changed, 408 insertions(+), 33 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index e39b6aae5..d4055987c 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -1,8 +1,10 @@ from __future__ import annotations +import json import logging import os import subprocess +from dataclasses import asdict, dataclass, field from functools import partial from pathlib import Path @@ -14,6 +16,7 @@ from proteus.inference.transforms import unnormalize_parameters from proteus.utils.constants import element_list, gas_list from proteus.utils.coupler import get_proteus_directories, variable_is_logarithmic +from proteus.utils.helper import CommentFromStatus dtype = torch.double EPS_CLIP = 1e-10 @@ -29,6 +32,39 @@ DEFAULT_CHILD_TIMEOUT_S = 6 * 3600.0 _CHILD_TIMEOUT_ENV = 'PROTEUS_INFERENCE_CHILD_TIMEOUT_S' +# Whether a failed child run aborts the study or scores a bad objective value. +# Plumbed through the environment for the same reason as the timeout above. +_ABORT_ON_FAILURE_ENV = 'PROTEUS_INFERENCE_ABORT_ON_FAILURE' + +# Lines of child stderr retained in a failure report. PROTEUS writes its own +# diagnostics to the run's logfile, but a run that dies before the logger is +# set up (a rejected config, a missing environment variable) leaves nothing +# behind except this stream, so it is kept rather than discarded. +STDERR_TAIL_LINES = 40 + +# Suffix for the file holding whatever a child wrote to its console. It is +# kept beside the run folder rather than inside it: PROTEUS empties its own +# output folder once it starts, which would unlink a file held open there and +# lose exactly the record this capture exists to keep. +CHILD_CONSOLE_SUFFIX = '_console.log' + +# Status written by PROTEUS before its output folder is cleaned, and never +# rewritten until the main loop starts. A child that dies in between leaves no +# status file at all, so a missing file is reported as such rather than being +# silently reported as a generic error. +STATUS_MISSING = -1 + +# Folder inside the study output holding one record per failed evaluation. +# Written by the workers as they fail and read back once at the end, so that +# the summary covers initial sampling and optimisation alike without the two +# paths having to share any state while they run. +FAILURE_RECORD_DIR = 'failures' + +# Fraction of evaluations that may fail before the summary escalates from a +# report to a warning. Above this, the sampled region is mostly unrunnable and +# the posterior is built on too few real evaluations to mean much. +FAILURE_FRACTION_WARN = 0.5 + # Config entries every worker overwrites in the reference config, regardless of # which parameters are being swept. Shared with the startup validation so the # configuration that is checked is the configuration that is run. @@ -37,6 +73,233 @@ 'params.out.logging': 'WARNING', } +# Config entries every run sets to the same thing, or to a value derived from +# the run index. Excluded from failure reports, which name the swept values. +_FIXED_PARAMETER_KEYS = set(WORKER_CONFIG_OVERRIDES) | {'params.out.path'} + + +def _tail(text: str | bytes | None, lines: int = STDERR_TAIL_LINES) -> str: + """Return the last `lines` lines of captured child output.""" + if not text: + return '' + if isinstance(text, bytes): + text = text.decode('utf-8', errors='replace') + return '\n'.join(text.splitlines()[-lines:]) + + +def _tail_file(path: Path, lines: int = STDERR_TAIL_LINES) -> str: + """Return the last `lines` lines of a child's console file. + + Returns an empty string when the file is missing or unreadable, so a + failure report is still produced when the capture itself went wrong. + """ + try: + with open(path, 'r', errors='replace') as f: + return _tail(f.read(), lines) + except OSError: + return '' + + +@dataclass(eq=False) +class ProteusRunFailure(RuntimeError): + """A single child PROTEUS run that did not produce a usable result. + + Carries everything needed to diagnose the run without opening the study + by hand: which evaluation it was, where its output landed, how it died, + what PROTEUS recorded in its status file, and the parameter values that + produced it. Raised for faults that are specific to one evaluation; faults + that would affect every evaluation (no `proteus` on PATH, an observable + that no helpfile column provides) stay as ordinary exceptions so they + abort the study instead of being scored as a bad sample. + """ + + reason: str + worker: int + iter: int + out_dir: str + exit_code: int | None = None + status: int = STATUS_MISSING + log_path: str | None = None + stderr_tail: str = '' + parameters: dict = field(default_factory=dict) + + @property + def status_desc(self) -> str: + """Human-readable form of the PROTEUS status code.""" + if self.status == STATUS_MISSING: + return 'no readable status file (died during start-up)' + return CommentFromStatus(self.status) + + def report(self) -> str: + """Multi-line description naming the cause and where to look next.""" + lines = [ + f'PROTEUS run failed for worker={self.worker} iter={self.iter}: {self.reason}', + f' status = {self.status} ({self.status_desc})', + ] + if self.exit_code is not None: + lines.append(f' exit code = {self.exit_code}') + lines.append(f' output = {self.out_dir}') + if self.log_path: + lines.append(f' logfile = {self.log_path}') + if self.parameters: + pretty = ', '.join(f'{k}={v:g}' for k, v in sorted(self.parameters.items())) + lines.append(f' parameters = {pretty}') + if self.stderr_tail: + lines.append(' last output from the child process:') + lines.extend(f' {line}' for line in self.stderr_tail.splitlines()) + return '\n'.join(lines) + + def __str__(self) -> str: + return self.report() + + def __reduce__(self): + # A failure raised inside a pool worker is pickled to be re-raised in + # the parent. BaseException.__reduce__ rebuilds from `self.args`, + # which a dataclass __init__ leaves empty, so the default would fail + # to reconstruct this class. Rebuild from the fields instead. + return ( + self.__class__, + ( + self.reason, + self.worker, + self.iter, + self.out_dir, + self.exit_code, + self.status, + self.log_path, + self.stderr_tail, + self.parameters, + ), + ) + + +def set_abort_on_failure(abort: bool = False) -> None: + """Record whether a failed child run should abort the whole study. + + Stored in the environment so it is visible to the main process and to any + spawned pool workers, matching how the child timeout is plumbed. + """ + os.environ[_ABORT_ON_FAILURE_ENV] = '1' if abort else '0' + + +def abort_on_failure() -> bool: + """Return whether a failed child run should abort the whole study. + + Defaults to False: an inference sweep is expected to visit parameter + combinations the simulator cannot integrate, and treating those as fatal + would make most studies unrunnable. Set the inference config field + `abort_on_failure` to stop at the first failure instead. + """ + return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' + + +def read_status(out_abs: Path | str) -> int: + """Read the PROTEUS status code from a finished run's output folder. + + Parameters + ---------- + - out_abs (Path | str): Absolute path to the run's output folder. + + Returns + ---------- + - int: The status code, or `STATUS_MISSING` when no readable status file + exists. A missing file is itself diagnostic: PROTEUS deletes the status + it writes at start-up when it cleans the output folder, and does not + write another until the main loop begins. + """ + try: + with open(Path(out_abs) / 'status', 'r') as f: + return int(f.readlines()[0].strip()) + except Exception: + return STATUS_MISSING + + +def find_run_logfile(out_abs: Path | str) -> str | None: + """Return the newest PROTEUS logfile in a run's output folder, if any. + + PROTEUS captures uncaught exceptions into this file, so it usually holds + the traceback for a crashed run. It does not exist for a run that failed + before the logger was configured. + """ + logs = sorted(Path(out_abs).glob('proteus_*.log')) + return str(logs[-1]) if logs else None + + +def run_output_dir(output: str, worker: int, iter: int) -> tuple[Path, Path]: + """Return the output folder of a single evaluation, relative and absolute. + + Parameters + ---------- + - output (str): Study output folder, relative to the PROTEUS output root. + - worker (int): Worker identifier. Initial samples use -1. + - iter (int): Iteration identifier within that worker. + + Returns + ---------- + - tuple[Path, Path]: The path as the simulator config records it, and the + absolute path on disk. + """ + out_dir = Path(output) / 'workers' / f'w_{worker}' / f'i_{iter}' + return out_dir, Path(get_proteus_directories(str(out_dir))['output']) + + +def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | None: + """Write a failure record into the study's `failures` folder. + + One file per failed evaluation, named for the worker and iteration that + produced it, so that concurrent workers never write to the same file and + no lock is needed. The end-of-study summary reads them back. + + Parameters + ---------- + - study_abs (Path | str): Absolute path to the study output folder. + - failure (ProteusRunFailure): The failure to record. + + Returns + ---------- + - str | None: Path written, or None if the record could not be written. + Recording is best-effort: a study must not be brought down by a fault in + its own bookkeeping, so the failure being reported still reaches the log. + """ + record = asdict(failure) + record['status_desc'] = failure.status_desc + target = Path(study_abs) / FAILURE_RECORD_DIR / f'w{failure.worker}_i{failure.iter}.json' + try: + target.parent.mkdir(parents=True, exist_ok=True) + with open(target, 'w') as f: + json.dump(record, f, indent=2, sort_keys=True) + except (OSError, TypeError, ValueError) as err: + log.warning( + f'Could not record the failure of worker={failure.worker} ' + f'iter={failure.iter}: {err}' + ) + return None + return str(target) + + +def read_failure_records(study_abs: Path | str) -> list[dict]: + """Read back every failure record written during a study. + + Parameters + ---------- + - study_abs (Path | str): Absolute path to the study output folder. + + Returns + ---------- + - list[dict]: One entry per failed evaluation, ordered by worker then + iteration. Unreadable records are skipped with a warning rather than + aborting the summary, which would hide the failures that did parse. + """ + records = [] + for path in sorted((Path(study_abs) / FAILURE_RECORD_DIR).glob('w*_i*.json')): + try: + with open(path, 'r') as f: + records.append(json.load(f)) + except (OSError, json.JSONDecodeError) as err: + log.warning(f'Skipping unreadable failure record {path}: {err}') + return sorted(records, key=lambda r: (r.get('worker', 0), r.get('iter', 0))) + + def set_child_timeout(seconds: float | None = None) -> None: """Record the per-child PROTEUS timeout for inference worker processes. @@ -64,6 +327,7 @@ def child_timeout_s() -> float | None: return DEFAULT_CHILD_TIMEOUT_S return val if val > 0 else None + def apply_nested_updates(config: dict, updates: dict) -> dict: """Set dot-separated keys in a nested config dict, in place. @@ -151,13 +415,19 @@ def run_proteus( ---------- - observables_dict (dict): Mapping of observable names to their simulated values. - status (int): Status code indicating the outcome of the simulation. + + Raises: + ProteusRunFailure: If this particular run did not produce a usable + result. Carries the status code, the path to the run's logfile and + the parameters that produced it. + RuntimeError: If the `proteus` command itself cannot be executed, which + would affect every run rather than this one. + KeyError: If a requested observable is absent from the output, which + likewise applies to every run. """ # Construct run-specific paths - run_id = Path('workers') / f'w_{worker}' / f'i_{iter}' - out_dir = Path(output) / run_id - - out_abs = Path(get_proteus_directories(str(out_dir))['output']) + out_dir, out_abs = run_output_dir(output, worker, iter) out_cfg = out_abs / 'input.toml' out_csv = out_abs / 'runtime_helpfile.csv' @@ -177,51 +447,110 @@ def run_proteus( env = dict(**os.environ) env['OMP_NUM_THREADS'] = '1' - # Run PROTEUS + # Swept parameter values only, for the failure report. The output path and + # the worker overrides are fixed for every run and add no diagnostic value. + swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} + + def _failure( + reason: str, exit_code: int | None, stderr_tail: str = '' + ) -> ProteusRunFailure: + """Assemble a failure report for this run. + + The status file is read here rather than at the point of the raise so + that a crashed run is described by what PROTEUS recorded about itself, + not only by its exit code. + """ + return ProteusRunFailure( + reason=reason, + worker=worker, + iter=iter, + out_dir=str(out_abs), + exit_code=exit_code, + status=read_status(out_abs), + log_path=find_run_logfile(out_abs), + stderr_tail=stderr_tail, + parameters=swept, + ) + + # Run PROTEUS. Output is kept rather than discarded: a run that dies + # before its logger is configured leaves no logfile behind, so this stream + # is the only record of why it refused to start. It goes to a file rather + # than a pipe because a long run of a chatty module would otherwise buffer + # hours of output in memory, in every worker at once, to retain a few + # lines of it. command = ['proteus', 'start', '-c', str(out_cfg), '--offline'] + console = out_abs.parent / f'{out_abs.name}{CHILD_CONSOLE_SUFFIX}' + console.parent.mkdir(parents=True, exist_ok=True) + # Opened outside the try so that a failure to create it is not mistaken + # for the simulator being absent. + stream = open(console, 'w') try: subprocess.run( command, check=True, text=True, env=env, - stdout=subprocess.DEVNULL, + stdout=stream, stderr=subprocess.STDOUT, timeout=child_timeout_s(), ) except FileNotFoundError as err: + # Applies to every run, not just this one, so it is not a sample that + # can be scored badly and skipped. log.error(f"Cannot execute '{command[0]}': command not found") raise RuntimeError("Failed to run PROTEUS: 'proteus' command not found") from err except subprocess.TimeoutExpired as err: - log.error( - f'PROTEUS run exceeded the {child_timeout_s()} s timeout for ' - f'worker={worker} iter={iter} outdir={str(out_dir)}' - ) - raise RuntimeError( - f'PROTEUS run timed out after {child_timeout_s()} s for worker={worker} iter={iter}' + timeout = child_timeout_s() + raise _failure( + f'exceeded the {timeout} s timeout', + exit_code=None, + stderr_tail=_tail_file(console), ) from err except subprocess.CalledProcessError as err: - log.error(f'PROTEUS run failed for worker={worker} iter={iter} outdir={str(out_dir)}') - raise RuntimeError( - f'Failed to run PROTEUS for worker={worker} iter={iter}; exit code {err.returncode}' + raise _failure( + 'the simulator exited with an error', + exit_code=err.returncode, + stderr_tail=_tail_file(console), ) from err + finally: + stream.close() # Re-write config in case simulator mutates or removes it update_toml(ref_config, parameters, str(out_cfg)) # Read status file - status = 20 # default to Generic Error - try: - with open(out_abs / 'status', 'r') as f: - status = int(f.readlines()[0].strip()) - except Exception as e: - log.warning(f'Failed to read status file for worker={worker} iter={iter}: {e}') + status = read_status(out_abs) - # Read simulator output - df_row = dict(pd.read_csv(out_csv, delimiter=r'\s+').iloc[-1]) + # Read simulator output. A run that exits cleanly but writes no usable + # helpfile (killed mid-write, or stopped before the first row) is a failed + # sample, not a crash of the study. + try: + df_row = dict(pd.read_csv(out_csv, delimiter=r'\s+').iloc[-1]) + except ( + FileNotFoundError, + OSError, + pd.errors.EmptyDataError, + pd.errors.ParserError, + IndexError, + ) as err: + # A truncated whitespace-delimited file usually presents as a ragged + # row (ParserError) rather than an empty one, so both are caught. + raise _failure( + f'exited cleanly but produced no readable output ({out_csv.name})', + exit_code=0, + ) from err # Handle case where atmosphere has escaped # Set VMRs and MMW to zero + if 'P_surf' not in df_row: + # The helpfile schema is the same for every run, so a missing column + # is a fault of the setup rather than of this sample. Scoring it as a + # bad sample would let the study spend its whole budget returning the + # same failure value and then report success. + raise KeyError( + f"Simulator output has no 'P_surf' column ({out_csv}); " + 'every run will produce the same result' + ) if bool(df_row['P_surf'] < 1e-30): df_row['atm_kg_per_mol'] = 0.0 for g in gas_list: @@ -330,17 +659,63 @@ def J( # Map normalized x to raw parameter dict and run PROTEUS raw = {parameters[i]: x[0, i].item() for i in range(len(parameters))} - sim_vals, sim_status = run_proteus( - parameters=raw, - worker=worker, - iter=iter, - observables=list(true_observables.keys()), - ref_config=ref_config, - output=output, - ) + try: + sim_vals, sim_status = run_proteus( + parameters=raw, + worker=worker, + iter=iter, + observables=list(true_observables.keys()), + ref_config=ref_config, + output=output, + ) + except ProteusRunFailure as failure: + # A parameter combination the simulator cannot integrate is an + # expected outcome of sweeping a wide box, so it is scored as a poor + # sample and the study continues. Every such run is reported in full, + # once, because the alternative is a silently under-sampled study. + # Recorded before the abort check, so an aborted study still leaves + # the record of what stopped it. + record_failure(get_proteus_directories(output)['output'], failure) + if abort_on_failure(): + raise + log.warning(failure.report()) + return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) - # If status indicates failure, return very bad objective value - if (20 <= sim_status <= 29) or (sim_status in [0, 1]) or (sim_status in failure_codes): + # If status indicates failure, return very bad objective value. + # Reached by runs that exit cleanly but stop in an error state, such as a + # run halted through its keepalive file (status 25). An unreadable status + # is counted as a failure too: the run's own account of itself is missing, + # so its output cannot be trusted. + if ( + (20 <= sim_status <= 28) + or (sim_status in (0, 1, STATUS_MISSING)) + or (sim_status in failure_codes) + ): + desc = ( + 'no status file written' + if sim_status == STATUS_MISSING + else CommentFromStatus(sim_status) + ) + # Recorded alongside the runs that crashed, so the end-of-study summary + # counts both kinds of failure rather than only the noisy kind. + _, out_abs = run_output_dir(output, worker, iter) + record_failure( + get_proteus_directories(output)['output'], + ProteusRunFailure( + reason='exited cleanly but stopped in a failure state', + worker=worker, + iter=iter, + out_dir=str(out_abs), + exit_code=0, + status=sim_status, + log_path=find_run_logfile(out_abs), + parameters=raw, + ), + ) + log.warning( + f'PROTEUS run for worker={worker} iter={iter} finished in a failure state: ' + f'status {sim_status} ({desc})' + ) return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) # Compute value of objective function given these results From bbe04a216a14cfbb8c2b49ff34203604954f8fed Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 15:02:00 +0200 Subject: [PATCH 05/72] add functions to summarise failures in proteus workers and check if all evaluations failed --- src/proteus/inference/utils.py | 114 ++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index 872c83f7d..05b1c4605 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging +from collections import Counter from collections.abc import Sequence from datetime import datetime from functools import partial @@ -32,7 +33,13 @@ from gpytorch.kernels import MaternKernel, RBFKernel from gpytorch.priors.torch_priors import LogNormalPrior -from proteus.inference.objective import EPS_CLIP, eval_obj +from proteus.inference.objective import ( + BAD_OBJ_VALUE, + EPS_CLIP, + FAILURE_FRACTION_WARN, + eval_obj, + read_failure_records, +) from proteus.inference.transforms import unnormalize_parameters from proteus.utils.constants import gas_list @@ -148,6 +155,86 @@ def load_dataset_csv(fpath: str) -> dict[str, torch.Tensor]: return {'X': X, 'Y': Y} +def summarise_failures(output: str, n_attempted: int) -> int: + """Collect the study's failure records into a table and report on them. + + A sweep over a wide parameter box is expected to reach combinations the + simulator cannot integrate, and those evaluations carry the failure score + rather than a fit quality. Without a count, a study in which most + evaluations failed is indistinguishable from one that converged, so the + tally, the breakdown by cause, and the per-run paths are reported together + at the end of the study. + + Parameters + ---------- + - output (str): Absolute path to the study output folder. + - n_attempted (int): Total evaluations attempted, initial samples included. + + Returns + ---------- + - int: Number of failed evaluations. + """ + records = read_failure_records(output) + n_failed = len(records) + + log.info('-----------------------------------') + if not n_failed: + log.info(f'Simulation failures: none, all {n_attempted} evaluations were usable') + log.info('-----------------------------------') + return 0 + + # Fixed diagnostic columns first, then one column per swept parameter, so + # the table can be sorted on a parameter to see which region fails. + rows = [] + for rec in records: + row = { + key: rec.get(key) + for key in ( + 'worker', + 'iter', + 'status', + 'status_desc', + 'exit_code', + 'reason', + 'out_dir', + 'log_path', + ) + } + row.update(rec.get('parameters') or {}) + rows.append(row) + csv_path = Path(output) / 'failures.csv' + pd.DataFrame(rows).to_csv(csv_path, index=False) + + frac = n_failed / max(n_attempted, 1) + log.info( + f'Simulation failures: {n_failed} of {n_attempted} evaluations ' + f'({100 * frac:.1f}%) did not produce a usable result' + ) + log.info(f'{"Cause":52s} | Count') + for desc, count in Counter( + r.get('status_desc') or 'unknown' for r in records + ).most_common(): + log.info(f'{str(desc):52s} {count}') + log.info(f'Full list: {csv_path}') + + # A few concrete places to look. The simulator writes its own traceback to + # these logfiles, so they carry the cause that the status code only names. + for rec in records[:3]: + if rec.get('log_path'): + log.info(f' {rec["log_path"]}') + + if frac > FAILURE_FRACTION_WARN: + log.warning( + f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations failed, so the ' + f'result below rests on {n_attempted - n_failed} real evaluations. Narrow the ' + 'parameter ranges to a region the simulator can integrate, or check the ' + 'reference config against the causes listed above.' + ) + log.info('-----------------------------------') + + return n_failed + + def print_results(D, logs, config, output, n_init): """Identify the best evaluation and log its observables and inferred parameters. @@ -172,8 +259,31 @@ def print_results(D, logs, config, output, n_init): X = D['X'] Y = D['Y'] + # Count the evaluations that failed, so a study built mostly on failures + # is not read as a converged result. A failed run scores BAD_OBJ_VALUE. + optim_Y = Y[n_init:] + n_optim = len(optim_Y) + n_failed = int((optim_Y <= BAD_OBJ_VALUE).sum().item()) + if n_failed: + log.warning( + f'{n_failed} of {n_optim} optimisation evaluations failed. ' + 'Their objective values are the failure score, not a fit quality; ' + 'the per-run reports above name the cause of each.' + ) + + # Every evaluation failed, so the best of them is still a failed run and + # has no output to report. Say so rather than failing later on its + # missing helpfile. + if n_optim and n_failed == n_optim: + raise RuntimeError( + f'All {n_optim} optimisation evaluations failed, so there is no best ' + 'fit to report. The per-run reports above name the cause of each; ' + 'the most common causes are a reference config the simulator ' + 'refuses and a parameter range that leaves the model unphysical.' + ) + # Find best index, ignoring the initial points - i_opt: int = Y[n_init:].argmax() + n_init + i_opt: int = optim_Y.argmax() + n_init log_opt = logs[i_opt] J_opt: float = Y[i_opt].item() From b709ba7ea5ef952eeb7df145ea7571d7f615c632 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 15:02:26 +0200 Subject: [PATCH 06/72] abort on failure and summarise failures --- src/proteus/inference/inference.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index 8af4710d7..f6028bf02 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -35,9 +35,10 @@ WORKER_CONFIG_OVERRIDES, apply_nested_updates, prot_builder, + set_abort_on_failure, set_child_timeout, ) -from proteus.inference.utils import print_results, str_time +from proteus.inference.utils import print_results, str_time, summarise_failures from proteus.utils.coupler import get_proteus_directories from proteus.utils.helper import safe_rm from proteus.utils.logs import setup_logger @@ -219,6 +220,11 @@ def run_inference(config): # plumbed to worker processes through the environment. set_child_timeout(config.get('child_timeout_s')) + # Whether a failed simulation stops the study or is scored as a poor + # sample. Defaults to scoring, because a sweep over a wide parameter box + # is expected to reach combinations the simulator cannot integrate. + set_abort_on_failure(bool(config.get('abort_on_failure', False))) + # Default for configs that pre-date this field config.setdefault('failure_codes', []) @@ -268,6 +274,12 @@ def run_inference(config): log.info(f'This took: {t_1 - t_0:.2f} seconds') log.info('-----------------------------------') + # Account for the simulations that did not produce a usable result. Runs + # before the best-fit summary, so the reader sees how much of the study was + # real before reading what it concluded, and so the breakdown is still + # reported when every evaluation failed and the summary refuses to print. + summarise_failures(dirs['output'], len(D_final['X'])) + # Print summary of true vs. simulated observables and inferred parameters best_config = print_results(D_final, logs, config, dirs['output'], n_init) From f230c855468802786230964dce45b200ec5ef514 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 13 Sep 2026 15:08:53 +0200 Subject: [PATCH 07/72] split worker into wrapper and _worker_loop, detect dead workers for failure handling --- src/proteus/inference/async_BO.py | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index 989e97caf..75b3f6879 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -104,6 +104,68 @@ def worker( - log_list (Manager.list): Shared list to store per-eval log dicts. - output_dir (str): Output directory for the whole inference call (abspath). + Returns + ---------- + - None + """ + try: + _worker_loop( + process_fun, + build_obj, + D_shared, + B, + T, + T0, + x_init, + n_init, + lock, + max_len, + worker_id, + log_list, + output_dir, + ) + except BaseException: + # A worker that dies takes its traceback with it: multiprocessing + # prints it to the parent's stderr without consulting the logging + # configuration, so nothing reaches the study logfile. Record it here + # while the worker still can, then let it propagate so the exit code + # still marks the process as failed. + log.exception(f'Worker {worker_id} stopped early and will run no further evaluations') + raise + finally: + # Release this worker's busy point. Left in place, it steers the + # surviving workers away from a region nothing is actually exploring. + try: + with lock: + B.pop(worker_id, None) + except Exception: + log.warning(f'Worker {worker_id} could not release its busy point') + + +def _worker_loop( + process_fun, + build_obj, + D_shared, + B, + T, + T0: float, + x_init: torch.Tensor, + n_init: int, + lock, + max_len: int, + worker_id: int, + log_list, + output_dir: str, +) -> None: + """Run BO iterations until the evaluation budget is reached. + + The body of `worker`, separated so that failure reporting and busy-point + release wrap every exit path. + + Parameters + ---------- + - See `worker`; arguments are forwarded unchanged. + Returns ---------- - None @@ -300,4 +362,42 @@ def parallel_process( logs = list(log_list) T_elapsed = [t - T0 for t in list(T)] + # A worker that dies mid-study leaves the run looking complete: the + # remaining workers carry on, the results are saved, and the best-fit + # summary is printed from whatever was collected. Report the shortfall. + # A worker killed by a signal reports a negative code (-9 for an + # out-of-memory kill), so the test is "not zero" rather than "positive". + died = [wid for wid, p in enumerate(procs) if p.exitcode != 0] + if died: + names = ', '.join(str(wid) for wid in died) + log.error( + f'{len(died)} of {n_workers} workers stopped before the evaluation budget ' + f'was reached (workers {names}). Their exit codes were ' + f'{[procs[wid].exitcode for wid in died]}; see the messages above for the ' + f'cause. Results below are based on {len(D_final["X"])} evaluations ' + f'rather than the {max_len} requested.' + ) + # Nothing was added to the initial sample, so there is no optimisation to + # report and the best-fit summary would describe the initial design alone. + if len(D_final['X']) <= n_init: + if died: + cause = ( + f'{len(died)} of {n_workers} workers stopped early; see the messages ' + 'above for the cause.' + ) + else: + # Every worker exited on its first budget check. `max_steps` is + # reduced by one per additional worker, so this is what a study + # with fewer optimisation steps than workers looks like. + cause = ( + 'No worker failed. Each worker stops once the dataset reaches ' + f'{max_steps} rows ({max_len} requested, less one per worker beyond ' + f'the first), which the {n_init} initial samples already satisfy. ' + f'Raise n_steps to at least n_workers ({n_workers}).' + ) + raise RuntimeError( + 'No optimisation steps completed: the dataset still holds only the ' + f'{n_init} initial samples. ' + cause + ) + return D_final, logs, T_elapsed From 204b095a20c37e78a41b7f316a6308f453c9be2a Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 12:37:04 +0200 Subject: [PATCH 08/72] modify to select workers by key instead of position --- src/proteus/inference/BO.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/proteus/inference/BO.py b/src/proteus/inference/BO.py index 3aef51f7c..e7535a122 100644 --- a/src/proteus/inference/BO.py +++ b/src/proteus/inference/BO.py @@ -77,12 +77,13 @@ def BO_step(D, B, f, k, acqf, lock, worker_id, x_in=None): with lock: X = D['X'] Y = D['Y'] - busys = list(B.values()) + # Select by key, not by position: a worker that has finished or + # stopped is absent from B, so the position of an entry in the + # values list does not identify the worker that owns it. + busys = [v for wid, v in B.items() if wid != worker_id] t_1_lock = time.perf_counter() - busys = torch.cat(busys, dim=0) - d = X.shape[-1] best = Y.max().item() @@ -122,10 +123,14 @@ def BO_step(D, B, f, k, acqf, lock, worker_id, x_in=None): t_1_ac = time.perf_counter() - mask = torch.ones(busys.size(0), dtype=torch.bool) - mask[worker_id] = False - b = busys[mask] - dist = torch.min(torch.cdist(b, x)).item() + # Distance to the nearest point another worker is currently evaluating. + # Undefined when no other worker is busy + if busys: + b = torch.cat(busys, dim=0) + dist = torch.min(torch.cdist(b, x)).item() + else: + b = torch.zeros((0, d), dtype=dtype) + dist = None if d == 1: plot_iter( From 55fcc837ee879e770760f16b19433bb9892e1e26 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 12:38:39 +0200 Subject: [PATCH 09/72] update docs with new failure functionality --- docs/How-to/inference.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index 09cc38434..b412e4535 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -122,6 +122,7 @@ The system generates several outputs in: - `logs.csv`: Detailed logs of each BO step - `Ts.csv`: Timestamps for performance analysis - `init.csv`: Data used as an initial guess for starting the optimisation +- `failures.csv`: One row per simulation that did not produce a usable result, written only when at least one did fail (see [Failed simulations](#failed-simulations)) ### Plots The BO scheme will generate many plots upon completion. @@ -144,10 +145,33 @@ Plots prefixed with `result_` show the results of the optimisation. ### Results Summary The system prints the final results including: + - Best found parameters - Corresponding simulated observables - Comparison with target observables +### Failed simulations + +A sweep over a wide parameter box is expected to reach combinations PROTEUS +cannot integrate. Such an evaluation is scored with a fixed bad objective value +and the study carries on, so that one unphysical corner does not end a run that +has been going for hours. + +Each failed evaluation is reported as it happens, naming the status code PROTEUS +recorded, the run's output folder, the logfile holding its traceback, and the +parameter values that produced it. At the end of the study the failures are +collected into `failures.csv` and summarised: how many of the evaluations failed, +a breakdown by cause, and the first few logfiles to open. If more than half the +evaluations failed, the summary says so as a warning, because the result then +rests on far fewer real evaluations than the step count suggests. If every +optimisation evaluation failed there is no best fit to report, and the study +stops with an error rather than presenting the least-bad failure as a result. + +Set `abort_on_failure = true` in the inference config to stop at the first failed +simulation instead. This is useful while setting a study up, when the first +failure is more likely to be a mistake in the reference config than a genuinely +unrunnable corner of the parameter space. + ## Customization ### Adding New Parameters From 4e27e7ba4b7e6a53460e586310b3dd445561ced0 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 13:23:53 +0200 Subject: [PATCH 10/72] add function to refuse configuration at error level (not info) --- src/proteus/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/proteus/cli.py b/src/proteus/cli.py index 71d3dbf80..aab6a68a8 100644 --- a/src/proteus/cli.py +++ b/src/proteus/cli.py @@ -56,6 +56,7 @@ def _should_apply_deterministic(argv, environ) -> bool: else: os.execvp(sys.argv[0], sys.argv) +import logging # noqa: E402 import shutil # noqa: E402 import subprocess # noqa: E402 import tempfile # noqa: E402 @@ -70,6 +71,8 @@ def _should_apply_deterministic(argv, environ) -> bool: from proteus.utils.helper import get_proteus_dir, resolve_fwl_data_dir # noqa: E402 from proteus.utils.logs import bootstrap_logger, setup_logger # noqa: E402 +log = logging.getLogger('fwl.' + __name__) + config_option = click.option( '-c', '--config', @@ -89,6 +92,20 @@ def _should_apply_deterministic(argv, environ) -> bool: ) +class ConfigRejectedError(click.ClickException): + """A refused configuration, reported at error level on the 'fwl' logger. + + click prints a ClickException as a bare ``Error: ...`` line that carries no + level, so a refusal arrived untagged among the level-tagged lines around + it. Overriding how it is shown keeps everything click gives the caller (no + traceback, exit code 1) while routing the text through the same logger and + formatter as the rest of the run, where it is marked ERROR. + """ + + def show(self, file=None) -> None: + log.error(self.format_message()) + + class ConfigAwareGroup(click.Group): """Command group that presents a refused configuration as a CLI error. @@ -103,7 +120,7 @@ def invoke(self, ctx): try: return super().invoke(ctx) except UnknownConfigKeyError as exc: - raise click.ClickException(str(exc)) from exc + raise ConfigRejectedError(str(exc)) from exc @click.group(cls=ConfigAwareGroup) From c457cd48b0c69c4985c45c042c99495af368337f Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 16:39:35 +0200 Subject: [PATCH 11/72] soften the failure code 11 logs (max runtime reached) --- src/proteus/inference/async_BO.py | 3 +- src/proteus/inference/gen_D_init.py | 3 +- src/proteus/inference/objective.py | 82 +++++++++++++++++-------- src/proteus/inference/utils.py | 95 ++++++++++++++++++----------- 4 files changed, 121 insertions(+), 62 deletions(-) diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index 75b3f6879..4fc241fc5 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -268,7 +268,8 @@ def parallel_process( - ref_config (str): Path to reference config to pass to objective_builder. - observables (dict): Target observables (keys) and values. - parameters (dict): Parameters (keys) with bounds (values) for inference. - - failure_codes (list[int]): Additional PROTEUS exit codes to treat as failures. + - failure_codes (list[int]): PROTEUS status codes that complete normally but + that this study excludes from the fit. Returns ---------- diff --git a/src/proteus/inference/gen_D_init.py b/src/proteus/inference/gen_D_init.py index 25d4b91c4..35866b379 100644 --- a/src/proteus/inference/gen_D_init.py +++ b/src/proteus/inference/gen_D_init.py @@ -226,7 +226,8 @@ def sample_from_bounds( - nsamp (int): Number of initial samples to evaluate. - seed (int): RNG seed for Halton sequence generation. - n_workers (int): Number of parallel workers to use for evaluation. - - failure_codes (list[int]): Additional PROTEUS exit codes to treat as failures. + - failure_codes (list[int]): PROTEUS status codes that complete normally but + that this study excludes from the fit. Returns ---------- diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index d4055987c..e28bdfce3 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -54,6 +54,15 @@ # silently reported as a generic error. STATUS_MISSING = -1 +# How an evaluation that carries no fit quality is classified. A run that +# crashed, or stopped in an error state, did not produce a result at all. A run +# that completed normally but ended on a status listed in the study's +# `failure_codes` did produce a result; the study simply does not fit against +# that outcome. Both score BAD_OBJ_VALUE, but only the first is a fault, so the +# two are named and counted apart rather than both being called failures. +CATEGORY_FAILURE = 'failure' +CATEGORY_EXCLUDED = 'excluded' + # Folder inside the study output holding one record per failed evaluation. # Written by the workers as they fail and read back once at the end, so that # the summary covers initial sampling and optimisation alike without the two @@ -107,10 +116,14 @@ class ProteusRunFailure(RuntimeError): Carries everything needed to diagnose the run without opening the study by hand: which evaluation it was, where its output landed, how it died, what PROTEUS recorded in its status file, and the parameter values that - produced it. Raised for faults that are specific to one evaluation; faults - that would affect every evaluation (no `proteus` on PATH, an observable - that no helpfile column provides) stay as ordinary exceptions so they - abort the study instead of being scored as a bad sample. + produced it. `category` separates a genuine fault from a run that completed + normally on a status the study excludes; both score the failure value, but + only the first is reported as something having gone wrong. + + Raised for faults that are specific to one evaluation; faults that would + affect every evaluation (no `proteus` on PATH, an observable that no + helpfile column provides) stay as ordinary exceptions so they abort the + study instead of being scored as a bad sample. """ reason: str @@ -122,6 +135,7 @@ class ProteusRunFailure(RuntimeError): log_path: str | None = None stderr_tail: str = '' parameters: dict = field(default_factory=dict) + category: str = CATEGORY_FAILURE @property def status_desc(self) -> str: @@ -132,8 +146,9 @@ def status_desc(self) -> str: def report(self) -> str: """Multi-line description naming the cause and where to look next.""" + verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' lines = [ - f'PROTEUS run failed for worker={self.worker} iter={self.iter}: {self.reason}', + f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', f' status = {self.status} ({self.status_desc})', ] if self.exit_code is not None: @@ -169,6 +184,7 @@ def __reduce__(self): self.log_path, self.stderr_tail, self.parameters, + self.category, ), ) @@ -650,7 +666,8 @@ def J( - iter (int): Iteration number. - output (str): Path to output folder relative to PROTEUS output folder. - ref_config (str): Reference TOML config path. - - failure_codes (list[int]): Additional PROTEUS exit codes to treat as failures. + - failure_codes (list[int]): PROTEUS status codes that complete normally but + that this study excludes from the fit. Returns ---------- @@ -681,28 +698,36 @@ def J( log.warning(failure.report()) return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) - # If status indicates failure, return very bad objective value. - # Reached by runs that exit cleanly but stop in an error state, such as a - # run halted through its keepalive file (status 25). An unreadable status - # is counted as a failure too: the run's own account of itself is missing, - # so its output cannot be trusted. - if ( - (20 <= sim_status <= 28) - or (sim_status in (0, 1, STATUS_MISSING)) - or (sim_status in failure_codes) - ): + # Runs that exit cleanly but stop in an error state, such as a run halted + # through its keepalive file (status 25), or that never reach the main loop + # (status 0 and 1). An unreadable status counts here too: the run's own + # account of itself is missing, so its output cannot be trusted. + failed = (20 <= sim_status <= 28) or (sim_status in (0, 1, STATUS_MISSING)) + + # Runs that completed normally on an outcome this study does not fit + # against, named by the `failure_codes` field of the inference config: a + # run stopped by its clock limit (status 11) or one whose volatiles all + # escaped (status 15), for instance. Nothing went wrong in such a run, so + # it is scored as a poor sample but is not reported as a fault. + excluded = (not failed) and (sim_status in failure_codes) + + # Either way the evaluation carries the failure score instead of a fit + # quality, and is recorded so that the end-of-study tally covers it. + if failed or excluded: desc = ( 'no status file written' if sim_status == STATUS_MISSING else CommentFromStatus(sim_status) ) - # Recorded alongside the runs that crashed, so the end-of-study summary - # counts both kinds of failure rather than only the noisy kind. _, out_abs = run_output_dir(output, worker, iter) record_failure( get_proteus_directories(output)['output'], ProteusRunFailure( - reason='exited cleanly but stopped in a failure state', + reason=( + 'exited cleanly but stopped in a failure state' + if failed + else 'completed on a status this study excludes' + ), worker=worker, iter=iter, out_dir=str(out_abs), @@ -710,12 +735,20 @@ def J( status=sim_status, log_path=find_run_logfile(out_abs), parameters=raw, + category=CATEGORY_FAILURE if failed else CATEGORY_EXCLUDED, ), ) - log.warning( - f'PROTEUS run for worker={worker} iter={iter} finished in a failure state: ' - f'status {sim_status} ({desc})' - ) + if failed: + log.warning( + f'PROTEUS run for worker={worker} iter={iter} did not produce a usable ' + f'result: status {sim_status} ({desc})' + ) + else: + log.info( + f'PROTEUS run for worker={worker} iter={iter} completed on status ' + f'{sim_status} ({desc}), which this study excludes; scored as a poor ' + 'sample' + ) return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) # Compute value of objective function given these results @@ -743,7 +776,8 @@ def prot_builder( - iter (int): Iteration number (seed) for reproducibility. - output (str): Path to output folder relative to PROTEUS output folder. - ref_config (str): Reference TOML config path. - - failure_codes (list[int]): Additional PROTEUS exit codes to treat as failures. + - failure_codes (list[int]): PROTEUS status codes that complete normally but + that this study excludes from the fit. Returns ---------- diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index 05b1c4605..dc330ca2b 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -35,6 +35,8 @@ from proteus.inference.objective import ( BAD_OBJ_VALUE, + CATEGORY_EXCLUDED, + CATEGORY_FAILURE, EPS_CLIP, FAILURE_FRACTION_WARN, eval_obj, @@ -156,14 +158,16 @@ def load_dataset_csv(fpath: str) -> dict[str, torch.Tensor]: def summarise_failures(output: str, n_attempted: int) -> int: - """Collect the study's failure records into a table and report on them. + """Collect the study's unscored evaluations into a table and report on them. A sweep over a wide parameter box is expected to reach combinations the - simulator cannot integrate, and those evaluations carry the failure score - rather than a fit quality. Without a count, a study in which most - evaluations failed is indistinguishable from one that converged, so the - tally, the breakdown by cause, and the per-run paths are reported together - at the end of the study. + simulator cannot integrate, and to reach outcomes the study itself excludes + through `failure_codes`. Both carry the failure score rather than a fit + quality. Without a count, a study in which most evaluations were never + scored is indistinguishable from one that converged, so the tally, the + breakdown by cause, and the per-run paths are reported together at the end + of the study. Runs that failed and runs that were excluded are counted + apart, because only the first kind means something went wrong. Parameters ---------- @@ -172,17 +176,23 @@ def summarise_failures(output: str, n_attempted: int) -> int: Returns ---------- - - int: Number of failed evaluations. + - int: Number of evaluations that carry the failure score. """ records = read_failure_records(output) - n_failed = len(records) + n_unscored = len(records) log.info('-----------------------------------') - if not n_failed: - log.info(f'Simulation failures: none, all {n_attempted} evaluations were usable') + if not n_unscored: + log.info(f'Unscored evaluations: none, all {n_attempted} evaluations were usable') log.info('-----------------------------------') return 0 + # A record with no category describes a genuine fault: it came either from + # the crash path, which never excludes, or from a study run before the two + # were separated. + n_excluded = sum(1 for r in records if r.get('category') == CATEGORY_EXCLUDED) + n_failed = n_unscored - n_excluded + # Fixed diagnostic columns first, then one column per swept parameter, so # the table can be sorted on a parameter to see which region fails. rows = [] @@ -192,6 +202,7 @@ def summarise_failures(output: str, n_attempted: int) -> int: for key in ( 'worker', 'iter', + 'category', 'status', 'status_desc', 'exit_code', @@ -200,21 +211,29 @@ def summarise_failures(output: str, n_attempted: int) -> int: 'log_path', ) } + row['category'] = row['category'] or CATEGORY_FAILURE row.update(rec.get('parameters') or {}) rows.append(row) csv_path = Path(output) / 'failures.csv' pd.DataFrame(rows).to_csv(csv_path, index=False) - frac = n_failed / max(n_attempted, 1) + frac = n_unscored / max(n_attempted, 1) log.info( - f'Simulation failures: {n_failed} of {n_attempted} evaluations ' - f'({100 * frac:.1f}%) did not produce a usable result' + f'Unscored evaluations: {n_unscored} of {n_attempted} evaluations ' + f'({100 * frac:.1f}%) carry the failure score rather than a fit quality' ) + log.info(f' {n_failed} did not produce a usable result') + log.info(f' {n_excluded} completed on a status this study excludes') + + # Grouped by cause, and labelled so that an excluded outcome is not read as + # something having gone wrong in the run that reached it. log.info(f'{"Cause":52s} | Count') - for desc, count in Counter( - r.get('status_desc') or 'unknown' for r in records + for (category, desc), count in Counter( + (r.get('category') or CATEGORY_FAILURE, r.get('status_desc') or 'unknown') + for r in records ).most_common(): - log.info(f'{str(desc):52s} {count}') + label = f'{desc} [excluded]' if category == CATEGORY_EXCLUDED else str(desc) + log.info(f'{label:52s} {count}') log.info(f'Full list: {csv_path}') # A few concrete places to look. The simulator writes its own traceback to @@ -225,14 +244,15 @@ def summarise_failures(output: str, n_attempted: int) -> int: if frac > FAILURE_FRACTION_WARN: log.warning( - f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations failed, so the ' - f'result below rests on {n_attempted - n_failed} real evaluations. Narrow the ' - 'parameter ranges to a region the simulator can integrate, or check the ' - 'reference config against the causes listed above.' + f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations were not scored ' + f'on fit quality, so the result below rests on {n_attempted - n_unscored} real ' + 'evaluations. Narrow the parameter ranges to a region the simulator can ' + 'integrate and the study accepts, or check the reference config against the ' + 'causes listed above.' ) log.info('-----------------------------------') - return n_failed + return n_unscored def print_results(D, logs, config, output, n_init): @@ -259,27 +279,30 @@ def print_results(D, logs, config, output, n_init): X = D['X'] Y = D['Y'] - # Count the evaluations that failed, so a study built mostly on failures - # is not read as a converged result. A failed run scores BAD_OBJ_VALUE. + # Count the evaluations that were never scored on fit quality, so a study + # built mostly on those is not read as a converged result. Such a run + # scores BAD_OBJ_VALUE, whether it failed outright or completed on a + # status the study excludes; the objective value alone cannot tell the two + # apart, so the wording here covers both and the tally above splits them. optim_Y = Y[n_init:] n_optim = len(optim_Y) - n_failed = int((optim_Y <= BAD_OBJ_VALUE).sum().item()) - if n_failed: + n_unscored = int((optim_Y <= BAD_OBJ_VALUE).sum().item()) + if n_unscored: log.warning( - f'{n_failed} of {n_optim} optimisation evaluations failed. ' - 'Their objective values are the failure score, not a fit quality; ' - 'the per-run reports above name the cause of each.' + f'{n_unscored} of {n_optim} optimisation evaluations carry the failure ' + 'score rather than a fit quality, because they failed or completed on an ' + 'excluded status; the per-run reports above name each one.' ) - # Every evaluation failed, so the best of them is still a failed run and - # has no output to report. Say so rather than failing later on its - # missing helpfile. - if n_optim and n_failed == n_optim: + # No evaluation was scored, so the best of them is still a run with no fit + # to report. Say so rather than failing later on its missing helpfile. + if n_optim and n_unscored == n_optim: raise RuntimeError( - f'All {n_optim} optimisation evaluations failed, so there is no best ' - 'fit to report. The per-run reports above name the cause of each; ' - 'the most common causes are a reference config the simulator ' - 'refuses and a parameter range that leaves the model unphysical.' + f'None of the {n_optim} optimisation evaluations produced a fit quality, ' + 'so there is no best fit to report. The per-run reports above name the ' + 'cause of each; the most common causes are a reference config the ' + 'simulator refuses, a parameter range that leaves the model unphysical, ' + 'and a `failure_codes` list that excludes the outcome most runs reach.' ) # Find best index, ignoring the initial points From d6718b6a175fbcf2242140726bee45dc9f7f02b0 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 16:44:59 +0200 Subject: [PATCH 12/72] add abort_on_failure --- input/inference/example.infer.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/input/inference/example.infer.toml b/input/inference/example.infer.toml index 2f68c956c..39b61b29b 100644 --- a/input/inference/example.infer.toml +++ b/input/inference/example.infer.toml @@ -10,19 +10,24 @@ logging = "INFO" output = "bayesopt_infer_se_mat12_lei_5_100/" # Path to base (reference) config file relative to PROTEUS root folder -# ref_config = "output/bayesopt_se/init_coupler.toml" ref_config = "input/inference/example.toml" # Method for initialising the inference scheme (one of these must be 'none') init_samps = 3 # Number of random samples if starting from scratch. init_grid = 'none' # grid_demo/' # Path pre-computed grid (relative to PROTEUS output folder) -# Excluded exit codes (in addition to errors) +# Completion codes to exclude from the fit, on top of the error codes that are +# always excluded. A run ending on one of these completed normally; the study +# simply does not score that outcome. failure_codes = [11, ] # solidified (10), escaped (15), max_runtime (11) +# Stop the whole study at the first simulation that fails, instead of scoring +# it as a poor sample and carrying on. Off by default. +abort_on_failure = false + # Parameters for Bayesian optimisation n_workers = 5 # Number of parallel workers -kernel = "MAT1/2" # Kernel type for GP, "RBF" | "MAT1/2" | "MAT3/2" | "MAT5/2" +kernel = "MAT3/2" # Kernel type for GP, "RBF" | "MAT1/2" | "MAT3/2" | "MAT5/2" acqf = "LogEI" # Acquisition function, "UCB" | "LogEI" | "LogPI" n_steps = 100 # Total number of evaluations (i.e. BO steps) From 3718409309254116fc03f7dd435d74dbaf48bd25 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 16:48:27 +0200 Subject: [PATCH 13/72] add inference tests for config validation and failure handling --- tests/inference/test_async_bo.py | 356 ++++++++++++- tests/inference/test_bo.py | 110 ++++ tests/inference/test_inference.py | 185 +++++++ tests/inference/test_objective.py | 702 ++++++++++++++++++++++++- tests/inference/test_transforms.py | 15 +- tests/inference/test_utils_branches.py | 206 ++++++++ tests/test_cli.py | 52 +- 7 files changed, 1586 insertions(+), 40 deletions(-) diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index dc811de2a..1a6e69c31 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -223,10 +223,20 @@ class FakeProcess: def __init__(self, target, args): self.target = target self.args = args + self.exitcode = 0 created_processes.append(self) def start(self): - return None + # A worker that runs contributes at least one evaluation. Standing + # in for that keeps the dataset past the initial samples, which is + # what distinguishes a study that ran from one that did not. + shared = self.args[2] + shared['X'] = torch.cat( + (shared['X'], torch.tensor([[0.5]], dtype=torch.double)), dim=0 + ) + shared['Y'] = torch.cat( + (shared['Y'], torch.tensor([[0.7]], dtype=torch.double)), dim=0 + ) def join(self): return None @@ -269,7 +279,347 @@ def join(self): ) assert len(created_processes) == 2 - assert D_final['X'].shape == (1, 1) - assert D_final['Y'].shape == (1, 1) + # One initial sample plus one evaluation from each of the two workers. + assert D_final['X'].shape == (3, 1) + assert D_final['Y'].shape == (3, 1) assert logs == [None] assert elapsed == [] + + +# ============================================================================ +# Reporting workers that stop before the evaluation budget is reached +# ============================================================================ + + +def _mocked_parallel_process_env(monkeypatch, tmp_path, fake_process_cls, n_init_rows=1): + """Wire ``parallel_process`` to in-process fakes for the shared state. + + Returns nothing; the caller supplies the Process stand-in whose exit codes + and side effects define the scenario under test. + """ + + class FakeManager: + def dict(self, data=None): + return {} if data is None else dict(data) + + def list(self, data=None): + return [] if data is None else list(data) + + def Lock(self): + return _DummyLock() + + (tmp_path / 'init.csv').write_text('x_0,y\n0.1,0.2\n', encoding='utf-8') + monkeypatch.setattr( + async_mod, 'get_proteus_directories', lambda _output: {'output': str(tmp_path)} + ) + monkeypatch.setattr(async_mod, 'Manager', FakeManager) + monkeypatch.setattr(async_mod, 'Process', fake_process_cls) + monkeypatch.setattr( + async_mod, + 'load_dataset_csv', + lambda _path: { + 'X': torch.tensor([[0.1]] * n_init_rows, dtype=torch.double), + 'Y': torch.tensor([[0.2]] * n_init_rows, dtype=torch.double), + }, + ) + monkeypatch.setattr( + async_mod, + 'init_locs', + lambda n_workers, _D_shared, acqf='LogEI': torch.tensor( + [[0.2], [0.8]], dtype=torch.double + )[:n_workers], + ) + monkeypatch.setattr(async_mod, 'get_kernel', lambda *args, **kwargs: object()) + + +@pytest.mark.unit +def test_parallel_process_reports_a_worker_that_stopped_early(monkeypatch, tmp_path, caplog): + """A worker that dies mid-study leaves the run looking complete: the others + carry on and the results are saved. The shortfall is reported by worker id + and evaluation count so the summary that follows is not read as a full + sweep of the requested budget. + """ + + class FakeProcess: + _next = [0] + + def __init__(self, target, args): + self.args = args + self.worker_id = FakeProcess._next[0] + FakeProcess._next[0] += 1 + # Worker 1 is killed by a signal, as an out-of-memory kill does, + # which reports a negative code rather than a positive one. + # Worker 0 contributes one evaluation. + self.exitcode = -9 if self.worker_id == 1 else 0 + + def start(self): + if self.exitcode == 0: + shared = self.args[2] + shared['X'] = torch.cat( + (shared['X'], torch.tensor([[0.5]], dtype=torch.double)), dim=0 + ) + shared['Y'] = torch.cat( + (shared['Y'], torch.tensor([[0.7]], dtype=torch.double)), dim=0 + ) + + def join(self): + return None + + _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess) + + with caplog.at_level('ERROR'): + D_final, _logs, _elapsed = async_mod.parallel_process( + objective_builder=lambda **kwargs: lambda x: x, + kernel='MAT3/2', + acqf='LogEI', + n_workers=2, + max_len=6, + output='dummy', + seed=1, + ref_config='ref.toml', + observables={'obs': 1.0}, + parameters={'a': [0.0, 1.0]}, + failure_codes=[], + ) + + # The partial study is still returned, so the evaluations that did complete + # are not thrown away. + assert D_final['X'].shape == (2, 1) + reported = '\n'.join(record.getMessage() for record in caplog.records) + # Identity guard: the dead worker is named, not merely counted. A + # regression that reported "1 worker failed" without the id would leave + # the user with nowhere to look. + assert 'workers 1' in reported + assert '1 of 2 workers' in reported + # The signal code is reported as it stands. A guard written as + # "exitcode > 0" would miss a killed worker entirely. + assert '-9' in reported + # Budget guard: the count actually achieved is contrasted with the count + # requested, which is what makes the shortfall visible. + assert '2 evaluations' in reported and '6 requested' in reported + + +@pytest.mark.unit +def test_parallel_process_stays_silent_when_every_worker_completes( + monkeypatch, tmp_path, caplog +): + """A study in which no worker died reports no shortfall. Without this, the + failure message above would be indistinguishable from routine noise. + """ + + class FakeProcess: + def __init__(self, target, args): + self.args = args + self.exitcode = 0 + + def start(self): + shared = self.args[2] + shared['X'] = torch.cat( + (shared['X'], torch.tensor([[0.5]], dtype=torch.double)), dim=0 + ) + shared['Y'] = torch.cat( + (shared['Y'], torch.tensor([[0.7]], dtype=torch.double)), dim=0 + ) + + def join(self): + return None + + _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess) + + with caplog.at_level('ERROR'): + D_final, _logs, _elapsed = async_mod.parallel_process( + objective_builder=lambda **kwargs: lambda x: x, + kernel='MAT3/2', + acqf='LogEI', + n_workers=2, + max_len=6, + output='dummy', + seed=1, + ref_config='ref.toml', + observables={'obs': 1.0}, + parameters={'a': [0.0, 1.0]}, + failure_codes=[], + ) + + assert D_final['X'].shape == (3, 1) + assert [r for r in caplog.records if r.levelname == 'ERROR'] == [] + + +@pytest.mark.unit +def test_parallel_process_refuses_a_study_with_no_completed_steps(monkeypatch, tmp_path): + """When the dataset never grows past the initial samples there is no + optimisation to report, and the best-fit summary downstream would describe + the initial design while presenting it as an inference result. The study + stops instead, naming the reason. + """ + + class FakeProcess: + def __init__(self, target, args): + self.args = args + self.exitcode = 1 + + def start(self): + return None + + def join(self): + return None + + _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess) + + with pytest.raises(RuntimeError) as excinfo: + async_mod.parallel_process( + objective_builder=lambda **kwargs: lambda x: x, + kernel='MAT3/2', + acqf='LogEI', + n_workers=2, + max_len=6, + output='dummy', + seed=1, + ref_config='ref.toml', + observables={'obs': 1.0}, + parameters={'a': [0.0, 1.0]}, + failure_codes=[], + ) + message = str(excinfo.value) + assert 'No optimisation steps completed' in message + # The cause is attributed to the workers, not to the evaluation budget, + # because they reported non-zero exit codes. + assert '2 of 2 workers stopped early' in message + + +@pytest.mark.unit +def test_worker_releases_its_busy_point_and_records_why_it_stopped(tmp_path, caplog): + """A worker that fails records the cause in the study log before it dies, + and releases the point it had claimed. Neither happens on its own: + multiprocessing prints a dead worker's traceback straight to the parent's + stderr without consulting the logging configuration, and a claimed point + left in place steers the surviving workers away from a region nothing is + exploring. + """ + D_shared = { + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[0.2]], dtype=torch.double), + } + B = {0: torch.tensor([[0.3]], dtype=torch.double)} + + def exploding_process_fun(**_kwargs): + raise RuntimeError('objective evaluation failed') + + with caplog.at_level('ERROR'): + with pytest.raises(RuntimeError, match='objective evaluation failed'): + async_mod.worker( + process_fun=exploding_process_fun, + build_obj=lambda **kwargs: lambda x: x, + D_shared=D_shared, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.3]], dtype=torch.double), + n_init=1, + lock=_DummyLock(), + max_len=4, + worker_id=0, + log_list=[], + output_dir=str(tmp_path), + ) + + # The claimed point is released. + assert 0 not in B + # The cause reached the study log, with a traceback attached. + records = [r for r in caplog.records if r.levelname == 'ERROR'] + assert any('Worker 0 stopped early' in r.getMessage() for r in records) + assert any(r.exc_info is not None for r in records) + # Nothing was appended to the shared dataset, so a failed evaluation + # cannot masquerade as a completed one. + assert D_shared['X'].shape == (1, 1) + + +@pytest.mark.unit +def test_worker_releases_its_busy_point_after_a_normal_finish(tmp_path): + """A worker that reaches the evaluation budget also releases its claimed + point. Left behind, it would bias the acquisition for every worker still + running through the tail of the study. + """ + D_shared = { + 'X': torch.tensor([[0.1], [0.2]], dtype=torch.double), + 'Y': torch.tensor([[0.3], [0.4]], dtype=torch.double), + } + B = { + 0: torch.tensor([[0.5]], dtype=torch.double), + 1: torch.tensor([[0.6]], dtype=torch.double), + } + + # max_len is already reached, so the loop exits without an evaluation. + async_mod.worker( + process_fun=lambda **_kwargs: pytest.fail('no evaluation should run'), + build_obj=lambda **kwargs: lambda x: x, + D_shared=D_shared, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.5]], dtype=torch.double), + n_init=2, + lock=_DummyLock(), + max_len=2, + worker_id=0, + log_list=[], + output_dir=str(tmp_path), + ) + + assert 0 not in B + # Only this worker's claim is released; the other worker is still running. + assert 1 in B + + +@pytest.mark.unit +def test_parallel_process_names_the_real_step_budget_when_no_worker_failed( + monkeypatch, tmp_path +): + """A study configured with fewer optimisation steps than workers finishes + without any worker failing and without any step being taken. The refusal + must quote the row count the workers actually stop at, which is the + requested budget less one per worker beyond the first, or the advice reads + as false against the numbers the user set. + """ + + class FakeProcess: + def __init__(self, target, args): + self.args = args + self.exitcode = 0 + + def start(self): + # Every worker sees the budget already met and exits at once, + # contributing nothing to the dataset. + return None + + def join(self): + return None + + # Six initial samples against a budget of six with two workers: the worker + # threshold is 6 - (2 - 1) = 5, which the initial samples already exceed. + _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess, n_init_rows=6) + + with pytest.raises(RuntimeError) as excinfo: + async_mod.parallel_process( + objective_builder=lambda **kwargs: lambda x: x, + kernel='MAT3/2', + acqf='LogEI', + n_workers=2, + max_len=6, + output='dummy', + seed=1, + ref_config='ref.toml', + observables={'obs': 1.0}, + parameters={'a': [0.0, 1.0]}, + failure_codes=[], + ) + message = str(excinfo.value) + assert 'No worker failed' in message + # The threshold the workers actually apply. + assert 'reaches 5 rows' in message + # Discrimination: quoting the requested budget here instead would state + # that six initial samples satisfy a six-row threshold, which is false. + assert 'reaches 6 rows' not in message + # The requested budget is still named, so the two numbers can be related. + assert '6 requested' in message + assert 'Raise n_steps to at least n_workers (2)' in message diff --git a/tests/inference/test_bo.py b/tests/inference/test_bo.py index 4ac04a434..863c56ced 100644 --- a/tests/inference/test_bo.py +++ b/tests/inference/test_bo.py @@ -423,3 +423,113 @@ def test_quadratic_objective_returns_zero_at_target(): y_near = objective(torch.tensor([[0.45, 0.55]], dtype=torch.double)) # ratio of (far-target)^2 to (near-target)^2 = ((0.3,0.3))^2 / ((0.15,0.15))^2 = 4 assert y_near.item() / y_far.item() == pytest.approx(0.25, rel=1e-9) + + +# ============================================================================ +# Busy-point bookkeeping when workers come and go +# ============================================================================ + + +def _patched_bo_step_deps(monkeypatch, candidate=0.8): + """Replace the GP fit and acquisition optimisation with fixed stand-ins. + + Leaves the busy-point handling under test as the only live logic. + """ + monkeypatch.setattr(bo_mod, 'SingleTaskGP', lambda **kwargs: _DummyGP()) + monkeypatch.setattr(bo_mod, 'ExactMarginalLogLikelihood', lambda _lik, _gp: object()) + monkeypatch.setattr(bo_mod, 'fit_gpytorch_mll', lambda *args, **kwargs: None) + monkeypatch.setattr(bo_mod, 'get_acqf', lambda *args, **kwargs: object()) + monkeypatch.setattr( + bo_mod, + 'optimize_acqf', + lambda **kwargs: (torch.tensor([[candidate]], dtype=torch.double), None), + ) + monkeypatch.setattr(bo_mod, 'plot_iter', lambda **kwargs: None) + + +@pytest.mark.unit +def test_bo_step_identifies_busy_points_by_worker_id_not_position(monkeypatch): + """Busy points are matched to their owner by worker id. Once a worker has + stopped and released its claim, the remaining entries no longer sit at the + position their worker id implies, so a positional lookup reads another + worker's point as its own. + """ + _patched_bo_step_deps(monkeypatch, candidate=0.8) + + D = { + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[1.0]], dtype=torch.double), + } + # Worker 1 has stopped and released its point. Worker 2 is still running, + # and is the caller here: its own claim must be excluded, the others kept. + # Two other workers are present so the nearest is not also the furthest, + # which a single other point would make indistinguishable. + B = { + 0: torch.tensor([[0.1]], dtype=torch.double), + 2: torch.tensor([[0.75]], dtype=torch.double), + 3: torch.tensor([[0.79]], dtype=torch.double), + } + + _x, y, *_rest, dist = bo_mod.BO_step( + D=D, + B=B, + f=lambda _x: torch.tensor([[0.9]], dtype=torch.double), + k=object(), + acqf='UCB', + lock=_DummyLock(), + worker_id=2, + ) + + assert y[0, 0].item() == pytest.approx(0.9) + # Nearest other claim is worker 3 at 0.79, from the candidate at 0.8. + assert dist == pytest.approx(0.01) + # Nearest, not furthest: worker 0 sits at 0.1, giving 0.7. A regression to + # torch.max would report that instead. + assert abs(dist - 0.7) > 0.5 + # The caller's own claim at 0.75 is excluded. Including it would give + # 0.05, which is neither of the two values above. + assert abs(dist - 0.05) > 0.02 + + +@pytest.mark.unit +def test_bo_step_reports_no_distance_when_no_other_worker_is_busy(monkeypatch): + """With no other worker running, there is no nearest busy point and the + distance is undefined rather than zero. This is the steady state of a + single-worker study and the tail of every multi-worker one. + """ + _patched_bo_step_deps(monkeypatch, candidate=0.4) + + D = { + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[1.0]], dtype=torch.double), + } + B = {0: torch.tensor([[0.2]], dtype=torch.double)} + + x, y, *_rest, dist = bo_mod.BO_step( + D=D, + B=B, + f=lambda _x: torch.tensor([[0.6]], dtype=torch.double), + k=object(), + acqf='UCB', + lock=_DummyLock(), + worker_id=0, + ) + + # The step still completes and proposes its candidate. + assert x[0, 0].item() == pytest.approx(0.4) + assert y[0, 0].item() == pytest.approx(0.6) + # Undefined, not zero: a zero would read as another worker sitting exactly + # on this candidate and would suppress the diversity term. + assert dist is None + + # Edge case: an entirely empty busy map behaves the same way. + _x2, _y2, *_rest2, dist2 = bo_mod.BO_step( + D=D, + B={}, + f=lambda _x: torch.tensor([[0.6]], dtype=torch.double), + k=object(), + acqf='UCB', + lock=_DummyLock(), + worker_id=0, + ) + assert dist2 is None diff --git a/tests/inference/test_inference.py b/tests/inference/test_inference.py index 228289408..a75997d9b 100644 --- a/tests/inference/test_inference.py +++ b/tests/inference/test_inference.py @@ -10,6 +10,7 @@ from __future__ import annotations import multiprocessing as mp +from pathlib import Path import pytest import toml @@ -22,9 +23,12 @@ pytest.importorskip('gpytorch') import proteus.inference.inference as inference_mod # noqa: E402 +from proteus.config import UnknownConfigKeyError # noqa: E402 pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] +BASE_CONFIG = str(Path(__file__).parent / 'base.toml') + # Pytest can hang on process completion when using multiprocessing by default. mp.set_start_method('spawn', force=True) @@ -140,6 +144,187 @@ def fake_run_inference(cfg): assert set(observed['config'].keys()) == set(expected.keys()) +# ============================================================================ +# Reference-config validation before any worker is launched +# ============================================================================ + + +@pytest.mark.unit +def test_parameter_bounds_converts_pairs_and_rejects_malformed_ranges(): + """``parameter_bounds`` accepts an increasing pair of numbers and returns + it as floats, and rejects every other shape a user could write: a single + value, a non-numeric entry, a decreasing pair, and the degenerate pair + where the two ends coincide and the parameter has no range to search. + """ + converted = inference_mod.parameter_bounds( + {'planet.mass_tot': [1, 3], 'interior_struct.core_frac': (0.3, 0.7)} + ) + assert converted['planet.mass_tot'] == (pytest.approx(1.0), pytest.approx(3.0)) + # Integer TOML literals must arrive as floats, matching the values the + # optimiser writes back into each worker's config. + assert all(isinstance(v, float) for v in converted['planet.mass_tot']) + assert converted['interior_struct.core_frac'] == ( + pytest.approx(0.3), + pytest.approx(0.7), + ) + + with pytest.raises(ValueError, match='pair of numbers'): + inference_mod.parameter_bounds({'planet.mass_tot': [1.0]}) + with pytest.raises(ValueError, match='pair of numbers'): + inference_mod.parameter_bounds({'planet.mass_tot': 'auto'}) + with pytest.raises(ValueError, match='must increase'): + inference_mod.parameter_bounds({'planet.mass_tot': [3.0, 1.0]}) + # Edge case: coincident bounds are a zero-width range, not a fixed value. + with pytest.raises(ValueError, match='must increase'): + inference_mod.parameter_bounds({'planet.mass_tot': [2.0, 2.0]}) + # Edge case: TOML admits `inf`, which satisfies "increases" and clears the + # schema's own range checks, then makes every unnormalised sample infinite. + with pytest.raises(ValueError, match='must be finite'): + inference_mod.parameter_bounds({'planet.mass_tot': [1.0, float('inf')]}) + with pytest.raises(ValueError, match='must be finite'): + inference_mod.parameter_bounds({'planet.mass_tot': [float('nan'), 3.0]}) + + +@pytest.mark.unit +def test_validate_reference_config_accepts_a_runnable_sweep(): + """A reference config that PROTEUS accepts, swept over parameters that stay + inside the schema at both ends, passes validation. Each accepted sweep is + paired with a neighbouring rejected one, so a validator gutted to an + immediate return fails this test rather than passing it. + """ + bounds = {'planet.mass_tot': [0.7, 3.0], 'interior_struct.core_frac': [0.3, 0.9]} + inference_mod.validate_reference_config(BASE_CONFIG, bounds) + # Liveness: widening one range past the schema limit must be refused, which + # proves the accepted case above was actually checked. + with pytest.raises(ValueError) as excinfo: + inference_mod.validate_reference_config( + BASE_CONFIG, {**bounds, 'interior_struct.core_frac': [0.3, 1.5]} + ) + assert 'core_frac' in str(excinfo.value) + # Only the widened range is at fault; the untouched one must not be named. + assert 'mass_tot' not in str(excinfo.value) + + # Edge case: an empty sweep still validates the file itself, so a broken + # reference config is caught even when nothing is being optimised. + inference_mod.validate_reference_config(BASE_CONFIG, {}) + + +@pytest.mark.unit +def test_validate_reference_config_rejects_a_mistyped_parameter_name(): + """A parameter name that no config field matches is reported as an + unrecognised key. Without this check the name would be written into each + worker's config as a new orphan section and every worker would refuse to + start, midway through the study. + """ + bounds = {'planet.mass_tott': [0.7, 3.0]} + with pytest.raises(UnknownConfigKeyError) as excinfo: + inference_mod.validate_reference_config(BASE_CONFIG, bounds) + + message = str(excinfo.value) + assert 'planet.mass_tott' in message + # The key is absent from the file on disk, so the message must say the + # sweep introduced it rather than blaming the reference config alone. + assert 'bounds' in message + # Discrimination: the correctly spelled name must not be flagged, which + # would happen if the walk reported every swept key rather than orphans. + assert 'planet.mass_tot"' not in message + + +@pytest.mark.unit +def test_validate_reference_config_rejects_a_bound_outside_the_schema_range(): + """A range whose upper end leaves the interval the schema allows is + rejected, and the message names the end that failed. ``core_frac`` is + constrained to the open interval (0, 1), so 0.3 is accepted and 1.5 is + not; only a check at both ends of the range catches this. + """ + with pytest.raises(ValueError, match='upper bounds'): + inference_mod.validate_reference_config( + BASE_CONFIG, {'interior_struct.core_frac': [0.3, 1.5]} + ) + # The same fault at the other end is attributed to the other end. + with pytest.raises(ValueError, match='lower bounds'): + inference_mod.validate_reference_config( + BASE_CONFIG, {'interior_struct.core_frac': [-0.2, 0.9]} + ) + # Discrimination: a range wholly inside (0, 1) must pass, so the failures + # above come from the bounds and not from the reference config itself. + inference_mod.validate_reference_config( + BASE_CONFIG, {'interior_struct.core_frac': [0.3, 0.9]} + ) + + +@pytest.mark.unit +def test_validate_reference_config_rejects_a_faulty_reference_file(tmp_path): + """A fault in the reference config itself is attributed to the file, not + to the parameter sweep, so the user knows which file to edit. + """ + raw = toml.load(BASE_CONFIG) + raw['planet']['mass_tott'] = 1.0 + faulty = tmp_path / 'faulty.toml' + faulty.write_text(toml.dumps(raw), encoding='utf-8') + + with pytest.raises(UnknownConfigKeyError) as excinfo: + inference_mod.validate_reference_config(str(faulty), {'planet.mass_tot': [0.7, 3.0]}) + + message = str(excinfo.value) + assert 'planet.mass_tott' in message + # Attribution: the file is named without the bounds qualifier, which is + # only appended when the sweep is what introduced the fault. + assert f'in {faulty}:' in message + + +@pytest.mark.unit +def test_run_inference_validates_reference_config_before_emptying_output(monkeypatch, tmp_path): + """``run_inference`` validates the reference config before it empties the + study output folder and before it generates any initial design. Re-running + a finished study with a typo'd parameter name must cost the user neither + simulation time nor the previous study's results. + """ + config = { + 'output': 'unit_inference', + 'logging': 'INFO', + 'n_workers': 1, + 'ref_config': BASE_CONFIG, + 'n_steps': 1, + 'kernel': 'MAT3/2', + 'acqf': 'LogEI', + 'seed': 1, + 'observables': {'P_surf': 1.0}, + 'parameters': {'planet.mass_tott': [0.7, 3.0]}, + } + # Stand in for a completed earlier study occupying the same output folder. + output_root = tmp_path / 'output' + output_root.mkdir() + previous = output_root / 'init.csv' + previous.write_text('x_0,y\n0.5,1.0\n', encoding='utf-8') + + monkeypatch.setattr( + inference_mod, + 'get_proteus_directories', + lambda _output: {'output': str(output_root), 'proteus': ''}, + ) + # `safe_rm` is deliberately left real: the point of the test is that it + # never runs. + monkeypatch.setattr(inference_mod, 'setup_logger', lambda **_kwargs: None) + monkeypatch.setattr(inference_mod, 'str_time', lambda: '2026-04-30 00:00:00 UTC') + monkeypatch.setattr(inference_mod.os, 'cpu_count', lambda: 8) + + create_init_calls: list = [] + monkeypatch.setattr( + inference_mod, 'create_init', lambda *a, **kw: create_init_calls.append((a, kw)) + ) + + with pytest.raises(UnknownConfigKeyError, match='planet.mass_tott'): + inference_mod.run_inference(config) + # Ordering: the guard must fire before the initial design is generated. + assert create_init_calls == [] + # ...and before the output folder is emptied. A guard placed after the + # `safe_rm` call would leave this file deleted. + assert previous.read_text(encoding='utf-8') == 'x_0,y\n0.5,1.0\n' + # Nothing downstream of the guard may have run at all. + assert not (output_root / 'ref_config.toml').exists() + + # ============================================================================ # Regression: no stray prints + docstring uses current schema # ============================================================================ diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 917291ac1..bb13b8a0b 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -8,6 +8,8 @@ from __future__ import annotations +import logging +import pickle import subprocess import pandas as pd @@ -79,6 +81,32 @@ def test_update_toml_updates_nested_keys(tmp_path): assert loaded['new']['branch']['leaf'] == 3 +@pytest.mark.unit +def test_apply_nested_updates_mutates_in_place_and_rejects_value_paths(): + """``apply_nested_updates`` writes dotted keys into the dict it was given, + creating the sections a new key needs, and refuses a path that descends + through an entry holding a value. The refusal matters because a swept + parameter name is user-supplied: ``planet.mass_tot.value`` would otherwise + fail with an attribute error naming nothing the user wrote. + """ + config = {'section': {'value': 1}} + returned = objective_mod.apply_nested_updates( + config, {'section.value': 2, 'new.branch.leaf': 3} + ) + assert config['section']['value'] == 2 + assert config['new']['branch']['leaf'] == 3 + # In-place: the same object is handed back, so a caller holding the + # original reference sees the updates. + assert returned is config + + with pytest.raises(ValueError, match="'section.value' holds a value"): + objective_mod.apply_nested_updates(config, {'section.value.deeper': 4}) + # The refused key is not written. Updates are applied as they are walked, + # so an earlier key in the same call would already have been applied; this + # pins only that the rejected one was not. + assert config['section']['value'] == 2 + + @pytest.mark.unit def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): """``run_proteus`` handles the escaped-atmosphere case (P_surf=0): @@ -118,7 +146,11 @@ def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): assert obs['P_surf'] == pytest.approx(0.0) assert obs['atm_kg_per_mol'] == pytest.approx(0.0) assert len(updates) == 2 - assert status == 20 + # No status file was written, which is reported as such rather than as a + # generic error: a run that dies during start-up and a run that reaches + # the main loop and fails there call for different investigations. + assert status == objective_mod.STATUS_MISSING + assert status != 20 @pytest.mark.unit @@ -164,40 +196,69 @@ def _fake_run(*args, **kwargs): @pytest.mark.unit def test_run_proteus_raises_when_command_fails(monkeypatch, tmp_path): - """A non-zero exit from the proteus binary is wrapped as - ``RuntimeError`` with an 'exit code N' message; the exit code is - surfaced so the caller can diagnose the failure mode. + """A non-zero exit from the proteus binary is reported as a + ``ProteusRunFailure`` naming the run, its exit code, and the status the + simulator recorded for itself, so the failure mode can be diagnosed + without opening the study by hand. """ out_abs = tmp_path / 'sim' out_abs.mkdir(parents=True) + # The simulator recorded an atmosphere-model error before exiting. The + # report must carry this, not a code inferred from the exit status. + (out_abs / 'status').write_text('22\nError (Atmosphere model)\n', encoding='utf-8') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} ) monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) - monkeypatch.setattr( - objective_mod.subprocess, - 'run', - lambda *args, **kwargs: (_ for _ in ()).throw( - subprocess.CalledProcessError(returncode=3, cmd=['proteus']) - ), - ) - with pytest.raises(RuntimeError, match='exit code 3') as excinfo: + def _fake_run(*_args, **kwargs): + # The real simulator writes to the stream it is handed before it dies. + kwargs['stdout'].write('boom\n') + kwargs['stdout'].flush() + raise subprocess.CalledProcessError(returncode=3, cmd=['proteus']) + + monkeypatch.setattr(objective_mod.subprocess, 'run', _fake_run) + + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: objective_mod.run_proteus( - parameters={}, + parameters={'planet.mass_tot': 1.25}, worker=0, iter=0, observables=['P_surf'], ref_config='reference.toml', output='dummy_output', ) + failure = excinfo.value # Cause-preservation guard: the original CalledProcessError must be # chained via __cause__ so the operator sees the failing command. - assert isinstance(excinfo.value.__cause__, subprocess.CalledProcessError) + assert isinstance(failure.__cause__, subprocess.CalledProcessError) # Exit-code-fidelity guard: a regression that always reported # 'exit code 0' or hardcoded a different code would still pass a # plain regex match if loose, so pin the integer through the cause. - assert excinfo.value.__cause__.returncode == 3 + assert failure.__cause__.returncode == 3 + assert failure.exit_code == 3 + # Status fidelity: the status file is read on the failure path. A + # regression that raised before reading it would report the missing + # sentinel, and one that kept the old hard-coded fallback would report 20. + assert failure.status == 22 + assert 'Atmosphere' in failure.status_desc + # The swept parameter is named; the fixed per-run overrides are not, + # because they carry no information about which sample failed. + assert failure.parameters == {'planet.mass_tot': pytest.approx(1.25)} + assert 'params.out.path' not in failure.parameters + # Captured child output is retained for runs that die before their own + # logger exists, which is the only record such a run leaves behind. + assert 'boom' in failure.stderr_tail + # The capture is kept beside the run folder, not inside it: the simulator + # empties its own output directory once it starts, which would unlink a + # file held open there. + console = out_abs.parent / f'{out_abs.name}{objective_mod.CHILD_CONSOLE_SUFFIX}' + assert console.is_file() + assert 'boom' in console.read_text(encoding='utf-8') + assert not (out_abs / console.name).exists() + rendered = failure.report() + assert 'worker=0 iter=0' in rendered + assert 'planet.mass_tot=1.25' in rendered @pytest.mark.unit @@ -245,7 +306,7 @@ def test_run_proteus_raises_on_missing_observable(monkeypatch, tmp_path): output='dummy_output', ) assert obs['P_surf'] == pytest.approx(1.0) - assert status == 20 + assert status == objective_mod.STATUS_MISSING @pytest.mark.unit @@ -368,3 +429,612 @@ def fake_J(x, **kwargs): assert captured['x'][0, 0].item() == pytest.approx(1.0) assert captured['x'][0, 1].item() == pytest.approx(1.5) + + +# ============================================================================ +# Failure reporting for a single simulator run +# ============================================================================ + + +@pytest.mark.unit +def test_run_proteus_failure_distinguishes_a_missing_status_from_a_generic_error( + monkeypatch, tmp_path +): + """A run that dies before writing a status file is reported as having + written none, rather than as a generic configuration error. The two call + for different investigations: the first points at the simulator's start-up + (environment, reference data), the second at the model configuration. + """ + out_abs = tmp_path / 'sim' + out_abs.mkdir(parents=True) + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} + ) + monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) + + def _fake_run(*_args, **kwargs): + kwargs['stdout'].write('Error: no\n') + kwargs['stdout'].flush() + raise subprocess.CalledProcessError(returncode=1, cmd=['proteus']) + + monkeypatch.setattr(objective_mod.subprocess, 'run', _fake_run) + + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=3, + iter=4, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + failure = excinfo.value + assert failure.status == objective_mod.STATUS_MISSING + assert 'no readable status file' in failure.status_desc + # Discrimination: the previous behaviour reported code 20 for this case, + # which reads as a configuration fault the user does not have. + assert failure.status != 20 + assert 'Generic' not in failure.status_desc + # No logfile exists either, so the report must not invent one. + assert failure.log_path is None + assert 'logfile' not in failure.report() + + # Edge case: the same run with a status file present reports that status, + # which proves the sentinel above came from the absent file and not from a + # reader that always fails. + (out_abs / 'status').write_text('21\nError (Interior model)\n', encoding='utf-8') + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=3, + iter=4, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + assert excinfo.value.status == 21 + + +@pytest.mark.unit +def test_run_proteus_failure_points_at_the_simulator_logfile(monkeypatch, tmp_path): + """When the failed run left a logfile, the report names it. That file holds + the traceback the simulator captured for itself, and is the only place the + cause of a mid-run crash is recorded. + """ + out_abs = tmp_path / 'sim' + out_abs.mkdir(parents=True) + (out_abs / 'proteus_00.log').write_text('early\n', encoding='utf-8') + (out_abs / 'proteus_01.log').write_text('CRITICAL Uncaught exception\n', encoding='utf-8') + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} + ) + monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) + monkeypatch.setattr( + objective_mod.subprocess, + 'run', + lambda *args, **kwargs: (_ for _ in ()).throw( + subprocess.CalledProcessError(returncode=1, cmd=['proteus']) + ), + ) + + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + # The newest logfile is the one the failed run wrote; an earlier one + # belongs to a previous attempt in the same folder. + assert excinfo.value.log_path.endswith('proteus_01.log') + assert 'proteus_01.log' in excinfo.value.report() + + +@pytest.mark.unit +def test_run_proteus_reports_a_clean_exit_that_produced_no_output(monkeypatch, tmp_path): + """A run that exits zero but writes no readable helpfile is reported as a + failed sample rather than crashing the study with a bare parser error. The + exit code is recorded as zero so the report does not suggest a crash. + """ + out_abs = tmp_path / 'sim' + out_abs.mkdir(parents=True) + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} + ) + monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) + monkeypatch.setattr(objective_mod.subprocess, 'run', lambda *args, **kwargs: None) + + # No helpfile at all. + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + assert excinfo.value.exit_code == 0 + assert 'no readable output' in excinfo.value.reason + + # Edge case: a helpfile that exists but holds no rows. + (out_abs / 'runtime_helpfile.csv').write_text('', encoding='utf-8') + with pytest.raises(objective_mod.ProteusRunFailure): + objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + + # Discrimination: a helpfile with a usable row completes normally, so the + # two failures above come from the output and not from an unconditional + # raise on this code path. + pd.DataFrame([{'P_surf': 2.5}]).to_csv( + out_abs / 'runtime_helpfile.csv', sep=' ', index=False + ) + obs, _status = objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + assert obs['P_surf'] == pytest.approx(2.5) + + +@pytest.mark.unit +def test_proteus_run_failure_survives_the_trip_back_from_a_pool_worker(): + """A failure raised inside a pool worker is pickled and re-raised in the + parent process. Every reported field must survive that round trip, or the + parent sees a reconstruction error in place of the diagnosis. + """ + original = objective_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=2, + iter=7, + out_dir='/study/workers/w_2/i_7', + exit_code=1, + status=27, + log_path='/study/workers/w_2/i_7/proteus_00.log', + stderr_tail='Error: outgassing failed', + parameters={'planet.mass_tot': 2.0}, + ) + restored = pickle.loads(pickle.dumps(original)) + + assert isinstance(restored, objective_mod.ProteusRunFailure) + assert restored.report() == original.report() + # Field-level guard: an equal report could still hide a dropped field that + # the renderer omits when empty, so pin the values that steer diagnosis. + assert restored.status == 27 + assert restored.worker == 2 and restored.iter == 7 + assert restored.parameters == {'planet.mass_tot': pytest.approx(2.0)} + assert restored.log_path == original.log_path + assert restored.category == objective_mod.CATEGORY_FAILURE + + # The category rides along in the same tuple, and it decides whether the + # parent calls the run a fault. A field dropped from the reconstruction + # would fall back to the 'failure' default and go unnoticed on a failure, + # so the round trip is checked on the other value too. + excluded = objective_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=2, + iter=7, + out_dir='/study/workers/w_2/i_7', + exit_code=0, + status=11, + category=objective_mod.CATEGORY_EXCLUDED, + ) + restored_excluded = pickle.loads(pickle.dumps(excluded)) + assert restored_excluded.category == objective_mod.CATEGORY_EXCLUDED + assert 'excluded for worker=2' in restored_excluded.report() + assert 'failed for worker=2' not in restored_excluded.report() + + +@pytest.mark.unit +def test_J_scores_a_failed_run_badly_and_keeps_the_study_running(monkeypatch, tmp_path, caplog): + """A parameter combination the simulator cannot integrate is scored as a + poor sample so the sweep continues, and the failure is reported once in + full and recorded for the end-of-study tally. Aborting instead would end a + study on the first unphysical corner of the parameter box. + """ + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} + ) + failure = objective_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=1, + iter=2, + out_dir='/study/workers/w_1/i_2', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 3.0}, + ) + + def _fail(**_kwargs): + raise failure + + monkeypatch.setattr(objective_mod, 'run_proteus', _fail) + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + + with caplog.at_level('WARNING'): + value = objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'P_surf': 1.0}, + worker=1, + iter=2, + output='dummy_output', + ref_config='reference.toml', + ) + + assert value.shape == (1, 1) + assert value.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) + # The score must be far below any value a successful run can produce, or + # the optimiser would be drawn toward the region that fails. + assert value.item() < -10.0 + # Reported once, in full: the status description and the output folder are + # what let the user find the run. + reported = '\n'.join(record.getMessage() for record in caplog.records) + assert 'Interior model' in reported + assert '/study/workers/w_1/i_2' in reported + + # The same failure is left on disk for the end-of-study tally, because a + # log line scrolls past and a study that failed mostly needs a count. + recorded = objective_mod.read_failure_records(tmp_path) + assert [(r['worker'], r['iter'], r['status']) for r in recorded] == [(1, 2, 21)] + assert recorded[0]['parameters']['planet.mass_tot'] == pytest.approx(3.0) + + # Opting in turns the same failure into a hard stop. `set_abort_on_failure` + # is the writer under test; monkeypatch restores the variable afterwards. + objective_mod.set_abort_on_failure(True) + with pytest.raises(objective_mod.ProteusRunFailure): + objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'P_surf': 1.0}, + worker=1, + iter=2, + output='dummy_output', + ref_config='reference.toml', + ) + + +@pytest.mark.unit +def test_J_scores_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_path, caplog): + """A run that exits cleanly but records an error status is scored badly and + named in the log. Status 25 is the only error code reachable this way: it + is written when a run is stopped through its keepalive file, and the + simulator then terminates normally. + + 'R_obs' is used as the observable because it is compared linearly, which + gives the exact-match objective a closed form to pin against. + """ + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} + ) + monkeypatch.setattr( + objective_mod, + 'run_proteus', + lambda **_kwargs: ({'R_obs': 9.25e6}, 25), + ) + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + + with caplog.at_level('WARNING'): + value = objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=0, + iter=0, + output='dummy_output', + ref_config='reference.toml', + ) + assert value.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) + assert 'status 25' in '\n'.join(r.getMessage() for r in caplog.records) + # Counted in the end-of-study tally alongside the runs that crashed. A + # tally that covered only crashes would understate a study stopped by hand. + recorded = objective_mod.read_failure_records(tmp_path) + assert [r['status'] for r in recorded] == [25] + assert recorded[0]['exit_code'] == 0 + + # Discrimination: the same observables under a completed status (13, + # "target time reached") are scored normally, which rules out a regression + # that returns the failure score for every run. + monkeypatch.setattr( + objective_mod, + 'run_proteus', + lambda **_kwargs: ({'R_obs': 9.25e6}, 13), + ) + good = objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=0, + iter=0, + output='dummy_output', + ref_config='reference.toml', + failure_codes=[], + ) + # Closed form for an exact match on a linear observable: the normalised + # difference is zero, so sq_dist is zero and the score is + # -log10(0 + EPS_CLIP) = -log10(1e-10) = 10. + assert good.item() == pytest.approx(10.0, rel=1e-9) + # Sign guard: a flipped objective would land at -10, which is still above + # BAD_OBJ_VALUE and would pass a bare "better than failure" assertion. + assert good.item() > 0 + # Scale guard: the failure score is -20, so the two are far apart. + assert good.item() - objective_mod.BAD_OBJ_VALUE > 25.0 + + +@pytest.mark.unit +def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): + """The failure range covers the error statuses the simulator can record. + Code 28 is the highest error the status table defines; 29 is a completion + ('planet evaporated') and no current code path writes it, so it must not + be scored as a failure by an off-by-one in the range bound. + """ + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} + ) + + def _score(status): + monkeypatch.setattr( + objective_mod, + 'run_proteus', + lambda **_kwargs: ({'R_obs': 9.25e6}, status), + ) + return objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=0, + iter=0, + output='dummy_output', + ref_config='reference.toml', + ).item() + + # Highest defined error code, and the escape-model error below it. + assert _score(28) == pytest.approx(objective_mod.BAD_OBJ_VALUE) + assert _score(21) == pytest.approx(objective_mod.BAD_OBJ_VALUE) + # Completion codes are scored on their observables. + assert _score(29) == pytest.approx(10.0, rel=1e-9) + assert _score(13) == pytest.approx(10.0, rel=1e-9) + # A run that never updated its status past 'Running' died mid-flight. + assert _score(1) == pytest.approx(objective_mod.BAD_OBJ_VALUE) + # An unreadable status file is treated as a failure, because the run's own + # account of itself is missing and its output cannot be trusted. + assert _score(objective_mod.STATUS_MISSING) == pytest.approx(objective_mod.BAD_OBJ_VALUE) + + +@pytest.mark.unit +def test_J_separates_an_excluded_outcome_from_a_failed_run(monkeypatch, tmp_path, caplog): + """A status named in `failure_codes` marks an outcome the study does not fit + against, not a fault. A run stopped by its clock limit (status 11) completed + normally, so it is scored as a poor sample and reported at info level, while + an error status (21, interior model) is reported as a run that produced + nothing usable. Reporting the first as the second sends the user looking for + a bug in a run that did exactly what it was configured to do. + """ + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} + ) + + def _score(status, worker): + monkeypatch.setattr( + objective_mod, + 'run_proteus', + lambda **_kwargs: ({'R_obs': 9.25e6}, status), + ) + return objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=worker, + iter=0, + output='dummy_output', + ref_config='reference.toml', + failure_codes=[11], + ).item() + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.objective'): + excluded = _score(11, worker=0) + + # The optimiser must still be steered away from the excluded region, so the + # score is the same one a failure carries. + assert excluded == pytest.approx(objective_mod.BAD_OBJ_VALUE) + assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] + reported = '\n'.join(r.getMessage() for r in caplog.records) + assert 'excludes' in reported + assert 'maximum clock runtime' in reported + assert 'failure state' not in reported + assert 'did not produce a usable result' not in reported + + # The record is kept for the end-of-study tally, labelled so the tally can + # count it apart from the runs that genuinely failed. + recorded = objective_mod.read_failure_records(tmp_path) + assert [(r['status'], r['category']) for r in recorded] == [ + (11, objective_mod.CATEGORY_EXCLUDED) + ] + + # Discrimination: an error status under the same call is still a failure, + # warned about and recorded under the other category. Without this the test + # would pass against a regression that labelled every run 'excluded'. + caplog.clear() + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.objective'): + failed = _score(21, worker=1) + assert failed == pytest.approx(objective_mod.BAD_OBJ_VALUE) + warnings = [r for r in caplog.records if r.levelname == 'WARNING'] + assert len(warnings) == 1 + assert 'did not produce a usable result' in warnings[0].getMessage() + recorded = objective_mod.read_failure_records(tmp_path) + assert [(r['status'], r['category']) for r in recorded] == [ + (11, objective_mod.CATEGORY_EXCLUDED), + (21, objective_mod.CATEGORY_FAILURE), + ] + + # Discrimination: a completion status that the study does not exclude is + # scored on its observables and leaves no record at all. The exact match on + # a linear observable has the closed form -log10(0 + 1e-10) = 10. + assert _score(13, worker=2) == pytest.approx(10.0, rel=1e-9) + assert len(objective_mod.read_failure_records(tmp_path)) == 2 + + +@pytest.mark.unit +def test_run_proteus_treats_a_missing_helpfile_column_as_a_setup_fault(monkeypatch, tmp_path): + """A helpfile with no 'P_surf' column raises rather than producing a scored + failure. The column set is a property of the simulator's output schema, so + it is the same for every run: scoring it as a bad sample would let the + study spend its whole budget returning the failure value and then report + success at the end. + """ + out_abs = tmp_path / 'sim' + out_abs.mkdir(parents=True) + pd.DataFrame([{'T_obs': 300.0}]).to_csv( + out_abs / 'runtime_helpfile.csv', sep=' ', index=False + ) + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} + ) + monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) + monkeypatch.setattr(objective_mod.subprocess, 'run', lambda *args, **kwargs: None) + + with pytest.raises(KeyError) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['T_obs'], + ref_config='reference.toml', + output='dummy_output', + ) + # Not a per-run failure: scoring this one badly and continuing is exactly + # the behaviour the raise exists to prevent. + assert not isinstance(excinfo.value, objective_mod.ProteusRunFailure) + assert 'P_surf' in str(excinfo.value) + assert 'every run' in str(excinfo.value) + + # Discrimination: the same helpfile with the column present completes, so + # the raise comes from the missing column and not from this code path + # rejecting every input. + pd.DataFrame([{'T_obs': 300.0, 'P_surf': 5.0}]).to_csv( + out_abs / 'runtime_helpfile.csv', sep=' ', index=False + ) + obs, _status = objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['T_obs'], + ref_config='reference.toml', + output='dummy_output', + ) + assert obs['T_obs'] == pytest.approx(300.0) + + +# ============================================================================ +# Failure records: written per evaluation, read back for the study summary +# ============================================================================ + + +@pytest.mark.unit +def test_run_output_dir_names_the_folder_the_simulator_is_given(monkeypatch, tmp_path): + """The per-evaluation folder is derived in one place, so the path a failure + report names is the path the simulator was told to write to. Initial + samples use worker -1, which must survive the same construction. + """ + monkeypatch.setattr( + objective_mod, + 'get_proteus_directories', + lambda path: {'output': str(tmp_path / path)}, + ) + + rel, absolute = objective_mod.run_output_dir('study', 2, 7) + assert rel.as_posix() == 'study/workers/w_2/i_7' + assert absolute == tmp_path / 'study' / 'workers' / 'w_2' / 'i_7' + + # Initial sampling identifies itself with worker -1 rather than a worker + # index, and must land in its own folder rather than colliding with w_1. + rel_init, _ = objective_mod.run_output_dir('study', -1, 7) + assert rel_init.as_posix() == 'study/workers/w_-1/i_7' + assert rel_init != rel + + +@pytest.mark.unit +def test_failure_records_round_trip_without_colliding_between_workers(tmp_path): + """Records are written one file per evaluation so that concurrent workers + never contend for the same file, and are read back ordered by worker then + iteration. The status description is stored rather than recomputed, so the + summary does not have to re-derive it from the code. + """ + first = objective_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=5, + out_dir='/study/workers/w_0/i_5', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 3.0}, + ) + # Same iteration, different worker: the pair is what makes the name unique. + second = objective_mod.ProteusRunFailure( + reason='exceeded the 3600.0 s timeout', + worker=1, + iter=5, + out_dir='/study/workers/w_1/i_5', + status=objective_mod.STATUS_MISSING, + parameters={'planet.mass_tot': 4.0}, + ) + + assert objective_mod.record_failure(tmp_path, second) is not None + assert objective_mod.record_failure(tmp_path, first) is not None + written = sorted(p.name for p in (tmp_path / objective_mod.FAILURE_RECORD_DIR).iterdir()) + assert written == ['w0_i5.json', 'w1_i5.json'] + + records = objective_mod.read_failure_records(tmp_path) + # Ordering guard: written second-then-first, read back in worker order. + assert [r['worker'] for r in records] == [0, 1] + assert records[0]['status'] == 21 + assert records[0]['status_desc'] == first.status_desc + assert records[0]['parameters']['planet.mass_tot'] == pytest.approx(3.0) + # A run that never wrote a status file is stored as such, not as a generic + # error, so the summary can separate start-up deaths from model faults. + assert records[1]['status'] == objective_mod.STATUS_MISSING + assert 'no readable status file' in records[1]['status_desc'] + + # Edge case: a corrupt record is skipped rather than aborting the summary, + # which would hide the records that did parse. + (tmp_path / objective_mod.FAILURE_RECORD_DIR / 'w2_i0.json').write_text('{not json') + assert len(objective_mod.read_failure_records(tmp_path)) == 2 + + +@pytest.mark.unit +def test_recording_a_failure_never_masks_the_failure_it_records(tmp_path): + """Bookkeeping must not bring down a study. When the record cannot be + written the writer reports that it could not, and the caller still has the + failure in hand to log and to score. + """ + blocked = tmp_path / 'not_a_directory' + blocked.write_text('this is a file, so no folder can be made beneath it') + failure = objective_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=0, + out_dir=str(tmp_path), + exit_code=1, + status=21, + ) + + assert objective_mod.record_failure(blocked, failure) is None + # Discrimination: the same failure records fine against a usable folder, so + # the None above came from the blocked path and not from a writer that + # always fails. + assert objective_mod.record_failure(tmp_path / 'study', failure) is not None + # Reading a study that never created the folder is empty, not an error. + assert objective_mod.read_failure_records(tmp_path / 'never_ran') == [] diff --git a/tests/inference/test_transforms.py b/tests/inference/test_transforms.py index 02cc44dae..d9b3a5435 100644 --- a/tests/inference/test_transforms.py +++ b/tests/inference/test_transforms.py @@ -345,11 +345,12 @@ def test_set_child_timeout_stores_in_env(monkeypatch): def test_run_proteus_wraps_timeout_as_runtime_error(monkeypatch, tmp_path): - """subprocess.TimeoutExpired is wrapped as RuntimeError with a 'timed out' - message so the inference harness receives a consistent error type. + """A run that exceeds its time limit is reported as a ProteusRunFailure + naming the limit, so the harness receives one error type for every fault + that is specific to a single run. Discrimination: a regression that re-raised the raw TimeoutExpired would - break the except-RuntimeError handler in the BO worker loop. + escape the handler in the objective wrapper and kill the worker. """ import subprocess @@ -365,7 +366,7 @@ def _fake_run(*args, **kwargs): monkeypatch.setattr(obj_mod.subprocess, 'run', _fake_run) - with pytest.raises(RuntimeError, match='timed out') as exc_info: + with pytest.raises(obj_mod.ProteusRunFailure, match='timeout') as exc_info: obj_mod.run_proteus( parameters={}, worker=0, @@ -375,6 +376,12 @@ def _fake_run(*args, **kwargs): output='dummy_output', ) assert isinstance(exc_info.value.__cause__, subprocess.TimeoutExpired) + # A wedged run never returns an exit code, so the report must omit it + # rather than invent one that would read as a crash. + assert exc_info.value.exit_code is None + # The failure stays a RuntimeError, which is what the surrounding code + # and any existing caller catches. + assert isinstance(exc_info.value, RuntimeError) # -------------------------------------------------------------------------- diff --git a/tests/inference/test_utils_branches.py b/tests/inference/test_utils_branches.py index 2c6b3e675..9a905c8ce 100644 --- a/tests/inference/test_utils_branches.py +++ b/tests/inference/test_utils_branches.py @@ -16,6 +16,7 @@ import logging import numpy as np +import pandas as pd import pytest # The Bayesian-optimisation stack ships as the optional `inference` extra, @@ -294,3 +295,208 @@ def test_get_kernel_raises_for_unknown_kernel_name(): # Edge: case-sensitive — 'rbf' is not 'RBF'. with pytest.raises(ValueError, match='Unknown kernel'): get_kernel('rbf', d=2) + + +# --------------------------------------------------------------------------- +# Accounting for the evaluations that failed +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_print_results_counts_unscored_runs_and_refuses_a_study_with_no_fit(tmp_path, caplog): + """Evaluations that failed, and those that completed on an excluded status, + both carry the failure score rather than a fit quality, so the summary says + how many of them there were. The objective value alone cannot tell the two + apart, so the wording covers both. When no optimisation evaluation produced + a fit quality there is no best fit at all, and the summary stops rather than + reporting the least-bad run as an inference result. + """ + from proteus.inference.objective import BAD_OBJ_VALUE + from proteus.inference.utils import print_results + + _make_worker_dir(tmp_path, worker=0, iteration=0, obs_value=0.1, param_value=0.5) + _make_worker_dir(tmp_path, worker=0, iteration=1, obs_value=0.9, param_value=1.0) + best_dir = _make_worker_dir(tmp_path, worker=0, iteration=2, obs_value=0.5, param_value=1.5) + + logs = [ + {'worker': 0, 'task_id': 0}, + {'worker': 0, 'task_id': 1}, + {'worker': 0, 'task_id': 2}, + ] + config = { + 'observables': {'H2O_vmr': 0.9}, + 'parameters': {'planet.mass_tot': [0.5, 1.5]}, + } + + # One of the two optimisation evaluations failed; the other is still the + # best fit and must be reported normally. + D = { + 'X': torch.tensor([[0.0], [1.0], [0.5]]), + 'Y': torch.tensor([[-1.0], [BAD_OBJ_VALUE], [2.0]]), + } + with caplog.at_level(logging.WARNING, logger='fwl.proteus.inference.utils'): + result = print_results(D, logs, config, str(tmp_path), n_init=1) + assert str(best_dir / 'init_coupler.toml') == str(result) + assert any( + '1 of 2 optimisation evaluations carry the failure score' in r.message + for r in caplog.records + ) + + # Discrimination: the same study with no failure score present reports no + # count, so the message above tracks the data and is not emitted always. + caplog.clear() + D_clean = { + 'X': torch.tensor([[0.0], [1.0], [0.5]]), + 'Y': torch.tensor([[-1.0], [1.0], [2.0]]), + } + with caplog.at_level(logging.WARNING, logger='fwl.proteus.inference.utils'): + print_results(D_clean, logs, config, str(tmp_path), n_init=1) + assert not [r for r in caplog.records if 'carry the failure score' in r.message] + + # No optimisation evaluation produced a fit quality: nothing to report. + D_dead = { + 'X': torch.tensor([[0.0], [1.0], [0.5]]), + 'Y': torch.tensor([[-1.0], [BAD_OBJ_VALUE], [BAD_OBJ_VALUE]]), + } + with pytest.raises( + RuntimeError, match='None of the 2 optimisation evaluations produced a fit quality' + ): + print_results(D_dead, logs, config, str(tmp_path), n_init=1) + + +@pytest.mark.unit +def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp_path, caplog): + """The end-of-study tally turns the per-run records into one table and one + breakdown by cause, and escalates to a warning once most of the study + failed. Without the escalation, a posterior built on a handful of real + evaluations reads the same as one built on all of them. + """ + from proteus.inference.objective import ProteusRunFailure, record_failure + from proteus.inference.utils import summarise_failures + + # Two runs that died the same way and one that died differently, so the + # breakdown has something to group. + for worker, status in ((0, 21), (1, 21), (2, 24)): + record_failure( + tmp_path, + ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=1, + status=status, + parameters={'planet.mass_tot': 1.0 + worker}, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + n_failed = summarise_failures(str(tmp_path), n_attempted=20) + + assert n_failed == 3 + messages = '\n'.join(r.message for r in caplog.records) + assert '3 of 20 evaluations' in messages + # Grouped by cause, so two runs that died the same way count as one line. + assert 'Interior model' in messages + # Below the escalation threshold (3/20 = 15%), the tally is reported but + # not warned about. + assert not [r for r in caplog.records if r.levelname == 'WARNING'] + + # The table carries the swept parameter alongside the diagnosis, so the + # failing region can be located without opening each run folder. + table = pd.read_csv(tmp_path / 'failures.csv') + assert len(table) == 3 + assert list(table['worker']) == [0, 1, 2] + assert sorted(table['status']) == [21, 21, 24] + assert table['planet.mass_tot'].max() == pytest.approx(3.0) + + # Discrimination: the same three failures against a smaller study cross the + # threshold and are warned about. A tally without the escalation would log + # identically in both cases. + caplog.clear() + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + summarise_failures(str(tmp_path), n_attempted=4) + warnings = [r for r in caplog.records if r.levelname == 'WARNING'] + assert len(warnings) == 1 + # The count of evaluations that are real is what the reader needs. + assert '1 real evaluations' in warnings[0].message + + +@pytest.mark.unit +def test_summarise_failures_counts_excluded_outcomes_apart_from_failures(tmp_path, caplog): + """A run that completed on a status the study excludes is tallied, but not + as a fault. Folding the two together would tell the user that a study whose + runs all reached their clock limit, exactly as configured, is a study full + of broken simulations. + """ + from proteus.inference.objective import ( + CATEGORY_EXCLUDED, + ProteusRunFailure, + record_failure, + ) + from proteus.inference.utils import summarise_failures + + record_failure( + tmp_path, + ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=0, + out_dir='/study/workers/w_0/i_0', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 1.0}, + ), + ) + for worker in (1, 2): + record_failure( + tmp_path, + ProteusRunFailure( + reason='completed on a status this study excludes', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=0, + status=11, + parameters={'planet.mass_tot': 1.0 + worker}, + category=CATEGORY_EXCLUDED, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + n_unscored = summarise_failures(str(tmp_path), n_attempted=20) + + # Both kinds are unscored, so both count toward how much of the study was + # real, but the breakdown names them apart. + assert n_unscored == 3 + messages = '\n'.join(r.message for r in caplog.records) + assert '1 did not produce a usable result' in messages + assert '2 completed on a status this study excludes' in messages + # The clock-limit outcome is labelled in the cause table rather than being + # listed beside the interior-model error as if it were one. + assert 'Completed (maximum clock runtime) [excluded]' in messages + assert 'Error (Interior model) [excluded]' not in messages + + # Carried into the table too, so the excluded rows can be filtered out when + # looking for the region that actually breaks the simulator. + table = pd.read_csv(tmp_path / 'failures.csv') + assert sorted(table['category']) == ['excluded', 'excluded', 'failure'] + assert sorted(table.loc[table['category'] == 'excluded', 'status']) == [11, 11] + + +@pytest.mark.unit +def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_path, caplog): + """A study in which nothing failed says so and writes no table. An empty + failures.csv would suggest the accounting had run and found nothing to + say about a study that in fact had nothing to report. + """ + from proteus.inference.utils import summarise_failures + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + n_failed = summarise_failures(str(tmp_path), n_attempted=12) + + assert n_failed == 0 + assert not (tmp_path / 'failures.csv').exists() + messages = '\n'.join(r.message for r in caplog.records) + assert 'none' in messages and '12 evaluations' in messages + assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] diff --git a/tests/test_cli.py b/tests/test_cli.py index 4e9e80bd0..197ab73b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ import builtins import importlib.util +import logging from pathlib import Path import pytest @@ -561,14 +562,27 @@ def fake_download_melting_curves(configuration, clean: bool = False): assert not any(c[0] == 'zalmoxis_eos' for c in calls) +def assert_refusal_logged_as_error(caplog, key: str) -> None: + """Check that a config refusal named `key` was logged at ERROR, not below. + + The refusal reaches the terminal through the 'fwl' logger rather than + click's own stream, so the captured records are what pins it. The level is + asserted as well as the text: a refusal emitted at INFO or WARNING carries + the same words and would otherwise pass. + """ + named = [rec for rec in caplog.records if key in rec.getMessage()] + assert named, f'no log record named {key}' + assert [rec.levelno for rec in named] == [logging.ERROR] * len(named) + + @pytest.mark.unit -def test_get_interiordata_reports_an_unknown_config_key_cleanly(monkeypatch, tmp_path): +def test_get_interiordata_reports_an_unknown_config_key_cleanly(monkeypatch, tmp_path, caplog): """A misspelled key stops the download and is reported as a CLI error. The download commands act on the configuration, so acting on one whose keys were silently discarded would fetch data for a setup the user did not ask - for. The failure has to arrive in the CLI's own error style with the key - named, not as a traceback. + for. The failure has to arrive at error level with the key named, not as a + traceback. """ import tomllib @@ -593,10 +607,11 @@ def test_get_interiordata_reports_an_unknown_config_key_cleanly(monkeypatch, tmp with open(cfg, 'w') as f: tomlkit.dump(raw, f) - res = runner.invoke(cli.cli, ['get', 'interiordata', '--config-path', str(cfg)]) + with caplog.at_level(logging.INFO, logger='fwl'): + res = runner.invoke(cli.cli, ['get', 'interiordata', '--config-path', str(cfg)]) assert res.exit_code != 0 - assert 'planet.mass_total' in res.output - # A ClickException prints "Error: ..." and does not surface a traceback. + assert_refusal_logged_as_error(caplog, 'planet.mass_total') + # A ClickException does not surface a traceback. assert 'Traceback' not in res.output # The config-dependent download is not reached; the config-independent one # ahead of it may already have run, which is why only the former is pinned. @@ -1195,11 +1210,12 @@ def fake_grid_from_config(path, test_run=False): assert received[0][1] is False -def test_start_reports_an_unknown_config_key_cleanly(tmp_path): - """``proteus start`` refuses a misspelled key in the CLI's own error style. +def test_start_reports_an_unknown_config_key_cleanly(tmp_path, caplog): + """``proteus start`` refuses a misspelled key at error level. This is the command most runs go through, so a refusal that arrives as a - bare traceback leaves the name of the offending key buried in it. + bare traceback leaves the name of the offending key buried in it, and one + emitted below error level reads as ordinary progress output. """ import tomllib @@ -1214,9 +1230,10 @@ def test_start_reports_an_unknown_config_key_cleanly(tmp_path): with open(cfg, 'w') as f: tomlkit.dump(raw, f) - res = runner.invoke(cli.cli, ['start', '-c', str(cfg), '--offline']) + with caplog.at_level(logging.INFO, logger='fwl'): + res = runner.invoke(cli.cli, ['start', '-c', str(cfg), '--offline']) assert res.exit_code != 0 - assert 'planet.mass_total' in res.output + assert_refusal_logged_as_error(caplog, 'planet.mass_total') assert 'Traceback' not in res.output @@ -1242,13 +1259,13 @@ def boom(*_args, **_kwargs): assert 'something else went wrong entirely' in str(res.exception) -def test_grid_reports_an_unknown_key_in_the_base_config_cleanly(tmp_path, monkeypatch): - """``proteus grid`` refuses a base config with a misspelled key, in CLI style. +def test_grid_reports_an_unknown_key_in_the_base_config_cleanly(tmp_path, monkeypatch, caplog): + """``proteus grid`` refuses a base config with a misspelled key, at error level. Case config files are written out from the parsed base config, so an unrecognised key in the base never reaches them and the grid would otherwise run every case on a default nobody chose. The refusal has to name - the key and arrive as a CLI error, since the whole ensemble depends on it. + the key and arrive as an error, since the whole ensemble depends on it. """ import tomllib @@ -1282,10 +1299,11 @@ def test_grid_reports_an_unknown_key_in_the_base_config_cleanly(tmp_path, monkey ' values = [0.7]\n' ) - res = runner.invoke(cli.cli, ['grid', '-c', str(grid_toml), '--dry-run']) + with caplog.at_level(logging.INFO, logger='fwl'): + res = runner.invoke(cli.cli, ['grid', '-c', str(grid_toml), '--dry-run']) assert res.exit_code != 0 - assert 'params.dt.maxium' in res.output - # A ClickException prints "Error: ..."; an unwrapped raise prints a traceback. + assert_refusal_logged_as_error(caplog, 'params.dt.maxium') + # An unwrapped raise would print a traceback instead. assert 'Traceback' not in res.output From 51d0cccf1baace4b798f382ca17d95f6aa1db00a Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 16:55:07 +0200 Subject: [PATCH 14/72] update docs with softened handling of excluded outcomes (eg status 11) --- docs/How-to/inference.md | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index b412e4535..9d5ceb958 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -122,7 +122,7 @@ The system generates several outputs in: - `logs.csv`: Detailed logs of each BO step - `Ts.csv`: Timestamps for performance analysis - `init.csv`: Data used as an initial guess for starting the optimisation -- `failures.csv`: One row per simulation that did not produce a usable result, written only when at least one did fail (see [Failed simulations](#failed-simulations)) +- `failures.csv`: One row per simulation that carries the failure score instead of a fit quality, written only when there is at least one (see [Failed and excluded simulations](#failed-and-excluded-simulations)) ### Plots The BO scheme will generate many plots upon completion. @@ -150,27 +150,31 @@ The system prints the final results including: - Corresponding simulated observables - Comparison with target observables -### Failed simulations +### Failed and excluded simulations -A sweep over a wide parameter box is expected to reach combinations PROTEUS -cannot integrate. Such an evaluation is scored with a fixed bad objective value -and the study carries on, so that one unphysical corner does not end a run that -has been going for hours. +Two kinds of evaluation carry the fixed bad objective value instead of a fit +quality, and the study carries on in both cases so that one awkward corner does +not end a run that has been going for hours. -Each failed evaluation is reported as it happens, naming the status code PROTEUS -recorded, the run's output folder, the logfile holding its traceback, and the -parameter values that produced it. At the end of the study the failures are -collected into `failures.csv` and summarised: how many of the evaluations failed, -a breakdown by cause, and the first few logfiles to open. If more than half the -evaluations failed, the summary says so as a warning, because the result then -rests on far fewer real evaluations than the step count suggests. If every -optimisation evaluation failed there is no best fit to report, and the study -stops with an error rather than presenting the least-bad failure as a result. +**Failures.** A sweep over a wide parameter box is expected to reach +combinations PROTEUS cannot integrate. A run that crashes, is killed, or stops +on an error status (20 to 28) produced nothing usable. Each one is reported as a +warning when it happens, naming the status code PROTEUS recorded, the run's +output folder, the logfile holding its traceback, and the parameter values that +produced it. + +**Exclusions.** The `failure_codes` field lists completion statuses the study +does not want to fit against, such as `11` (maximum clock runtime) or `15` +(volatiles escaped). These are reported at info level. + +At the end of the study both kinds are collected into `failures.csv`, which +carries a `category` column of `failure` or `excluded`, and summarised. If more than half the evaluations went unscored, the summary says so as a warning, because the result then rests on far fewer real evaluations than the step count suggests. If no optimisation evaluation produced a fit quality there is no best fit to report, and the study stops with an error rather than presenting the least-bad run as a result. Set `abort_on_failure = true` in the inference config to stop at the first failed simulation instead. This is useful while setting a study up, when the first failure is more likely to be a mistake in the reference config than a genuinely -unrunnable corner of the parameter space. +unrunnable corner of the parameter space. It applies to failures only; an +excluded outcome never stops the study. ## Customization From 90bdbbb9ff9831c6e5ccd5a38be48ea49723085c Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 14 Sep 2026 17:08:34 +0200 Subject: [PATCH 15/72] fix worker console-log error when plotting --- src/proteus/inference/plot.py | 6 +++-- tests/inference/test_plot.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/proteus/inference/plot.py b/src/proteus/inference/plot.py index 4a742cb71..bf6418e17 100644 --- a/src/proteus/inference/plot.py +++ b/src/proteus/inference/plot.py @@ -562,8 +562,10 @@ def plot_result_correlation(pars: dict, obs: dict, directory): par_keys = list(pars.keys()) obs_keys = list(obs.keys()) - # Get directories for all cases of interest - cases = sorted((Path(directory) / 'workers').glob('w_*/i_*')) + # Get directories for all cases of interest. Filtered to directories only: + # a worker's `_console.log` capture file (or any other stray sibling) also + # matches the `i_*` glob but is not a case directory. + cases = sorted(p for p in (Path(directory) / 'workers').glob('w_*/i_*') if p.is_dir()) # Extract parameters and observables X, Y = [], [] diff --git a/tests/inference/test_plot.py b/tests/inference/test_plot.py index 542419c75..f914cb194 100644 --- a/tests/inference/test_plot.py +++ b/tests/inference/test_plot.py @@ -585,6 +585,48 @@ def test_plot_result_correlation_multi_par_multi_obs(monkeypatch, tmp_path, capl assert 'Missing helpfile for' in caplog.text +@pytest.mark.unit +def test_plot_result_correlation_ignores_stray_console_log_file(monkeypatch, tmp_path): + """A worker's console-log capture file must not be treated as a case dir. + + Regression for a crash where a stray file such as ``i_0_console.log``, + sitting beside the real ``i_0`` case directory in a worker folder, matched + the ``i_*`` glob used to find cases. ``toml.load`` then received a file + path, not a directory, and raised ``NotADirectoryError`` when the code + appended ``init_coupler.toml`` to it. + """ + workers = tmp_path / 'workers' + case_ok = workers / 'w_-1' / 'i_0' + case_ok.mkdir(parents=True) + (case_ok / 'init_coupler.toml').write_text( + toml.dumps({'planet': {'mass_tot': 1.5}}), + encoding='utf-8', + ) + pd.DataFrame([{'P_surf': 1.0}]).to_csv( + case_ok / 'runtime_helpfile.csv', sep=' ', index=False + ) + + # Sibling capture file that matches the `i_*` glob but is not a case dir. + (workers / 'w_-1' / 'i_0_console.log').write_text('log output\n', encoding='utf-8') + + axis = MagicMock() + axis.__getitem__.return_value = axis + fig = MagicMock() + mock_plt = MagicMock() + mock_plt.subplots.return_value = (fig, axis) + monkeypatch.setattr(plot_mod, 'plt', mock_plt) + monkeypatch.setattr(plot_mod, 'variable_is_logarithmic', lambda _k: False) + + # Must not raise NotADirectoryError from treating the log file as a case. + plot_mod.plot_result_correlation( + pars={'planet.mass_tot': [0.7, 3.0]}, + obs={'P_surf': 1.0}, + directory=str(tmp_path), + ) + + fig.savefig.assert_called_once() + + def test_plot_result_correlation_two_par_two_obs_uses_2d_axes(monkeypatch, tmp_path): """n_par > 1 and n_obs > 1 takes the ``axs[j, i]`` 2D indexing branch. From 8b7c6534ff6de2479a03d4050049dc29e7e62018 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Tue, 15 Sep 2026 10:47:16 +0200 Subject: [PATCH 16/72] small tweaks to failures text in docs --- docs/How-to/inference.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index 9d5ceb958..df6b5286f 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -153,8 +153,7 @@ The system prints the final results including: ### Failed and excluded simulations Two kinds of evaluation carry the fixed bad objective value instead of a fit -quality, and the study carries on in both cases so that one awkward corner does -not end a run that has been going for hours. +quality, and the study carries on in both cases. **Failures.** A sweep over a wide parameter box is expected to reach combinations PROTEUS cannot integrate. A run that crashes, is killed, or stops @@ -168,13 +167,10 @@ does not want to fit against, such as `11` (maximum clock runtime) or `15` (volatiles escaped). These are reported at info level. At the end of the study both kinds are collected into `failures.csv`, which -carries a `category` column of `failure` or `excluded`, and summarised. If more than half the evaluations went unscored, the summary says so as a warning, because the result then rests on far fewer real evaluations than the step count suggests. If no optimisation evaluation produced a fit quality there is no best fit to report, and the study stops with an error rather than presenting the least-bad run as a result. +carries a `category` column of `failure` or `excluded`, and summarised. If more than half the evaluations went unscored, the summary says so as a warning, because the result then rests on far fewer real evaluations than the step count suggests. If no optimisation evaluation produced a fit quality there is no best fit to report, and the study stops with an error. Set `abort_on_failure = true` in the inference config to stop at the first failed -simulation instead. This is useful while setting a study up, when the first -failure is more likely to be a mistake in the reference config than a genuinely -unrunnable corner of the parameter space. It applies to failures only; an -excluded outcome never stops the study. +simulation instead. This is useful while setting a study up, when the first failure is more likely to be a mistake in the reference config than an unrunnable corner of the parameter space. It applies to failures only; an excluded outcome never stops the study. ## Customization From c236b78593c664c2bbbde2382dccc0e1de909341 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 12:42:01 +0200 Subject: [PATCH 17/72] update logs w.r.t failures --- src/proteus/inference/utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index dc330ca2b..de62063bc 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -220,7 +220,8 @@ def summarise_failures(output: str, n_attempted: int) -> int: frac = n_unscored / max(n_attempted, 1) log.info( f'Unscored evaluations: {n_unscored} of {n_attempted} evaluations ' - f'({100 * frac:.1f}%) carry the failure score rather than a fit quality' + f'({100 * frac:.1f}%, initial samples included) carry the failure score ' + 'rather than a fit quality' ) log.info(f' {n_failed} did not produce a usable result') log.info(f' {n_excluded} completed on a status this study excludes') @@ -234,14 +235,16 @@ def summarise_failures(output: str, n_attempted: int) -> int: ).most_common(): label = f'{desc} [excluded]' if category == CATEGORY_EXCLUDED else str(desc) log.info(f'{label:52s} {count}') + # A few concrete places to look, labelled and counted so the sample is not + # read as the whole list. The simulator writes its own traceback to these + # logfiles, so they carry the cause that the status code only names. + sample = [rec['log_path'] for rec in records if rec.get('log_path')][:3] + if sample: + log.info(f'Logfiles ({len(sample)} of {n_unscored} shown):') + for log_path in sample: + log.info(f' {log_path}') log.info(f'Full list: {csv_path}') - # A few concrete places to look. The simulator writes its own traceback to - # these logfiles, so they carry the cause that the status code only names. - for rec in records[:3]: - if rec.get('log_path'): - log.info(f' {rec["log_path"]}') - if frac > FAILURE_FRACTION_WARN: log.warning( f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations were not scored ' @@ -300,8 +303,7 @@ def print_results(D, logs, config, output, n_init): raise RuntimeError( f'None of the {n_optim} optimisation evaluations produced a fit quality, ' 'so there is no best fit to report. The per-run reports above name the ' - 'cause of each; the most common causes are a reference config the ' - 'simulator refuses, a parameter range that leaves the model unphysical, ' + 'cause of each; the most common causes are a parameter range that leaves the model unphysical, ' 'and a `failure_codes` list that excludes the outcome most runs reach.' ) From 32fd89a6d832c46a583ae84f59b1c52f4300090e Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 13:06:14 +0200 Subject: [PATCH 18/72] fix abort_on_failure for runs that exit cleanly but write a failure status --- src/proteus/inference/objective.py | 39 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index e28bdfce3..38fda5fa4 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -720,25 +720,32 @@ def J( else CommentFromStatus(sim_status) ) _, out_abs = run_output_dir(output, worker, iter) - record_failure( - get_proteus_directories(output)['output'], - ProteusRunFailure( - reason=( - 'exited cleanly but stopped in a failure state' - if failed - else 'completed on a status this study excludes' - ), - worker=worker, - iter=iter, - out_dir=str(out_abs), - exit_code=0, - status=sim_status, - log_path=find_run_logfile(out_abs), - parameters=raw, - category=CATEGORY_FAILURE if failed else CATEGORY_EXCLUDED, + # Built once, so the entry left on disk and the exception raised under + # `abort_on_failure` describe the same run. + failure = ProteusRunFailure( + reason=( + 'exited cleanly but stopped in a failure state' + if failed + else 'completed on a status this study excludes' ), + worker=worker, + iter=iter, + out_dir=str(out_abs), + exit_code=0, + status=sim_status, + log_path=find_run_logfile(out_abs), + parameters=raw, + category=CATEGORY_FAILURE if failed else CATEGORY_EXCLUDED, ) + # Recorded before the abort check, so an aborted study still leaves + # the record of what stopped it. + record_failure(get_proteus_directories(output)['output'], failure) if failed: + # A clean exit on an error status is as much a fault as a crash, + # so it honours `abort_on_failure` the same way. An excluded + # outcome never does: nothing went wrong in such a run. + if abort_on_failure(): + raise failure log.warning( f'PROTEUS run for worker={worker} iter={iter} did not produce a usable ' f'result: status {sim_status} ({desc})' From 32c539114abbe860eb5154a608d7db6331127c63 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 13:08:05 +0200 Subject: [PATCH 19/72] only flag unscored exactly at -20 --- src/proteus/inference/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index de62063bc..6e2f74a34 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -289,7 +289,7 @@ def print_results(D, logs, config, output, n_init): # apart, so the wording here covers both and the tally above splits them. optim_Y = Y[n_init:] n_optim = len(optim_Y) - n_unscored = int((optim_Y <= BAD_OBJ_VALUE).sum().item()) + n_unscored = int((optim_Y == BAD_OBJ_VALUE).sum().item()) if n_unscored: log.warning( f'{n_unscored} of {n_optim} optimisation evaluations carry the failure ' From bbfc8495aabbf6e3d9b33c03efae0eddf2f432d1 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 13:11:38 +0200 Subject: [PATCH 20/72] update tests with failure handling updates --- tests/inference/test_objective.py | 79 ++++++++++++++++++++++++++ tests/inference/test_utils_branches.py | 76 +++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index bb13b8a0b..62c88dfba 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -772,6 +772,85 @@ def test_J_scores_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_pa assert good.item() - objective_mod.BAD_OBJ_VALUE > 25.0 +@pytest.mark.unit +def test_J_aborts_on_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_path): + """`abort_on_failure` stops the study on a run that exited cleanly but + recorded an error status, the same way it stops on a run that crashed. + Both are faults; only the route by which the simulator reported them + differs, so honouring the setting on one and not the other would let a + study set up with `abort_on_failure = true` run to completion on a + reference config that fails every evaluation. + + The asymmetry the setting must keep: an excluded status completed + normally, so it is scored as a poor sample and the study carries on even + with aborting enabled. + """ + monkeypatch.setattr( + objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} + ) + + def _run(status, worker, iter, codes=()): + monkeypatch.setattr( + objective_mod, + 'run_proteus', + lambda **_kwargs: ({'R_obs': 9.25e6}, status), + ) + return objective_mod.J( + x=torch.tensor([[0.5]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=worker, + iter=iter, + output='dummy_output', + ref_config='reference.toml', + failure_codes=list(codes), + ) + + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '1') + + # Status 25: written when a run is stopped through its keepalive file, so + # the simulator exits 0 and the fault is visible only in the status file. + with pytest.raises(objective_mod.ProteusRunFailure) as caught: + _run(25, worker=0, iter=0) + assert caught.value.status == 25 + assert caught.value.category == objective_mod.CATEGORY_FAILURE + # Exit code 0 is the whole point of this path: the abort must not depend + # on the child having exited non-zero. + assert caught.value.exit_code == 0 + + # Boundary of the failure set: STATUS_MISSING is the lowest code treated + # as a fault, and the run's own account of itself is absent, so it cannot + # be scored. A range check written as `20 <= status <= 28` alone would + # miss it. + with pytest.raises(objective_mod.ProteusRunFailure) as missing: + _run(objective_mod.STATUS_MISSING, worker=0, iter=1) + assert missing.value.status == objective_mod.STATUS_MISSING + + # The record is written before the abort, so an aborted study still says + # on disk what stopped it rather than leaving only the traceback. + recorded = objective_mod.read_failure_records(tmp_path) + # Ordered by (worker, iter), so the status-25 run at iter 0 comes first. + assert [r['status'] for r in recorded] == [25, objective_mod.STATUS_MISSING] + + # Discrimination against a fix that aborts on `failed or excluded`: an + # excluded status is scored as a poor sample and returns normally. + excluded = _run(11, worker=1, iter=0, codes=(11,)) + assert excluded.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) + # Boundedness: the failure score sits far below anything a completed run + # can reach, so the optimiser is not drawn toward the excluded region. + assert excluded.item() < -10.0 + assert objective_mod.read_failure_records(tmp_path)[-1]['category'] == ( + objective_mod.CATEGORY_EXCLUDED + ) + + # Discrimination against a regression that raises unconditionally: with + # the setting off, the same error status is scored and the study goes on. + monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + scored = _run(25, worker=2, iter=0) + assert scored.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) + assert scored.item() < -10.0 + + @pytest.mark.unit def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): """The failure range covers the error statuses the simulator can record. diff --git a/tests/inference/test_utils_branches.py b/tests/inference/test_utils_branches.py index 9a905c8ce..c4072fe9c 100644 --- a/tests/inference/test_utils_branches.py +++ b/tests/inference/test_utils_branches.py @@ -500,3 +500,79 @@ def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_pa messages = '\n'.join(r.message for r in caplog.records) assert 'none' in messages and '12 evaluations' in messages assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] + + +@pytest.mark.unit +def test_summarise_failures_labels_the_logfile_sample_and_counts_the_whole_study( + tmp_path, caplog +): + """The tally covers every evaluation attempted, initial samples included, + while the warning raised alongside the best fit covers the optimisation + steps alone. The logfile lines are a sample of at most three, so they are + labelled with how many of the total they show and printed above the pointer + to the full table; unlabelled, three paths below a "Full list" line read as + the complete set. + """ + from proteus.inference.objective import ProteusRunFailure, record_failure + from proteus.inference.utils import summarise_failures + + # Four records, the first of which has no logfile: the run died before the + # child wrote one. The sample must skip it and still offer three paths. + for worker in range(4): + record_failure( + tmp_path, + ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=1, + status=21, + log_path=None if worker == 0 else f'/study/w_{worker}/proteus_00.log', + parameters={'planet.mass_tot': 1.0 + worker}, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + summarise_failures(str(tmp_path), n_attempted=20) + + lines = [r.message for r in caplog.records] + messages = '\n'.join(lines) + # The denominator of the tally is the whole study, stated in the line + # itself so it cannot be confused with the optimisation-only warning. + assert '4 of 20 evaluations' in messages + assert 'initial samples included' in messages + # Three shown out of four unscored, not four out of four. + assert 'Logfiles (3 of 4 shown):' in messages + shown = [line.strip() for line in lines if line.strip().endswith('proteus_00.log')] + assert len(shown) == 3 + # The record without a logfile is skipped rather than truncating the + # sample to the two paths that follow it in the first three records. + assert '/study/w_1/proteus_00.log' in shown + assert '/study/w_3/proteus_00.log' in shown + + # The pointer to the complete table comes after the sample, so the sample + # cannot be read as a continuation of it. + i_sample = next(i for i, line in enumerate(lines) if line.startswith('Logfiles (')) + i_full = next(i for i, line in enumerate(lines) if line.startswith('Full list:')) + assert i_sample < i_full + + # Discrimination: with no logfile recorded anywhere, no sample block is + # emitted at all, so the label tracks the data rather than always printing. + caplog.clear() + (tmp_path / 'failures').rename(tmp_path / 'failures_old') + record_failure( + tmp_path, + ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=1, + out_dir='/study/workers/w_0/i_1', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 1.0}, + ), + ) + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): + summarise_failures(str(tmp_path), n_attempted=20) + assert not [r for r in caplog.records if r.message.startswith('Logfiles (')] From 64428b2fe6c8f0b352ff36316ba397793a311289 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 15:32:03 +0200 Subject: [PATCH 21/72] fix logging when spawn method is used (MacOS) --- src/proteus/inference/async_BO.py | 32 ++++++++++++++++++++++ src/proteus/utils/logs.py | 45 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index 4fc241fc5..d5ee63a1c 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -28,6 +28,7 @@ from proteus.inference.BO import BO_step, init_locs from proteus.inference.utils import get_kernel, load_dataset_csv, save_dataset_csv from proteus.utils.coupler import get_proteus_directories +from proteus.utils.logs import attach_worker_logfile # Tensor dtype for all computations dtype = torch.double @@ -65,6 +66,19 @@ def checkpoint(D: dict, logs: list, Ts: list, output_dir: str) -> None: ) +def _parent_logfile() -> str | None: + """Path of the logfile the study's logger is writing, if it has one. + + Read in the parent, because a spawned worker has no logging configuration + of its own to read it from. Returning the handler's own path rather than + rebuilding it keeps the logfile named in one place only. + """ + for handler in logging.getLogger('fwl').handlers: + if isinstance(handler, logging.FileHandler): + return handler.baseFilename + return None + + def worker( process_fun, build_obj, @@ -79,6 +93,8 @@ def worker( worker_id: int, log_list, output_dir: str, + logpath: str | None = None, + log_level: int = logging.INFO, ) -> None: """Worker subprocess that performs asynchronous BO steps. @@ -103,11 +119,21 @@ def worker( - worker_id (int): Unique identifier of this worker. - log_list (Manager.list): Shared list to store per-eval log dicts. - output_dir (str): Output directory for the whole inference call (abspath). + - logpath (str | None): Study logfile to reopen when this process has no + logging configuration of its own. None leaves logging untouched. + - log_level (int): Numeric level to log at, read from the parent. Returns ---------- - None """ + # A spawned worker inherits no logging configuration on MacOS, + # so everything below would go to stderr and miss the study logfile. + # Reattach before any work starts, so that a failure in + # the very first iteration is still recorded where the study reads it. + if logpath: + attach_worker_logfile(logpath, log_level) + try: _worker_loop( process_fun, @@ -327,6 +353,10 @@ def parallel_process( # Set up step constraint max_steps = max_len - (n_workers - 1) + # Read in the parent: a spawned worker has none of this to read from. + worker_logpath = _parent_logfile() + worker_log_level = logging.getLogger('fwl').level + # Spawn worker processes procs = [] for wid in range(n_workers): @@ -349,6 +379,8 @@ def parallel_process( wid, log_list, output_abspath, + worker_logpath, + worker_log_level, ), ) p.start() diff --git a/src/proteus/utils/logs.py b/src/proteus/utils/logs.py index f427e57ef..0e0795c9e 100644 --- a/src/proteus/utils/logs.py +++ b/src/proteus/utils/logs.py @@ -207,6 +207,51 @@ def bootstrap_logger(level: str = 'INFO'): return custom_logger +def attach_worker_logfile(logpath: str, level_code: int = logging.INFO): + """Attach an appending file handler to the 'fwl' logger if it has none. + + A worker process started with the 'spawn' method (default on macOS) + re-imports the package instead of inheriting the parent's logging + configuration. Records it emits then find no handler and fall through to + ``logging.lastResort``, which writes to stderr, so nothing the worker + reports reaches the study logfile, including the traceback of a worker + that dies. Under 'fork', the parent's handlers are inherited already + and this is a no-op. + + The file is opened in append mode, where ``setup_logger`` recreates it: + the parent owns the logfile, and a worker must add to it rather than + truncate the record of the run so far. + + Parameters + ---------- + logpath : str + Path of the logfile the parent's logger is already writing. + level_code : int + Numeric log level, read from the parent's logger before the worker + starts, so the worker honours the level the study was run at. + + Returns + ------- + logging.Logger + The 'fwl' logger. + """ + custom_logger = logging.getLogger('fwl') + + # Do not touch an already-configured logger. Under 'fork' this process + # inherited the parent's handlers, and adding a second one would write + # every worker line to the logfile twice. + if custom_logger.handlers: + return custom_logger + + fh = logging.FileHandler(logpath, mode='a') + fh.setFormatter(logging.Formatter('[ %(levelname)-5s ] %(message)s')) + fh.setLevel(level_code) + custom_logger.addHandler(fh) + custom_logger.setLevel(level_code) + + return custom_logger + + def GetCurrentLogfileIndex(output_dir: str): """ Get the index of the current logfile, returning -1 if none exists From 628c04bd2aa74b916c738e4f8789c8738392f0dd Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Wed, 16 Sep 2026 15:34:59 +0200 Subject: [PATCH 22/72] update tests with logger fix --- tests/inference/test_async_bo.py | 114 +++++++++++++++++++++++++++++++ tests/utils/test_logs.py | 106 ++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index 1a6e69c31..07471d195 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -8,6 +8,8 @@ from __future__ import annotations +import logging + import pandas as pd import pytest @@ -623,3 +625,115 @@ def join(self): # The requested budget is still named, so the two numbers can be related. assert '6 requested' in message assert 'Raise n_steps to at least n_workers (2)' in message + + +@pytest.mark.unit +def test_worker_writes_its_traceback_to_the_study_logfile(tmp_path): + """A worker started with the 'spawn' method inherits no logging + configuration, so the report of its death would go to stderr and never + reach the study logfile. Given the logfile path, the worker reopens it + and the traceback lands where the study is read from. + + The 'fwl' logger is emptied here to stand in for a spawned process, which + is what the parent's handlers are absent in; pytest's own capture would + otherwise hide the gap this covers. + """ + logger = logging.getLogger('fwl') + saved_handlers, saved_level = list(logger.handlers), logger.level + logger.handlers.clear() + + logpath = tmp_path / 'infer.log' + logpath.write_text('[ INFO ] study started\n', encoding='utf-8') + + D_shared = { + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[0.2]], dtype=torch.double), + } + # Keyed by this worker's own id, so the release below is a real check. + B = {3: torch.tensor([[0.3]], dtype=torch.double)} + + def exploding_process_fun(**_kwargs): + raise RuntimeError('objective evaluation failed') + + try: + with pytest.raises(RuntimeError, match='objective evaluation failed'): + async_mod.worker( + process_fun=exploding_process_fun, + build_obj=lambda **kwargs: lambda x: x, + D_shared=D_shared, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.3]], dtype=torch.double), + n_init=1, + lock=_DummyLock(), + max_len=4, + worker_id=3, + log_list=[], + output_dir=str(tmp_path), + logpath=str(logpath), + log_level=logging.INFO, + ) + for handler in logging.getLogger('fwl').handlers: + handler.flush() + text = logpath.read_text(encoding='utf-8') + finally: + logger.handlers.clear() + logger.handlers.extend(saved_handlers) + logger.setLevel(saved_level) + + assert 'Worker 3 stopped early' in text + # The cause, not just the headline: a report without the traceback body + # would leave the study with no more than the fact that something failed. + assert 'RuntimeError: objective evaluation failed' in text + # Appended, never recreated: the lines written before the worker started + # are what place the failure in the run. + assert 'study started' in text + # The busy point is still released on the way out, so the logfile change + # has not displaced the behaviour the failure path already had. + assert B == {} + + +@pytest.mark.unit +def test_worker_without_a_logfile_path_leaves_logging_untouched(tmp_path, caplog): + """Under 'fork' the parent's handlers are inherited, so `parallel_process` + passes no path and the worker must not attach one of its own; a second + handler on the same file would double every line. The failure is still + reported through whatever configuration the process already has. + """ + logger = logging.getLogger('fwl') + before = list(logger.handlers) + + D_shared = { + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[0.2]], dtype=torch.double), + } + B = {0: torch.tensor([[0.3]], dtype=torch.double)} + + def exploding_process_fun(**_kwargs): + raise RuntimeError('objective evaluation failed') + + with caplog.at_level('ERROR'): + with pytest.raises(RuntimeError, match='objective evaluation failed'): + async_mod.worker( + process_fun=exploding_process_fun, + build_obj=lambda **kwargs: lambda x: x, + D_shared=D_shared, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.3]], dtype=torch.double), + n_init=1, + lock=_DummyLock(), + max_len=4, + worker_id=0, + log_list=[], + output_dir=str(tmp_path), + ) + + # No handler added, and none taken away. + assert list(logger.handlers) == before + # No stray logfile created beside the run output. + assert not (tmp_path / 'infer.log').exists() + reported = '\n'.join(record.getMessage() for record in caplog.records) + assert 'Worker 0 stopped early' in reported diff --git a/tests/utils/test_logs.py b/tests/utils/test_logs.py index fbdde053b..3a27e5c13 100644 --- a/tests/utils/test_logs.py +++ b/tests/utils/test_logs.py @@ -21,6 +21,7 @@ GetCurrentLogfileIndex, GetLogfilePath, StreamToLogger, + attach_worker_logfile, bootstrap_logger, setup_logger, ) @@ -894,3 +895,108 @@ def test_preserves_directory_path(self): path = GetLogfilePath(dirpath, 42) assert path.startswith(dirpath) assert 'proteus_42.log' in path + + +@pytest.fixture +def clean_fwl_logger(): + """Give a test the 'fwl' logger with no handlers, and restore it after. + + The logger is process-global, so a test that attaches a handler to it + would otherwise leak that handler into every test that runs later. + """ + logger = logging.getLogger('fwl') + saved_handlers, saved_level = list(logger.handlers), logger.level + logger.handlers.clear() + try: + yield logger + finally: + logger.handlers.clear() + logger.handlers.extend(saved_handlers) + logger.setLevel(saved_level) + + +@pytest.mark.unit +def test_attach_worker_logfile_appends_to_an_existing_study_logfile(clean_fwl_logger, tmp_path): + """A worker process with no logging configuration of its own reopens the + study logfile and adds to it. Appending rather than recreating is the + whole contract: `setup_logger` deletes the logfile it opens, so a worker + calling that instead would erase everything the study had logged before + the worker started. + """ + logpath = tmp_path / 'infer.log' + logpath.write_text('[ INFO ] parent wrote this first\n', encoding='utf-8') + + attach_worker_logfile(str(logpath), logging.INFO) + logging.getLogger('fwl.worker').error('worker stopped early') + + for handler in clean_fwl_logger.handlers: + handler.flush() + text = logpath.read_text(encoding='utf-8') + + # The pre-existing content survives: this is the assertion that fails if + # the handler is ever opened in 'w' mode. + assert 'parent wrote this first' in text + assert 'worker stopped early' in text + # Written through the study's file format, so worker lines are not visibly + # different from the parent's. + assert '[ ERROR ] worker stopped early' in text + + +@pytest.mark.unit +def test_attach_worker_logfile_carries_a_traceback_and_respects_the_level( + clean_fwl_logger, tmp_path +): + """The message this exists for is a dying worker's traceback, so the + exception text must reach the file, and a level set above INFO must still + suppress the ordinary INFO chatter that would otherwise bloat the logfile. + """ + logpath = tmp_path / 'infer.log' + attach_worker_logfile(str(logpath), logging.WARNING) + + worker_log = logging.getLogger('fwl.worker') + worker_log.info('routine iteration finished') + try: + raise ValueError('objective evaluation failed') + except ValueError: + worker_log.exception('Worker 3 stopped early') + + for handler in clean_fwl_logger.handlers: + handler.flush() + text = logpath.read_text(encoding='utf-8') + + assert 'Worker 3 stopped early' in text + # The traceback body, not just the message: a handler without exc_info + # support would log the first line and drop the cause. + assert 'ValueError: objective evaluation failed' in text + assert 'Traceback (most recent call last)' in text + # Boundary of the configured level: INFO sits one step below WARNING and + # must be dropped, which also rules out a handler left at level NOTSET. + assert 'routine iteration finished' not in text + + +@pytest.mark.unit +def test_attach_worker_logfile_leaves_an_already_configured_logger_alone( + clean_fwl_logger, tmp_path +): + """Under the 'fork' start method a worker inherits the parent's handlers. + Adding a second one there would write every worker line to the logfile + twice, so the call must be a no-op whenever handlers already exist. + """ + inherited = tmp_path / 'inherited.log' + handler = logging.FileHandler(inherited) + handler.setFormatter(logging.Formatter('[ %(levelname)-5s ] %(message)s')) + clean_fwl_logger.addHandler(handler) + clean_fwl_logger.setLevel(logging.INFO) + + untouched = tmp_path / 'should_not_be_written.log' + attach_worker_logfile(str(untouched), logging.INFO) + + assert len(clean_fwl_logger.handlers) == 1 + # The second path is never opened, so no stray logfile appears beside the + # study's own. + assert not untouched.exists() + + logging.getLogger('fwl.worker').warning('one line only') + handler.flush() + # Exactly one copy: a duplicate handler would give two. + assert inherited.read_text(encoding='utf-8').count('one line only') == 1 From 7f7a994f8faaf5eb4e0298d77b24f6e75498431a Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Thu, 17 Sep 2026 14:31:28 +0200 Subject: [PATCH 23/72] reduce spam in output log --- src/proteus/inference/objective.py | 42 ++++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 38fda5fa4..f62ce4b1e 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -144,6 +144,21 @@ def status_desc(self) -> str: return 'no readable status file (died during start-up)' return CommentFromStatus(self.status) + def summary(self) -> str: + """Single-line description naming the outcome and where to look next. + """ + verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' + parts = [ + f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', + f'status {self.status} ({self.status_desc})', + ] + # A zero exit code is the norm for every path except a crash, where it + # is the one number that says which signal or error ended the run. + if self.exit_code: + parts.append(f'exit code {self.exit_code}') + parts.append(f'output {self.out_dir}') + return '; '.join(parts) + def report(self) -> str: """Multi-line description naming the cause and where to look next.""" verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' @@ -688,14 +703,15 @@ def J( except ProteusRunFailure as failure: # A parameter combination the simulator cannot integrate is an # expected outcome of sweeping a wide box, so it is scored as a poor - # sample and the study continues. Every such run is reported in full, - # once, because the alternative is a silently under-sampled study. + # sample and the study continues. Every such run is reported once, + # and the full report goes to the failure record. # Recorded before the abort check, so an aborted study still leaves # the record of what stopped it. record_failure(get_proteus_directories(output)['output'], failure) if abort_on_failure(): raise - log.warning(failure.report()) + log.warning(failure.summary()) + log.debug(failure.report()) return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) # Runs that exit cleanly but stop in an error state, such as a run halted @@ -714,11 +730,6 @@ def J( # Either way the evaluation carries the failure score instead of a fit # quality, and is recorded so that the end-of-study tally covers it. if failed or excluded: - desc = ( - 'no status file written' - if sim_status == STATUS_MISSING - else CommentFromStatus(sim_status) - ) _, out_abs = run_output_dir(output, worker, iter) # Built once, so the entry left on disk and the exception raised under # `abort_on_failure` describe the same run. @@ -746,16 +757,13 @@ def J( # outcome never does: nothing went wrong in such a run. if abort_on_failure(): raise failure - log.warning( - f'PROTEUS run for worker={worker} iter={iter} did not produce a usable ' - f'result: status {sim_status} ({desc})' - ) + log.warning(failure.summary()) else: - log.info( - f'PROTEUS run for worker={worker} iter={iter} completed on status ' - f'{sim_status} ({desc}), which this study excludes; scored as a poor ' - 'sample' - ) + # Nothing went wrong in such a run, so it is reported at info + # level and, like a fault, on one line. + log.info(failure.summary()) + # The rest of the report is kept out of the study log + log.debug(failure.report()) return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) # Compute value of objective function given these results From 5756b9358f98487414b7acd308f82f4ff8781612 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Thu, 17 Sep 2026 15:51:53 +0200 Subject: [PATCH 24/72] update test with less verbose logging --- tests/inference/test_objective.py | 63 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 62c88dfba..14a307282 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -588,6 +588,63 @@ def test_run_proteus_reports_a_clean_exit_that_produced_no_output(monkeypatch, t assert obs['P_surf'] == pytest.approx(2.5) +@pytest.mark.unit +def test_failure_summary_is_one_line_and_names_where_the_detail_is_kept(): + """The line a study logs for each unscored run identifies the run, names + the status code and points at the output folder, and stays on one line + however much the child wrote before it died. The child console tail and the + swept parameters belong to the on-disk record, not to the console. + """ + tail = '\n'.join(f'flux warning {i}' for i in range(200)) + failure = objective_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=2, + iter=16, + out_dir='/study/workers/w_2/i_16', + exit_code=1, + status=22, + log_path='/study/workers/w_2/i_16/proteus_00.log', + stderr_tail=tail, + parameters={'planet.mass_tot': 1.25}, + ) + line = failure.summary() + + # Edge case: a 200-line tail is the situation the one-liner exists for. + assert '\n' not in line + assert 'flux warning 199' not in line + assert 'planet.mass_tot' not in line + # What has to survive the trim: who failed, what the status was, and the + # folder holding the logfile and the console capture. + assert 'worker=2 iter=16' in line + assert 'status 22' in line + assert 'Atmosphere' in line + assert 'exit code 1' in line + assert '/study/workers/w_2/i_16' in line + # Discrimination: the detail is not lost, only moved. A regression that + # trimmed `report` instead of adding a second renderer would fail here. + assert 'flux warning 199' in failure.report() + assert 'planet.mass_tot=1.25' in failure.report() + + # Limit input: an excluded run has nothing to report as a fault, so it is + # named as excluded and its exit code, always zero on that path, is left + # out rather than read as a crash code. + excluded = objective_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=0, + iter=10, + out_dir='/study/workers/w_0/i_10', + exit_code=0, + status=11, + category=objective_mod.CATEGORY_EXCLUDED, + ) + excluded_line = excluded.summary() + assert '\n' not in excluded_line + assert 'excluded for worker=0 iter=10' in excluded_line + assert 'failed for worker=0' not in excluded_line + assert 'exit code' not in excluded_line + assert 'status 11' in excluded_line + + @pytest.mark.unit def test_proteus_run_failure_survives_the_trip_back_from_a_pool_worker(): """A failure raised inside a pool worker is pickled and re-raised in the @@ -934,7 +991,7 @@ def _score(status, worker): assert 'excludes' in reported assert 'maximum clock runtime' in reported assert 'failure state' not in reported - assert 'did not produce a usable result' not in reported + assert 'failed for worker=0' not in reported # The record is kept for the end-of-study tally, labelled so the tally can # count it apart from the runs that genuinely failed. @@ -952,7 +1009,9 @@ def _score(status, worker): assert failed == pytest.approx(objective_mod.BAD_OBJ_VALUE) warnings = [r for r in caplog.records if r.levelname == 'WARNING'] assert len(warnings) == 1 - assert 'did not produce a usable result' in warnings[0].getMessage() + assert 'failed for worker=1' in warnings[0].getMessage() + assert 'stopped in a failure state' in warnings[0].getMessage() + assert 'status 21' in warnings[0].getMessage() recorded = objective_mod.read_failure_records(tmp_path) assert [(r['status'], r['category']) for r in recorded] == [ (11, objective_mod.CATEGORY_EXCLUDED), From acf4910fb182309b6b4a6d3c7ba10ed5596cc8f5 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Thu, 17 Sep 2026 16:00:23 +0200 Subject: [PATCH 25/72] fix ruff format --- src/proteus/inference/objective.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index f62ce4b1e..c89c23628 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -145,8 +145,7 @@ def status_desc(self) -> str: return CommentFromStatus(self.status) def summary(self) -> str: - """Single-line description naming the outcome and where to look next. - """ + """Single-line description naming the outcome and where to look next.""" verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' parts = [ f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', From 8670634734163373ef8eafad249107d0f2b2a03f Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 10:28:59 +0200 Subject: [PATCH 26/72] summarise 'failed and excluded simulations' more concisely --- docs/How-to/inference.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index df6b5286f..38f9840e1 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -152,25 +152,7 @@ The system prints the final results including: ### Failed and excluded simulations -Two kinds of evaluation carry the fixed bad objective value instead of a fit -quality, and the study carries on in both cases. - -**Failures.** A sweep over a wide parameter box is expected to reach -combinations PROTEUS cannot integrate. A run that crashes, is killed, or stops -on an error status (20 to 28) produced nothing usable. Each one is reported as a -warning when it happens, naming the status code PROTEUS recorded, the run's -output folder, the logfile holding its traceback, and the parameter values that -produced it. - -**Exclusions.** The `failure_codes` field lists completion statuses the study -does not want to fit against, such as `11` (maximum clock runtime) or `15` -(volatiles escaped). These are reported at info level. - -At the end of the study both kinds are collected into `failures.csv`, which -carries a `category` column of `failure` or `excluded`, and summarised. If more than half the evaluations went unscored, the summary says so as a warning, because the result then rests on far fewer real evaluations than the step count suggests. If no optimisation evaluation produced a fit quality there is no best fit to report, and the study stops with an error. - -Set `abort_on_failure = true` in the inference config to stop at the first failed -simulation instead. This is useful while setting a study up, when the first failure is more likely to be a mistake in the reference config than an unrunnable corner of the parameter space. It applies to failures only; an excluded outcome never stops the study. +During the inference run, some PROTEUS simulations might crash or fail, or stop on a status that is excluded in the inference configuration (e.g. maximum runtime reached). The run carries on when there are failures unless `abort_on_failure` is set to `true` in the inference config. At the end of the study all failures are written to `failures.csv` in the output folder, and summarised. ## Customization From 47aa3e9bdee4caaa9d377620e49a90af886890f6 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 10:46:41 +0200 Subject: [PATCH 27/72] remove p_surf column raise --- src/proteus/inference/objective.py | 9 ------ tests/inference/test_objective.py | 51 ------------------------------ 2 files changed, 60 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index c89c23628..92a3075ac 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -572,15 +572,6 @@ def _failure( # Handle case where atmosphere has escaped # Set VMRs and MMW to zero - if 'P_surf' not in df_row: - # The helpfile schema is the same for every run, so a missing column - # is a fault of the setup rather than of this sample. Scoring it as a - # bad sample would let the study spend its whole budget returning the - # same failure value and then report success. - raise KeyError( - f"Simulator output has no 'P_surf' column ({out_csv}); " - 'every run will produce the same result' - ) if bool(df_row['P_surf'] < 1e-30): df_row['atm_kg_per_mol'] = 0.0 for g in gas_list: diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 14a307282..641a0fbce 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -1025,57 +1025,6 @@ def _score(status, worker): assert len(objective_mod.read_failure_records(tmp_path)) == 2 -@pytest.mark.unit -def test_run_proteus_treats_a_missing_helpfile_column_as_a_setup_fault(monkeypatch, tmp_path): - """A helpfile with no 'P_surf' column raises rather than producing a scored - failure. The column set is a property of the simulator's output schema, so - it is the same for every run: scoring it as a bad sample would let the - study spend its whole budget returning the failure value and then report - success at the end. - """ - out_abs = tmp_path / 'sim' - out_abs.mkdir(parents=True) - pd.DataFrame([{'T_obs': 300.0}]).to_csv( - out_abs / 'runtime_helpfile.csv', sep=' ', index=False - ) - monkeypatch.setattr( - objective_mod, 'get_proteus_directories', lambda _path: {'output': str(out_abs)} - ) - monkeypatch.setattr(objective_mod, 'update_toml', lambda *_args, **_kwargs: None) - monkeypatch.setattr(objective_mod.subprocess, 'run', lambda *args, **kwargs: None) - - with pytest.raises(KeyError) as excinfo: - objective_mod.run_proteus( - parameters={}, - worker=0, - iter=0, - observables=['T_obs'], - ref_config='reference.toml', - output='dummy_output', - ) - # Not a per-run failure: scoring this one badly and continuing is exactly - # the behaviour the raise exists to prevent. - assert not isinstance(excinfo.value, objective_mod.ProteusRunFailure) - assert 'P_surf' in str(excinfo.value) - assert 'every run' in str(excinfo.value) - - # Discrimination: the same helpfile with the column present completes, so - # the raise comes from the missing column and not from this code path - # rejecting every input. - pd.DataFrame([{'T_obs': 300.0, 'P_surf': 5.0}]).to_csv( - out_abs / 'runtime_helpfile.csv', sep=' ', index=False - ) - obs, _status = objective_mod.run_proteus( - parameters={}, - worker=0, - iter=0, - observables=['T_obs'], - ref_config='reference.toml', - output='dummy_output', - ) - assert obs['T_obs'] == pytest.approx(300.0) - - # ============================================================================ # Failure records: written per evaluation, read back for the study summary # ============================================================================ From 969bf14265456af08bf1249b5e993fbc037313aa Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 11:12:32 +0200 Subject: [PATCH 28/72] improve error logs and comments --- src/proteus/inference/async_BO.py | 45 +++++++++++-------------------- tests/inference/test_async_bo.py | 33 ++++++++++------------- 2 files changed, 30 insertions(+), 48 deletions(-) diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index d5ee63a1c..524666972 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -67,7 +67,7 @@ def checkpoint(D: dict, logs: list, Ts: list, output_dir: str) -> None: def _parent_logfile() -> str | None: - """Path of the logfile the study's logger is writing, if it has one. + """Path of the logfile the inference run's logger is writing, if it has one. Read in the parent, because a spawned worker has no logging configuration of its own to read it from. Returning the handler's own path rather than @@ -119,18 +119,17 @@ def worker( - worker_id (int): Unique identifier of this worker. - log_list (Manager.list): Shared list to store per-eval log dicts. - output_dir (str): Output directory for the whole inference call (abspath). - - logpath (str | None): Study logfile to reopen when this process has no - logging configuration of its own. None leaves logging untouched. + - logpath (str | None): Inference run logfile to reopen when this process has no + logging configuration of its own. - log_level (int): Numeric level to log at, read from the parent. Returns ---------- - None """ - # A spawned worker inherits no logging configuration on MacOS, - # so everything below would go to stderr and miss the study logfile. + # A spawned worker inherits no logging configuration on MacOS. # Reattach before any work starts, so that a failure in - # the very first iteration is still recorded where the study reads it. + # the very first iteration is still recorded in the logfile. if logpath: attach_worker_logfile(logpath, log_level) @@ -153,14 +152,11 @@ def worker( except BaseException: # A worker that dies takes its traceback with it: multiprocessing # prints it to the parent's stderr without consulting the logging - # configuration, so nothing reaches the study logfile. Record it here - # while the worker still can, then let it propagate so the exit code - # still marks the process as failed. + # configuration, so nothing reaches the logfile. Record it here. log.exception(f'Worker {worker_id} stopped early and will run no further evaluations') raise finally: - # Release this worker's busy point. Left in place, it steers the - # surviving workers away from a region nothing is actually exploring. + # Release this worker's busy point. try: with lock: B.pop(worker_id, None) @@ -295,7 +291,7 @@ def parallel_process( - observables (dict): Target observables (keys) and values. - parameters (dict): Parameters (keys) with bounds (values) for inference. - failure_codes (list[int]): PROTEUS status codes that complete normally but - that this study excludes from the fit. + that this run excludes from the fit. Returns ---------- @@ -395,19 +391,15 @@ def parallel_process( logs = list(log_list) T_elapsed = [t - T0 for t in list(T)] - # A worker that dies mid-study leaves the run looking complete: the - # remaining workers carry on, the results are saved, and the best-fit - # summary is printed from whatever was collected. Report the shortfall. - # A worker killed by a signal reports a negative code (-9 for an - # out-of-memory kill), so the test is "not zero" rather than "positive". + # A worker that dies mid-run leaves the run looking complete. Report. died = [wid for wid, p in enumerate(procs) if p.exitcode != 0] if died: names = ', '.join(str(wid) for wid in died) log.error( f'{len(died)} of {n_workers} workers stopped before the evaluation budget ' f'was reached (workers {names}). Their exit codes were ' - f'{[procs[wid].exitcode for wid in died]}; see the messages above for the ' - f'cause. Results below are based on {len(D_final["X"])} evaluations ' + f'{[procs[wid].exitcode for wid in died]}.' + f' Results are based on {len(D_final["X"])} evaluations ' f'rather than the {max_len} requested.' ) # Nothing was added to the initial sample, so there is no optimisation to @@ -419,18 +411,13 @@ def parallel_process( 'above for the cause.' ) else: - # Every worker exited on its first budget check. `max_steps` is - # reduced by one per additional worker, so this is what a study - # with fewer optimisation steps than workers looks like. + # Every worker exited on its first budget check. + n_steps = max_len - n_init cause = ( - 'No worker failed. Each worker stops once the dataset reaches ' - f'{max_steps} rows ({max_len} requested, less one per worker beyond ' - f'the first), which the {n_init} initial samples already satisfy. ' + f'No worker failed: the config asks for {n_steps} optimisation ' + f'step{"" if n_steps == 1 else "s"} across {n_workers} workers. ' f'Raise n_steps to at least n_workers ({n_workers}).' ) - raise RuntimeError( - 'No optimisation steps completed: the dataset still holds only the ' - f'{n_init} initial samples. ' + cause - ) + raise RuntimeError('No optimisation steps completed. ' + cause) return D_final, logs, T_elapsed diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index 07471d195..7a0bfcf12 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -405,7 +405,7 @@ def join(self): def test_parallel_process_stays_silent_when_every_worker_completes( monkeypatch, tmp_path, caplog ): - """A study in which no worker died reports no shortfall. Without this, the + """An inference run in which no worker died reports no shortfall. Without this, the failure message above would be indistinguishable from routine noise. """ @@ -491,7 +491,7 @@ def join(self): @pytest.mark.unit def test_worker_releases_its_busy_point_and_records_why_it_stopped(tmp_path, caplog): - """A worker that fails records the cause in the study log before it dies, + """A worker that fails records the cause in the log before it dies, and releases the point it had claimed. Neither happens on its own: multiprocessing prints a dead worker's traceback straight to the parent's stderr without consulting the logging configuration, and a claimed point @@ -539,8 +539,7 @@ def exploding_process_fun(**_kwargs): @pytest.mark.unit def test_worker_releases_its_busy_point_after_a_normal_finish(tmp_path): """A worker that reaches the evaluation budget also releases its claimed - point. Left behind, it would bias the acquisition for every worker still - running through the tail of the study. + point. """ D_shared = { 'X': torch.tensor([[0.1], [0.2]], dtype=torch.double), @@ -577,11 +576,11 @@ def test_worker_releases_its_busy_point_after_a_normal_finish(tmp_path): def test_parallel_process_names_the_real_step_budget_when_no_worker_failed( monkeypatch, tmp_path ): - """A study configured with fewer optimisation steps than workers finishes + """An inference run configured with fewer optimisation steps than workers finishes without any worker failing and without any step being taken. The refusal - must quote the row count the workers actually stop at, which is the - requested budget less one per worker beyond the first, or the advice reads - as false against the numbers the user set. + must name the condition in the quantities the user set, n_steps against + n_workers, since the internal row threshold the workers apply is not a + number that appears anywhere in the study config. """ class FakeProcess: @@ -597,8 +596,9 @@ def start(self): def join(self): return None - # Six initial samples against a budget of six with two workers: the worker - # threshold is 6 - (2 - 1) = 5, which the initial samples already exceed. + # One optimisation step across two workers, the smallest configuration that + # reaches this branch: the worker threshold is 7 - (2 - 1) = 6, which the + # six initial samples already meet. _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess, n_init_rows=6) with pytest.raises(RuntimeError) as excinfo: @@ -607,7 +607,7 @@ def join(self): kernel='MAT3/2', acqf='LogEI', n_workers=2, - max_len=6, + max_len=7, output='dummy', seed=1, ref_config='ref.toml', @@ -617,13 +617,8 @@ def join(self): ) message = str(excinfo.value) assert 'No worker failed' in message - # The threshold the workers actually apply. - assert 'reaches 5 rows' in message - # Discrimination: quoting the requested budget here instead would state - # that six initial samples satisfy a six-row threshold, which is false. - assert 'reaches 6 rows' not in message - # The requested budget is still named, so the two numbers can be related. - assert '6 requested' in message + # Named in the config's own terms: one step requested, two workers to run it. + assert '1 optimisation step across 2 workers' in message assert 'Raise n_steps to at least n_workers (2)' in message @@ -631,7 +626,7 @@ def join(self): def test_worker_writes_its_traceback_to_the_study_logfile(tmp_path): """A worker started with the 'spawn' method inherits no logging configuration, so the report of its death would go to stderr and never - reach the study logfile. Given the logfile path, the worker reopens it + reach the logfile. Given the logfile path, the worker reopens it and the traceback lands where the study is read from. The 'fwl' logger is emptied here to stand in for a spawned process, which From b498ee5b288471ee818f8d4553eebee66f9767ad Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 13:03:52 +0200 Subject: [PATCH 29/72] refactor: move failure handling in inference.py, objective.py and utils.py to failures.py to reduce the amount of lines per file. --- src/proteus/inference/failures.py | 351 +++++++++++++++++++++++++++++ src/proteus/inference/inference.py | 4 +- src/proteus/inference/objective.py | 310 ++----------------------- src/proteus/inference/utils.py | 112 +-------- 4 files changed, 373 insertions(+), 404 deletions(-) create mode 100644 src/proteus/inference/failures.py diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py new file mode 100644 index 000000000..387f4e4fd --- /dev/null +++ b/src/proteus/inference/failures.py @@ -0,0 +1,351 @@ +"""Recording and reporting failing inference evaluations, and reporting outcomes +excluded through `failure_codes`. +""" + +from __future__ import annotations + +import logging +import os +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +import pandas as pd + +from proteus.utils.helper import CommentFromStatus + +log = logging.getLogger('fwl.' + __name__) + +# Whether a failed child run aborts the study or scores a bad objective value. +_ABORT_ON_FAILURE_ENV = 'PROTEUS_INFERENCE_ABORT_ON_FAILURE' + +# Suffix for the file holding whatever a child wrote to its console. +CHILD_CONSOLE_SUFFIX = '_console.log' + +# Status written by PROTEUS before its output folder is cleaned. +STATUS_MISSING = -1 + +# How an evaluation that failed is classified. A run that +# crashed, or stopped in an error state, did not produce a result at all. A run +# that completed normally but ended on a status listed in the study's +# `failure_codes` did produce a result, but the study does not fit against +# that outcome. Only the first is a fault. +CATEGORY_FAILURE = 'failure' +CATEGORY_EXCLUDED = 'excluded' + +# Table inside the study output holding one row per unscored evaluation. +# Appended by the workers as they fail and read back once at the end, so that +# the summary covers initial sampling and optimisation alike without the two +# paths having to share any state while they run. +FAILURE_CSV = 'failures.csv' + +# Fixed columns of that table, in order. The swept parameter values follow, one +# column each. The two paths are what the user opens after: +# the logfile for a run that got far enough to configure its logger, +# the console capture for one that did not. +_FAILURE_COLUMNS = ( + 'worker', + 'iter', + 'category', + 'status', + 'status_desc', + 'exit_code', + 'reason', + 'out_dir', + 'log_path', + 'console_path', +) + +# Fraction of evaluations that may fail before the summary escalates from a +# report to a warning. Above this, the sampled region is mostly unrunnable and +# the posterior is built on too few real evaluations to mean much. +FAILURE_FRACTION_WARN = 0.5 + + +@dataclass(eq=False) +class ProteusRunFailure(RuntimeError): + """A single child PROTEUS run that did not produce a usable result. + + Carries everything needed to diagnose the run without opening the study + by hand: which evaluation it was, where its output landed, how it died, + what PROTEUS recorded in its status file, and the parameter values that + produced it. `category` separates a genuine fault from a run with an + excluded status. Both score the failure value, but only the first is + reported as something having gone wrong. + + Faults that would affect every evaluation (eg no `proteus` on PATH) + stay as ordinary exceptions so they abort the study instead of being + scored as a bad sample. + """ + + reason: str + worker: int + iter: int + out_dir: str + exit_code: int | None = None + status: int = STATUS_MISSING + log_path: str | None = None + console_path: str | None = None + parameters: dict = field(default_factory=dict) + category: str = CATEGORY_FAILURE + + @property + def status_desc(self) -> str: + """Human-readable form of the PROTEUS status code.""" + if self.status == STATUS_MISSING: + return 'no readable status file (died during start-up)' + return CommentFromStatus(self.status) + + def summary(self) -> str: + """Single-line description naming the outcome and where to look next.""" + verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' + parts = [ + f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', + f'status {self.status} ({self.status_desc})', + ] + # A zero exit code is the norm for every path except a crash, where it + # is the one number that says which signal or error ended the run. + if self.exit_code: + parts.append(f'exit code {self.exit_code}') + parts.append(f'output {self.out_dir}') + return '; '.join(parts) + + def report(self) -> str: + """The summary plus the detail that is too long to log on one line.""" + lines = [self.summary()] + if self.log_path: + lines.append(f' logfile = {self.log_path}') + # Named whether or not a logfile exists: a run that died before its + # logger was configured left nothing else behind to read. + if self.console_path: + lines.append(f' console = {self.console_path}') + if self.parameters: + pretty = ', '.join(f'{k}={v:g}' for k, v in sorted(self.parameters.items())) + lines.append(f' parameters = {pretty}') + return '\n'.join(lines) + + def __str__(self) -> str: + return self.report() + + def __reduce__(self): + # A failure raised inside a pool worker is pickled to be re-raised in + # the parent. BaseException.__reduce__ rebuilds from `self.args`, + # which a dataclass __init__ leaves empty, so the default would fail + # to reconstruct this class. Rebuild from the fields instead. + return ( + self.__class__, + ( + self.reason, + self.worker, + self.iter, + self.out_dir, + self.exit_code, + self.status, + self.log_path, + self.console_path, + self.parameters, + self.category, + ), + ) + + +def set_abort_on_failure(abort: bool = False) -> None: + """Record whether a failed child run should abort the whole study. + + Stored in the environment so it is visible to the main process and to any + spawned pool workers, matching how the child timeout is plumbed. + """ + os.environ[_ABORT_ON_FAILURE_ENV] = '1' if abort else '0' + + +def abort_on_failure() -> bool: + """Return whether a failed child run should abort the whole study. + + Defaults to False: an inference sweep is expected to visit parameter + combinations the simulator cannot integrate, and treating those as fatal + would make most studies unrunnable. Set the inference config field + `abort_on_failure` to stop at the first failure instead. + """ + return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' + + +def read_status(out_abs: Path | str) -> int: + """Read the PROTEUS status code from a finished run's output folder. + + Parameters + ---------- + - out_abs (Path | str): Absolute path to the run's output folder. + + Returns + ---------- + - int: The status code, or `STATUS_MISSING` when no readable status file + exists. A missing file is itself diagnostic: PROTEUS deletes the status + it writes at start-up when it cleans the output folder, and does not + write another until the main loop begins. + """ + try: + with open(Path(out_abs) / 'status', 'r') as f: + return int(f.readlines()[0].strip()) + except Exception: + return STATUS_MISSING + + +def find_run_logfile(out_abs: Path | str) -> str | None: + """Return the newest PROTEUS logfile in a run's output folder, if any. + + PROTEUS captures uncaught exceptions into this file, so it usually holds + the traceback for a crashed run. It does not exist for a run that failed + before the logger was configured. + """ + logs = sorted(Path(out_abs).glob('proteus_*.log')) + return str(logs[-1]) if logs else None + + +def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | None: + """Append one row to the study's failure table. + + Workers are separate processes with no shared state, so each appends its + own row rather than handing the failure back to the parent. The first + worker to fail creates the file with its header through an exclusive + create, which exactly one caller can win, and every later row is a single + append. A row is one `write` call of well under a pipe buffer, which the + kernel adds whole, so no lock is needed on a local filesystem. + + Parameters + ---------- + - study_abs (Path | str): Absolute path to the study output folder. + - failure (ProteusRunFailure): The failure to record. + + Returns + ---------- + - str | None: Path written, or None if the row could not be written. + Recording is best-effort: a study must not be brought down by a fault in + its own bookkeeping, so the failure being reported still reaches the log. + """ + row = {key: getattr(failure, key) for key in _FAILURE_COLUMNS if key != 'status_desc'} + row['status_desc'] = failure.status_desc + row.update(failure.parameters) + ordered = {key: row[key] for key in (*_FAILURE_COLUMNS, *failure.parameters)} + + target = Path(study_abs) / FAILURE_CSV + line = _csv_row(ordered.values()) + try: + target.parent.mkdir(parents=True, exist_ok=True) + try: + with open(target, 'x') as f: + f.write(_csv_row(ordered.keys()) + line) + except FileExistsError: + with open(target, 'a') as f: + f.write(line) + except OSError as err: + log.warning( + f'Could not record the failure of worker={failure.worker} ' + f'iter={failure.iter}: {err}' + ) + return None + return str(target) + + +def _csv_row(values) -> str: + """Render one CSV line, quoting the fields that need it.""" + fields = [] + for value in values: + text = '' if value is None else str(value) + if any(c in text for c in ',"\n'): + text = '"' + text.replace('"', '""') + '"' + fields.append(text) + return ','.join(fields) + '\n' + + +def read_failure_records(study_abs: Path | str) -> list[dict]: + """Read back the failure table written during a study. + + Parameters + ---------- + - study_abs (Path | str): Absolute path to the study output folder. + + Returns + ---------- + - list[dict]: One entry per unscored evaluation, ordered by worker then + iteration. An unreadable table is reported and treated as empty rather + than aborting the summary it feeds. + """ + target = Path(study_abs) / FAILURE_CSV + if not target.is_file(): + return [] + try: + table = pd.read_csv(target) + except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError) as err: + log.warning(f'Skipping unreadable failure table {target}: {err}') + return [] + table = table.sort_values(['worker', 'iter'], kind='stable') + # An absent exit code or logfile reads back as NaN, which would print as + # 'nan' in the summary and compare equal to nothing. + return table.astype(object).where(table.notna(), None).to_dict('records') + + +def summarise_failures(output: str, n_attempted: int) -> int: + """Report on the study's unscored evaluations. + + The breakdown by cause, and the per-run paths are reported together at the end + of the inference run. Runs that failed and runs that were excluded are counted apart. + + Parameters + ---------- + - output (str): Absolute path to the study output folder. + - n_attempted (int): Total evaluations attempted, initial samples included. + + Returns + ---------- + - int: Number of evaluations that carry the failure score. + """ + records = read_failure_records(output) + n_unscored = len(records) + + log.info('-----------------------------------') + if not n_unscored: + log.info(f'Unscored evaluations: none, all {n_attempted} evaluations were usable') + log.info('-----------------------------------') + return 0 + + # A record with no category describes a genuine fault + n_excluded = sum(1 for r in records if r.get('category') == CATEGORY_EXCLUDED) + n_failed = n_unscored - n_excluded + + frac = n_unscored / max(n_attempted, 1) + log.info( + f'Unscored evaluations: {n_unscored} of {n_attempted} evaluations ' + f'({100 * frac:.1f}%, initial samples included) carry the failure score ' + 'rather than a fit quality' + ) + log.info(f' {n_failed} did not produce a usable result') + log.info(f' {n_excluded} completed on a status this study excludes') + + # Grouped by cause, and labelled so that an excluded outcome is not read as + # something having gone wrong in the run that reached it. + log.info(f'{"Cause":52s} | Count') + for (category, desc), count in Counter( + (r.get('category') or CATEGORY_FAILURE, r.get('status_desc') or 'unknown') + for r in records + ).most_common(): + label = f'{desc} [excluded]' if category == CATEGORY_EXCLUDED else str(desc) + log.info(f'{label:52s} {count}') + # A few concrete places to look. A run that died before configuring its logger + # has no logfile, and its console capture is then the only record of why it refused to start. + sample = [rec.get('log_path') or rec.get('console_path') for rec in records] + sample = [path for path in sample if path][:3] + if sample: + log.info(f'Logfiles ({len(sample)} of {n_unscored} shown):') + for log_path in sample: + log.info(f' {log_path}') + log.info(f'Full list: {Path(output) / FAILURE_CSV}') + + if frac > FAILURE_FRACTION_WARN: + log.warning( + f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations failed or were' + f'excluded, so the result rests on {n_attempted - n_unscored} real ' + 'evaluations.' + ) + log.info('-----------------------------------') + + return n_unscored diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index f6028bf02..261cdc325 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -30,15 +30,15 @@ # bayesopt source files from proteus.inference.async_BO import checkpoint, parallel_process +from proteus.inference.failures import set_abort_on_failure, summarise_failures from proteus.inference.gen_D_init import create_init from proteus.inference.objective import ( WORKER_CONFIG_OVERRIDES, apply_nested_updates, prot_builder, - set_abort_on_failure, set_child_timeout, ) -from proteus.inference.utils import print_results, str_time, summarise_failures +from proteus.inference.utils import print_results, str_time from proteus.utils.coupler import get_proteus_directories from proteus.utils.helper import safe_rm from proteus.utils.logs import setup_logger diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 92a3075ac..8a86aac8a 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -1,10 +1,8 @@ from __future__ import annotations -import json import logging import os import subprocess -from dataclasses import asdict, dataclass, field from functools import partial from pathlib import Path @@ -13,10 +11,20 @@ import torch from numpy import log10 +from proteus.inference.failures import ( + CATEGORY_EXCLUDED, + CATEGORY_FAILURE, + CHILD_CONSOLE_SUFFIX, + STATUS_MISSING, + ProteusRunFailure, + abort_on_failure, + find_run_logfile, + read_status, + record_failure, +) from proteus.inference.transforms import unnormalize_parameters from proteus.utils.constants import element_list, gas_list from proteus.utils.coupler import get_proteus_directories, variable_is_logarithmic -from proteus.utils.helper import CommentFromStatus dtype = torch.double EPS_CLIP = 1e-10 @@ -32,48 +40,6 @@ DEFAULT_CHILD_TIMEOUT_S = 6 * 3600.0 _CHILD_TIMEOUT_ENV = 'PROTEUS_INFERENCE_CHILD_TIMEOUT_S' -# Whether a failed child run aborts the study or scores a bad objective value. -# Plumbed through the environment for the same reason as the timeout above. -_ABORT_ON_FAILURE_ENV = 'PROTEUS_INFERENCE_ABORT_ON_FAILURE' - -# Lines of child stderr retained in a failure report. PROTEUS writes its own -# diagnostics to the run's logfile, but a run that dies before the logger is -# set up (a rejected config, a missing environment variable) leaves nothing -# behind except this stream, so it is kept rather than discarded. -STDERR_TAIL_LINES = 40 - -# Suffix for the file holding whatever a child wrote to its console. It is -# kept beside the run folder rather than inside it: PROTEUS empties its own -# output folder once it starts, which would unlink a file held open there and -# lose exactly the record this capture exists to keep. -CHILD_CONSOLE_SUFFIX = '_console.log' - -# Status written by PROTEUS before its output folder is cleaned, and never -# rewritten until the main loop starts. A child that dies in between leaves no -# status file at all, so a missing file is reported as such rather than being -# silently reported as a generic error. -STATUS_MISSING = -1 - -# How an evaluation that carries no fit quality is classified. A run that -# crashed, or stopped in an error state, did not produce a result at all. A run -# that completed normally but ended on a status listed in the study's -# `failure_codes` did produce a result; the study simply does not fit against -# that outcome. Both score BAD_OBJ_VALUE, but only the first is a fault, so the -# two are named and counted apart rather than both being called failures. -CATEGORY_FAILURE = 'failure' -CATEGORY_EXCLUDED = 'excluded' - -# Folder inside the study output holding one record per failed evaluation. -# Written by the workers as they fail and read back once at the end, so that -# the summary covers initial sampling and optimisation alike without the two -# paths having to share any state while they run. -FAILURE_RECORD_DIR = 'failures' - -# Fraction of evaluations that may fail before the summary escalates from a -# report to a warning. Above this, the sampled region is mostly unrunnable and -# the posterior is built on too few real evaluations to mean much. -FAILURE_FRACTION_WARN = 0.5 - # Config entries every worker overwrites in the reference config, regardless of # which parameters are being swept. Shared with the startup validation so the # configuration that is checked is the configuration that is run. @@ -87,174 +53,6 @@ _FIXED_PARAMETER_KEYS = set(WORKER_CONFIG_OVERRIDES) | {'params.out.path'} -def _tail(text: str | bytes | None, lines: int = STDERR_TAIL_LINES) -> str: - """Return the last `lines` lines of captured child output.""" - if not text: - return '' - if isinstance(text, bytes): - text = text.decode('utf-8', errors='replace') - return '\n'.join(text.splitlines()[-lines:]) - - -def _tail_file(path: Path, lines: int = STDERR_TAIL_LINES) -> str: - """Return the last `lines` lines of a child's console file. - - Returns an empty string when the file is missing or unreadable, so a - failure report is still produced when the capture itself went wrong. - """ - try: - with open(path, 'r', errors='replace') as f: - return _tail(f.read(), lines) - except OSError: - return '' - - -@dataclass(eq=False) -class ProteusRunFailure(RuntimeError): - """A single child PROTEUS run that did not produce a usable result. - - Carries everything needed to diagnose the run without opening the study - by hand: which evaluation it was, where its output landed, how it died, - what PROTEUS recorded in its status file, and the parameter values that - produced it. `category` separates a genuine fault from a run that completed - normally on a status the study excludes; both score the failure value, but - only the first is reported as something having gone wrong. - - Raised for faults that are specific to one evaluation; faults that would - affect every evaluation (no `proteus` on PATH, an observable that no - helpfile column provides) stay as ordinary exceptions so they abort the - study instead of being scored as a bad sample. - """ - - reason: str - worker: int - iter: int - out_dir: str - exit_code: int | None = None - status: int = STATUS_MISSING - log_path: str | None = None - stderr_tail: str = '' - parameters: dict = field(default_factory=dict) - category: str = CATEGORY_FAILURE - - @property - def status_desc(self) -> str: - """Human-readable form of the PROTEUS status code.""" - if self.status == STATUS_MISSING: - return 'no readable status file (died during start-up)' - return CommentFromStatus(self.status) - - def summary(self) -> str: - """Single-line description naming the outcome and where to look next.""" - verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' - parts = [ - f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', - f'status {self.status} ({self.status_desc})', - ] - # A zero exit code is the norm for every path except a crash, where it - # is the one number that says which signal or error ended the run. - if self.exit_code: - parts.append(f'exit code {self.exit_code}') - parts.append(f'output {self.out_dir}') - return '; '.join(parts) - - def report(self) -> str: - """Multi-line description naming the cause and where to look next.""" - verb = 'excluded' if self.category == CATEGORY_EXCLUDED else 'failed' - lines = [ - f'PROTEUS run {verb} for worker={self.worker} iter={self.iter}: {self.reason}', - f' status = {self.status} ({self.status_desc})', - ] - if self.exit_code is not None: - lines.append(f' exit code = {self.exit_code}') - lines.append(f' output = {self.out_dir}') - if self.log_path: - lines.append(f' logfile = {self.log_path}') - if self.parameters: - pretty = ', '.join(f'{k}={v:g}' for k, v in sorted(self.parameters.items())) - lines.append(f' parameters = {pretty}') - if self.stderr_tail: - lines.append(' last output from the child process:') - lines.extend(f' {line}' for line in self.stderr_tail.splitlines()) - return '\n'.join(lines) - - def __str__(self) -> str: - return self.report() - - def __reduce__(self): - # A failure raised inside a pool worker is pickled to be re-raised in - # the parent. BaseException.__reduce__ rebuilds from `self.args`, - # which a dataclass __init__ leaves empty, so the default would fail - # to reconstruct this class. Rebuild from the fields instead. - return ( - self.__class__, - ( - self.reason, - self.worker, - self.iter, - self.out_dir, - self.exit_code, - self.status, - self.log_path, - self.stderr_tail, - self.parameters, - self.category, - ), - ) - - -def set_abort_on_failure(abort: bool = False) -> None: - """Record whether a failed child run should abort the whole study. - - Stored in the environment so it is visible to the main process and to any - spawned pool workers, matching how the child timeout is plumbed. - """ - os.environ[_ABORT_ON_FAILURE_ENV] = '1' if abort else '0' - - -def abort_on_failure() -> bool: - """Return whether a failed child run should abort the whole study. - - Defaults to False: an inference sweep is expected to visit parameter - combinations the simulator cannot integrate, and treating those as fatal - would make most studies unrunnable. Set the inference config field - `abort_on_failure` to stop at the first failure instead. - """ - return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' - - -def read_status(out_abs: Path | str) -> int: - """Read the PROTEUS status code from a finished run's output folder. - - Parameters - ---------- - - out_abs (Path | str): Absolute path to the run's output folder. - - Returns - ---------- - - int: The status code, or `STATUS_MISSING` when no readable status file - exists. A missing file is itself diagnostic: PROTEUS deletes the status - it writes at start-up when it cleans the output folder, and does not - write another until the main loop begins. - """ - try: - with open(Path(out_abs) / 'status', 'r') as f: - return int(f.readlines()[0].strip()) - except Exception: - return STATUS_MISSING - - -def find_run_logfile(out_abs: Path | str) -> str | None: - """Return the newest PROTEUS logfile in a run's output folder, if any. - - PROTEUS captures uncaught exceptions into this file, so it usually holds - the traceback for a crashed run. It does not exist for a run that failed - before the logger was configured. - """ - logs = sorted(Path(out_abs).glob('proteus_*.log')) - return str(logs[-1]) if logs else None - - def run_output_dir(output: str, worker: int, iter: int) -> tuple[Path, Path]: """Return the output folder of a single evaluation, relative and absolute. @@ -273,63 +71,6 @@ def run_output_dir(output: str, worker: int, iter: int) -> tuple[Path, Path]: return out_dir, Path(get_proteus_directories(str(out_dir))['output']) -def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | None: - """Write a failure record into the study's `failures` folder. - - One file per failed evaluation, named for the worker and iteration that - produced it, so that concurrent workers never write to the same file and - no lock is needed. The end-of-study summary reads them back. - - Parameters - ---------- - - study_abs (Path | str): Absolute path to the study output folder. - - failure (ProteusRunFailure): The failure to record. - - Returns - ---------- - - str | None: Path written, or None if the record could not be written. - Recording is best-effort: a study must not be brought down by a fault in - its own bookkeeping, so the failure being reported still reaches the log. - """ - record = asdict(failure) - record['status_desc'] = failure.status_desc - target = Path(study_abs) / FAILURE_RECORD_DIR / f'w{failure.worker}_i{failure.iter}.json' - try: - target.parent.mkdir(parents=True, exist_ok=True) - with open(target, 'w') as f: - json.dump(record, f, indent=2, sort_keys=True) - except (OSError, TypeError, ValueError) as err: - log.warning( - f'Could not record the failure of worker={failure.worker} ' - f'iter={failure.iter}: {err}' - ) - return None - return str(target) - - -def read_failure_records(study_abs: Path | str) -> list[dict]: - """Read back every failure record written during a study. - - Parameters - ---------- - - study_abs (Path | str): Absolute path to the study output folder. - - Returns - ---------- - - list[dict]: One entry per failed evaluation, ordered by worker then - iteration. Unreadable records are skipped with a warning rather than - aborting the summary, which would hide the failures that did parse. - """ - records = [] - for path in sorted((Path(study_abs) / FAILURE_RECORD_DIR).glob('w*_i*.json')): - try: - with open(path, 'r') as f: - records.append(json.load(f)) - except (OSError, json.JSONDecodeError) as err: - log.warning(f'Skipping unreadable failure record {path}: {err}') - return sorted(records, key=lambda r: (r.get('worker', 0), r.get('iter', 0))) - - def set_child_timeout(seconds: float | None = None) -> None: """Record the per-child PROTEUS timeout for inference worker processes. @@ -481,9 +222,11 @@ def run_proteus( # the worker overrides are fixed for every run and add no diagnostic value. swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} - def _failure( - reason: str, exit_code: int | None, stderr_tail: str = '' - ) -> ProteusRunFailure: + # A run that dies before its logger is configured leaves no logfile behind, so + # this stream records it. + console = out_abs.parent / f'{out_abs.name}{CHILD_CONSOLE_SUFFIX}' + + def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: """Assemble a failure report for this run. The status file is read here rather than at the point of the raise so @@ -498,18 +241,11 @@ def _failure( exit_code=exit_code, status=read_status(out_abs), log_path=find_run_logfile(out_abs), - stderr_tail=stderr_tail, + console_path=str(console), parameters=swept, ) - # Run PROTEUS. Output is kept rather than discarded: a run that dies - # before its logger is configured leaves no logfile behind, so this stream - # is the only record of why it refused to start. It goes to a file rather - # than a pipe because a long run of a chatty module would otherwise buffer - # hours of output in memory, in every worker at once, to retain a few - # lines of it. command = ['proteus', 'start', '-c', str(out_cfg), '--offline'] - console = out_abs.parent / f'{out_abs.name}{CHILD_CONSOLE_SUFFIX}' console.parent.mkdir(parents=True, exist_ok=True) # Opened outside the try so that a failure to create it is not mistaken # for the simulator being absent. @@ -531,17 +267,9 @@ def _failure( raise RuntimeError("Failed to run PROTEUS: 'proteus' command not found") from err except subprocess.TimeoutExpired as err: timeout = child_timeout_s() - raise _failure( - f'exceeded the {timeout} s timeout', - exit_code=None, - stderr_tail=_tail_file(console), - ) from err + raise _failure(f'exceeded the {timeout} s timeout', exit_code=None) from err except subprocess.CalledProcessError as err: - raise _failure( - 'the simulator exited with an error', - exit_code=err.returncode, - stderr_tail=_tail_file(console), - ) from err + raise _failure('the simulator exited with an error', exit_code=err.returncode) from err finally: stream.close() diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index 6e2f74a34..bb6f843a3 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -16,7 +16,6 @@ from __future__ import annotations import logging -from collections import Counter from collections.abc import Sequence from datetime import datetime from functools import partial @@ -33,15 +32,7 @@ from gpytorch.kernels import MaternKernel, RBFKernel from gpytorch.priors.torch_priors import LogNormalPrior -from proteus.inference.objective import ( - BAD_OBJ_VALUE, - CATEGORY_EXCLUDED, - CATEGORY_FAILURE, - EPS_CLIP, - FAILURE_FRACTION_WARN, - eval_obj, - read_failure_records, -) +from proteus.inference.objective import BAD_OBJ_VALUE, EPS_CLIP, eval_obj from proteus.inference.transforms import unnormalize_parameters from proteus.utils.constants import gas_list @@ -157,107 +148,6 @@ def load_dataset_csv(fpath: str) -> dict[str, torch.Tensor]: return {'X': X, 'Y': Y} -def summarise_failures(output: str, n_attempted: int) -> int: - """Collect the study's unscored evaluations into a table and report on them. - - A sweep over a wide parameter box is expected to reach combinations the - simulator cannot integrate, and to reach outcomes the study itself excludes - through `failure_codes`. Both carry the failure score rather than a fit - quality. Without a count, a study in which most evaluations were never - scored is indistinguishable from one that converged, so the tally, the - breakdown by cause, and the per-run paths are reported together at the end - of the study. Runs that failed and runs that were excluded are counted - apart, because only the first kind means something went wrong. - - Parameters - ---------- - - output (str): Absolute path to the study output folder. - - n_attempted (int): Total evaluations attempted, initial samples included. - - Returns - ---------- - - int: Number of evaluations that carry the failure score. - """ - records = read_failure_records(output) - n_unscored = len(records) - - log.info('-----------------------------------') - if not n_unscored: - log.info(f'Unscored evaluations: none, all {n_attempted} evaluations were usable') - log.info('-----------------------------------') - return 0 - - # A record with no category describes a genuine fault: it came either from - # the crash path, which never excludes, or from a study run before the two - # were separated. - n_excluded = sum(1 for r in records if r.get('category') == CATEGORY_EXCLUDED) - n_failed = n_unscored - n_excluded - - # Fixed diagnostic columns first, then one column per swept parameter, so - # the table can be sorted on a parameter to see which region fails. - rows = [] - for rec in records: - row = { - key: rec.get(key) - for key in ( - 'worker', - 'iter', - 'category', - 'status', - 'status_desc', - 'exit_code', - 'reason', - 'out_dir', - 'log_path', - ) - } - row['category'] = row['category'] or CATEGORY_FAILURE - row.update(rec.get('parameters') or {}) - rows.append(row) - csv_path = Path(output) / 'failures.csv' - pd.DataFrame(rows).to_csv(csv_path, index=False) - - frac = n_unscored / max(n_attempted, 1) - log.info( - f'Unscored evaluations: {n_unscored} of {n_attempted} evaluations ' - f'({100 * frac:.1f}%, initial samples included) carry the failure score ' - 'rather than a fit quality' - ) - log.info(f' {n_failed} did not produce a usable result') - log.info(f' {n_excluded} completed on a status this study excludes') - - # Grouped by cause, and labelled so that an excluded outcome is not read as - # something having gone wrong in the run that reached it. - log.info(f'{"Cause":52s} | Count') - for (category, desc), count in Counter( - (r.get('category') or CATEGORY_FAILURE, r.get('status_desc') or 'unknown') - for r in records - ).most_common(): - label = f'{desc} [excluded]' if category == CATEGORY_EXCLUDED else str(desc) - log.info(f'{label:52s} {count}') - # A few concrete places to look, labelled and counted so the sample is not - # read as the whole list. The simulator writes its own traceback to these - # logfiles, so they carry the cause that the status code only names. - sample = [rec['log_path'] for rec in records if rec.get('log_path')][:3] - if sample: - log.info(f'Logfiles ({len(sample)} of {n_unscored} shown):') - for log_path in sample: - log.info(f' {log_path}') - log.info(f'Full list: {csv_path}') - - if frac > FAILURE_FRACTION_WARN: - log.warning( - f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations were not scored ' - f'on fit quality, so the result below rests on {n_attempted - n_unscored} real ' - 'evaluations. Narrow the parameter ranges to a region the simulator can ' - 'integrate and the study accepts, or check the reference config against the ' - 'causes listed above.' - ) - log.info('-----------------------------------') - - return n_unscored - - def print_results(D, logs, config, output, n_init): """Identify the best evaluation and log its observables and inferred parameters. From 1b97cdd91f34bfedfcf38a14ff9a6bf34452e618 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 13:05:09 +0200 Subject: [PATCH 30/72] update tests with failures refactor --- tests/inference/test_failures.py | 439 +++++++++++++++++++++++++ tests/inference/test_objective.py | 219 ++---------- tests/inference/test_utils_branches.py | 215 ------------ 3 files changed, 460 insertions(+), 413 deletions(-) create mode 100644 tests/inference/test_failures.py diff --git a/tests/inference/test_failures.py b/tests/inference/test_failures.py new file mode 100644 index 000000000..14106a1e7 --- /dev/null +++ b/tests/inference/test_failures.py @@ -0,0 +1,439 @@ +""" +Unit tests for recording and reporting unscored inference evaluations. + +Covers `proteus.inference.failures`: what a failed or excluded evaluation +carries, how one is appended to the study's failure table without workers +contending for it, how that table is read back, and the end-of-study tally. + +References: + - docs/How-to/testing.md + - docs/Explanations/test_framework.md +""" + +from __future__ import annotations + +import logging +import pickle + +import pandas as pd +import pytest + +import proteus.inference.failures as failures_mod + +pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] + + +@pytest.mark.unit +def test_failure_summary_is_one_line_and_names_where_the_detail_is_kept(): + """The line a study logs for each unscored run identifies the run, names + the status code and points at the output folder, and stays on one line + however many parameters the study sweeps. The swept values and the paths to + open next belong to the fuller report, not to the one-liner. + """ + # Edge case: a wide sweep is the situation the one-liner exists for. Twenty + # parameters rendered inline would run past any terminal width. + swept = {f'planet.param_{i}': float(i) for i in range(20)} + swept['planet.mass_tot'] = 1.25 + failure = failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=2, + iter=16, + out_dir='/study/workers/w_2/i_16', + exit_code=1, + status=22, + log_path='/study/workers/w_2/i_16/proteus_00.log', + console_path='/study/workers/w_2/i_16_console.log', + parameters=swept, + ) + line = failure.summary() + + assert '\n' not in line + assert 'planet.mass_tot' not in line + assert 'proteus_00.log' not in line + # What has to survive the trim: who failed, what the status was, and the + # folder holding the logfile and the console capture. + assert 'worker=2 iter=16' in line + assert 'status 22' in line + assert 'Atmosphere' in line + assert 'exit code 1' in line + assert '/study/workers/w_2/i_16' in line + # Discrimination: the detail is not lost, only moved. A regression that + # trimmed `report` instead of adding a second renderer would fail here. + rendered = failure.report() + assert 'planet.mass_tot=1.25' in rendered + assert 'proteus_00.log' in rendered + assert 'i_16_console.log' in rendered + # The report opens with the same one-liner, so nothing the summary names is + # dropped on the way to the fuller form. + assert rendered.splitlines()[0] == line + + # Limit input: an excluded run has nothing to report as a fault, so it is + # named as excluded and its exit code, always zero on that path, is left + # out rather than read as a crash code. + excluded = failures_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=0, + iter=10, + out_dir='/study/workers/w_0/i_10', + exit_code=0, + status=11, + category=failures_mod.CATEGORY_EXCLUDED, + ) + excluded_line = excluded.summary() + assert '\n' not in excluded_line + assert 'excluded for worker=0 iter=10' in excluded_line + assert 'failed for worker=0' not in excluded_line + assert 'exit code' not in excluded_line + assert 'status 11' in excluded_line + + +@pytest.mark.unit +def test_proteus_run_failure_survives_the_trip_back_from_a_pool_worker(): + """A failure raised inside a pool worker is pickled and re-raised in the + parent process. Every reported field must survive that round trip, or the + parent sees a reconstruction error in place of the diagnosis. + """ + original = failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=2, + iter=7, + out_dir='/study/workers/w_2/i_7', + exit_code=1, + status=27, + log_path='/study/workers/w_2/i_7/proteus_00.log', + console_path='/study/workers/w_2/i_7_console.log', + parameters={'planet.mass_tot': 2.0}, + ) + restored = pickle.loads(pickle.dumps(original)) + + assert isinstance(restored, failures_mod.ProteusRunFailure) + assert restored.report() == original.report() + # Field-level guard: an equal report could still hide a dropped field that + # the renderer omits when empty, so pin the values that steer diagnosis. + assert restored.status == 27 + assert restored.worker == 2 and restored.iter == 7 + assert restored.parameters == {'planet.mass_tot': pytest.approx(2.0)} + assert restored.log_path == original.log_path + assert restored.console_path == original.console_path + assert restored.category == failures_mod.CATEGORY_FAILURE + + # The category rides along in the same tuple, and it decides whether the + # parent calls the run a fault. A field dropped from the reconstruction + # would fall back to the 'failure' default and go unnoticed on a failure, + # so the round trip is checked on the other value too. + excluded = failures_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=2, + iter=7, + out_dir='/study/workers/w_2/i_7', + exit_code=0, + status=11, + category=failures_mod.CATEGORY_EXCLUDED, + ) + restored_excluded = pickle.loads(pickle.dumps(excluded)) + assert restored_excluded.category == failures_mod.CATEGORY_EXCLUDED + assert 'excluded for worker=2' in restored_excluded.report() + assert 'failed for worker=2' not in restored_excluded.report() + + +@pytest.mark.unit +def test_failure_records_round_trip_into_one_table(tmp_path): + """Each worker appends its own row to the study's failure table, and the + rows are read back ordered by worker then iteration whatever order they + arrived in. The status description is stored rather than recomputed, so the + summary does not have to re-derive it from the code. + """ + first = failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=5, + out_dir='/study/workers/w_0/i_5', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 3.0}, + ) + # Same iteration, different worker: two rows, not one overwriting the other. + second = failures_mod.ProteusRunFailure( + reason='exceeded the 3600.0 s timeout', + worker=1, + iter=5, + out_dir='/study/workers/w_1/i_5', + status=failures_mod.STATUS_MISSING, + parameters={'planet.mass_tot': 4.0}, + ) + + # Written out of order: the second worker fails first. + assert failures_mod.record_failure(tmp_path, second) is not None + assert failures_mod.record_failure(tmp_path, first) is not None + table = tmp_path / failures_mod.FAILURE_CSV + # One header however many workers append, so the table parses as one frame. + assert table.read_text().count('worker,iter,') == 1 + + records = failures_mod.read_failure_records(tmp_path) + # Ordering guard: written second-then-first, read back in worker order. + assert [r['worker'] for r in records] == [0, 1] + assert records[0]['status'] == 21 + assert records[0]['status_desc'] == first.status_desc + # Swept values are columns of their own, so the table can be sorted on a + # parameter to see which region of the box fails. + assert records[0]['planet.mass_tot'] == pytest.approx(3.0) + assert records[1]['planet.mass_tot'] == pytest.approx(4.0) + # A run that never wrote a status file is stored as such, not as a generic + # error, so the summary can separate start-up deaths from model faults. + assert records[1]['status'] == failures_mod.STATUS_MISSING + assert 'no readable status file' in records[1]['status_desc'] + # An absent exit code reads back as None, not as the string 'nan', which + # would print into the summary as though it were a code the child returned. + assert records[1]['exit_code'] is None + + # Edge case: a field holding the delimiter is quoted on the way out, or it + # would shift every later column of that row by one. + comma = failures_mod.ProteusRunFailure( + reason='the simulator exited with an error, code 3', + worker=2, + iter=0, + out_dir='/study/workers/w_2/i_0', + exit_code=3, + status=21, + parameters={'planet.mass_tot': 5.0}, + ) + assert failures_mod.record_failure(tmp_path, comma) is not None + reread = failures_mod.read_failure_records(tmp_path)[-1] + assert reread['reason'] == 'the simulator exited with an error, code 3' + assert reread['planet.mass_tot'] == pytest.approx(5.0) + + # Edge case: a table that cannot be parsed is reported and treated as empty + # rather than aborting the summary it feeds. A zero-length file is the + # reachable form of this: a worker killed between creating the table and + # writing its first row leaves exactly that behind. + table.write_text('') + assert failures_mod.read_failure_records(tmp_path) == [] + + +@pytest.mark.unit +def test_recording_a_failure_never_masks_the_failure_it_records(tmp_path): + """Bookkeeping must not bring down a study. When the record cannot be + written the writer reports that it could not, and the caller still has the + failure in hand to log and to score. + """ + blocked = tmp_path / 'not_a_directory' + blocked.write_text('this is a file, so no folder can be made beneath it') + failure = failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=0, + out_dir=str(tmp_path), + exit_code=1, + status=21, + ) + + assert failures_mod.record_failure(blocked, failure) is None + # Discrimination: the same failure records fine against a usable folder, so + # the None above came from the blocked path and not from a writer that + # always fails. + assert failures_mod.record_failure(tmp_path / 'study', failure) is not None + # Reading a study that never created the folder is empty, not an error. + assert failures_mod.read_failure_records(tmp_path / 'never_ran') == [] + + +@pytest.mark.unit +def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp_path, caplog): + """The end-of-study tally turns the per-run records into one table and one + breakdown by cause, and escalates to a warning once most of the study + failed. Without the escalation, a posterior built on a handful of real + evaluations reads the same as one built on all of them. + """ + + # Two runs that died the same way and one that died differently, so the + # breakdown has something to group. + for worker, status in ((0, 21), (1, 21), (2, 24)): + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=1, + status=status, + parameters={'planet.mass_tot': 1.0 + worker}, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + n_failed = failures_mod.summarise_failures(str(tmp_path), n_attempted=20) + + assert n_failed == 3 + messages = '\n'.join(r.message for r in caplog.records) + assert '3 of 20 evaluations' in messages + # Grouped by cause, so two runs that died the same way count as one line. + assert 'Interior model' in messages + # Below the escalation threshold (3/20 = 15%), the tally is reported but + # not warned about. + assert not [r for r in caplog.records if r.levelname == 'WARNING'] + + # The table carries the swept parameter alongside the diagnosis, so the + # failing region can be located without opening each run folder. + table = pd.read_csv(tmp_path / 'failures.csv') + assert len(table) == 3 + assert list(table['worker']) == [0, 1, 2] + assert sorted(table['status']) == [21, 21, 24] + assert table['planet.mass_tot'].max() == pytest.approx(3.0) + + # Discrimination: the same three failures against a smaller study cross the + # threshold and are warned about. A tally without the escalation would log + # identically in both cases. + caplog.clear() + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + failures_mod.summarise_failures(str(tmp_path), n_attempted=4) + warnings = [r for r in caplog.records if r.levelname == 'WARNING'] + assert len(warnings) == 1 + # The count of evaluations that are real is what the reader needs. + assert '1 real evaluations' in warnings[0].message + + +@pytest.mark.unit +def test_summarise_failures_counts_excluded_outcomes_apart_from_failures(tmp_path, caplog): + """A run that completed on a status the study excludes is tallied, but not + as a fault. Folding the two together would tell the user that a study whose + runs all reached their clock limit, exactly as configured, is a study full + of broken simulations. + """ + + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=0, + out_dir='/study/workers/w_0/i_0', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 1.0}, + ), + ) + for worker in (1, 2): + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=0, + status=11, + parameters={'planet.mass_tot': 1.0 + worker}, + category=failures_mod.CATEGORY_EXCLUDED, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + n_unscored = failures_mod.summarise_failures(str(tmp_path), n_attempted=20) + + # Both kinds are unscored, so both count toward how much of the study was + # real, but the breakdown names them apart. + assert n_unscored == 3 + messages = '\n'.join(r.message for r in caplog.records) + assert '1 did not produce a usable result' in messages + assert '2 completed on a status this study excludes' in messages + # The clock-limit outcome is labelled in the cause table rather than being + # listed beside the interior-model error as if it were one. + assert 'Completed (maximum clock runtime) [excluded]' in messages + assert 'Error (Interior model) [excluded]' not in messages + + # Carried into the table too, so the excluded rows can be filtered out when + # looking for the region that actually breaks the simulator. + table = pd.read_csv(tmp_path / 'failures.csv') + assert sorted(table['category']) == ['excluded', 'excluded', 'failure'] + assert sorted(table.loc[table['category'] == 'excluded', 'status']) == [11, 11] + + +@pytest.mark.unit +def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_path, caplog): + """A study in which nothing failed says so and writes no table. An empty + failures.csv would suggest the accounting had run and found nothing to + say about a study that in fact had nothing to report. + """ + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + n_failed = failures_mod.summarise_failures(str(tmp_path), n_attempted=12) + + assert n_failed == 0 + assert not (tmp_path / 'failures.csv').exists() + messages = '\n'.join(r.message for r in caplog.records) + assert 'none' in messages and '12 evaluations' in messages + assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] + + +@pytest.mark.unit +def test_summarise_failures_labels_the_logfile_sample_and_counts_the_whole_study( + tmp_path, caplog +): + """The tally covers every evaluation attempted, initial samples included, + while the warning raised alongside the best fit covers the optimisation + steps alone. The logfile lines are a sample of at most three, so they are + labelled with how many of the total they show and printed above the pointer + to the full table; unlabelled, three paths below a "Full list" line read as + the complete set. + """ + + # Four records, the first of which has no logfile: the run died before the + # child wrote one. The sample must skip it and still offer three paths. + for worker in range(4): + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=1, + status=21, + log_path=None if worker == 0 else f'/study/w_{worker}/proteus_00.log', + parameters={'planet.mass_tot': 1.0 + worker}, + ), + ) + + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + failures_mod.summarise_failures(str(tmp_path), n_attempted=20) + + lines = [r.message for r in caplog.records] + messages = '\n'.join(lines) + # The denominator of the tally is the whole study, stated in the line + # itself so it cannot be confused with the optimisation-only warning. + assert '4 of 20 evaluations' in messages + assert 'initial samples included' in messages + # Three shown out of four unscored, not four out of four. + assert 'Logfiles (3 of 4 shown):' in messages + shown = [line.strip() for line in lines if line.strip().endswith('proteus_00.log')] + assert len(shown) == 3 + # The record without a logfile is skipped rather than truncating the + # sample to the two paths that follow it in the first three records. + assert '/study/w_1/proteus_00.log' in shown + assert '/study/w_3/proteus_00.log' in shown + + # The pointer to the complete table comes after the sample, so the sample + # cannot be read as a continuation of it. + i_sample = next(i for i, line in enumerate(lines) if line.startswith('Logfiles (')) + i_full = next(i for i, line in enumerate(lines) if line.startswith('Full list:')) + assert i_sample < i_full + + # Discrimination: with no logfile recorded anywhere, no sample block is + # emitted at all, so the label tracks the data rather than always printing. + caplog.clear() + (tmp_path / 'failures.csv').rename(tmp_path / 'failures_old.csv') + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=0, + iter=1, + out_dir='/study/workers/w_0/i_1', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 1.0}, + ), + ) + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + failures_mod.summarise_failures(str(tmp_path), n_attempted=20) + assert not [r for r in caplog.records if r.message.startswith('Logfiles (')] diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 641a0fbce..c74fc71d1 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -9,7 +9,6 @@ from __future__ import annotations import logging -import pickle import subprocess import pandas as pd @@ -23,6 +22,7 @@ pytest.importorskip('botorch') pytest.importorskip('gpytorch') +import proteus.inference.failures as failures_mod # noqa: E402 import proteus.inference.objective as objective_mod # noqa: E402 pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] @@ -246,9 +246,6 @@ def _fake_run(*_args, **kwargs): # because they carry no information about which sample failed. assert failure.parameters == {'planet.mass_tot': pytest.approx(1.25)} assert 'params.out.path' not in failure.parameters - # Captured child output is retained for runs that die before their own - # logger exists, which is the only record such a run leaves behind. - assert 'boom' in failure.stderr_tail # The capture is kept beside the run folder, not inside it: the simulator # empties its own output directory once it starts, which would unlink a # file held open there. @@ -256,9 +253,14 @@ def _fake_run(*_args, **kwargs): assert console.is_file() assert 'boom' in console.read_text(encoding='utf-8') assert not (out_abs / console.name).exists() + # The failure names that capture rather than copying its contents. A run + # that dies before its own logger exists leaves nothing else to read, so a + # report that named no path would leave the cause unreachable. + assert failure.console_path == str(console) rendered = failure.report() assert 'worker=0 iter=0' in rendered assert 'planet.mass_tot=1.25' in rendered + assert str(console) in rendered @pytest.mark.unit @@ -588,111 +590,6 @@ def test_run_proteus_reports_a_clean_exit_that_produced_no_output(monkeypatch, t assert obs['P_surf'] == pytest.approx(2.5) -@pytest.mark.unit -def test_failure_summary_is_one_line_and_names_where_the_detail_is_kept(): - """The line a study logs for each unscored run identifies the run, names - the status code and points at the output folder, and stays on one line - however much the child wrote before it died. The child console tail and the - swept parameters belong to the on-disk record, not to the console. - """ - tail = '\n'.join(f'flux warning {i}' for i in range(200)) - failure = objective_mod.ProteusRunFailure( - reason='the simulator exited with an error', - worker=2, - iter=16, - out_dir='/study/workers/w_2/i_16', - exit_code=1, - status=22, - log_path='/study/workers/w_2/i_16/proteus_00.log', - stderr_tail=tail, - parameters={'planet.mass_tot': 1.25}, - ) - line = failure.summary() - - # Edge case: a 200-line tail is the situation the one-liner exists for. - assert '\n' not in line - assert 'flux warning 199' not in line - assert 'planet.mass_tot' not in line - # What has to survive the trim: who failed, what the status was, and the - # folder holding the logfile and the console capture. - assert 'worker=2 iter=16' in line - assert 'status 22' in line - assert 'Atmosphere' in line - assert 'exit code 1' in line - assert '/study/workers/w_2/i_16' in line - # Discrimination: the detail is not lost, only moved. A regression that - # trimmed `report` instead of adding a second renderer would fail here. - assert 'flux warning 199' in failure.report() - assert 'planet.mass_tot=1.25' in failure.report() - - # Limit input: an excluded run has nothing to report as a fault, so it is - # named as excluded and its exit code, always zero on that path, is left - # out rather than read as a crash code. - excluded = objective_mod.ProteusRunFailure( - reason='completed on a status this study excludes', - worker=0, - iter=10, - out_dir='/study/workers/w_0/i_10', - exit_code=0, - status=11, - category=objective_mod.CATEGORY_EXCLUDED, - ) - excluded_line = excluded.summary() - assert '\n' not in excluded_line - assert 'excluded for worker=0 iter=10' in excluded_line - assert 'failed for worker=0' not in excluded_line - assert 'exit code' not in excluded_line - assert 'status 11' in excluded_line - - -@pytest.mark.unit -def test_proteus_run_failure_survives_the_trip_back_from_a_pool_worker(): - """A failure raised inside a pool worker is pickled and re-raised in the - parent process. Every reported field must survive that round trip, or the - parent sees a reconstruction error in place of the diagnosis. - """ - original = objective_mod.ProteusRunFailure( - reason='the simulator exited with an error', - worker=2, - iter=7, - out_dir='/study/workers/w_2/i_7', - exit_code=1, - status=27, - log_path='/study/workers/w_2/i_7/proteus_00.log', - stderr_tail='Error: outgassing failed', - parameters={'planet.mass_tot': 2.0}, - ) - restored = pickle.loads(pickle.dumps(original)) - - assert isinstance(restored, objective_mod.ProteusRunFailure) - assert restored.report() == original.report() - # Field-level guard: an equal report could still hide a dropped field that - # the renderer omits when empty, so pin the values that steer diagnosis. - assert restored.status == 27 - assert restored.worker == 2 and restored.iter == 7 - assert restored.parameters == {'planet.mass_tot': pytest.approx(2.0)} - assert restored.log_path == original.log_path - assert restored.category == objective_mod.CATEGORY_FAILURE - - # The category rides along in the same tuple, and it decides whether the - # parent calls the run a fault. A field dropped from the reconstruction - # would fall back to the 'failure' default and go unnoticed on a failure, - # so the round trip is checked on the other value too. - excluded = objective_mod.ProteusRunFailure( - reason='completed on a status this study excludes', - worker=2, - iter=7, - out_dir='/study/workers/w_2/i_7', - exit_code=0, - status=11, - category=objective_mod.CATEGORY_EXCLUDED, - ) - restored_excluded = pickle.loads(pickle.dumps(excluded)) - assert restored_excluded.category == objective_mod.CATEGORY_EXCLUDED - assert 'excluded for worker=2' in restored_excluded.report() - assert 'failed for worker=2' not in restored_excluded.report() - - @pytest.mark.unit def test_J_scores_a_failed_run_badly_and_keeps_the_study_running(monkeypatch, tmp_path, caplog): """A parameter combination the simulator cannot integrate is scored as a @@ -717,7 +614,7 @@ def _fail(**_kwargs): raise failure monkeypatch.setattr(objective_mod, 'run_proteus', _fail) - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') with caplog.at_level('WARNING'): value = objective_mod.J( @@ -743,13 +640,13 @@ def _fail(**_kwargs): # The same failure is left on disk for the end-of-study tally, because a # log line scrolls past and a study that failed mostly needs a count. - recorded = objective_mod.read_failure_records(tmp_path) + recorded = failures_mod.read_failure_records(tmp_path) assert [(r['worker'], r['iter'], r['status']) for r in recorded] == [(1, 2, 21)] - assert recorded[0]['parameters']['planet.mass_tot'] == pytest.approx(3.0) + assert recorded[0]['planet.mass_tot'] == pytest.approx(3.0) # Opting in turns the same failure into a hard stop. `set_abort_on_failure` # is the writer under test; monkeypatch restores the variable afterwards. - objective_mod.set_abort_on_failure(True) + failures_mod.set_abort_on_failure(True) with pytest.raises(objective_mod.ProteusRunFailure): objective_mod.J( x=torch.tensor([[0.5]], dtype=torch.double), @@ -780,7 +677,7 @@ def test_J_scores_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_pa 'run_proteus', lambda **_kwargs: ({'R_obs': 9.25e6}, 25), ) - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') with caplog.at_level('WARNING'): value = objective_mod.J( @@ -796,7 +693,7 @@ def test_J_scores_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_pa assert 'status 25' in '\n'.join(r.getMessage() for r in caplog.records) # Counted in the end-of-study tally alongside the runs that crashed. A # tally that covered only crashes would understate a study stopped by hand. - recorded = objective_mod.read_failure_records(tmp_path) + recorded = failures_mod.read_failure_records(tmp_path) assert [r['status'] for r in recorded] == [25] assert recorded[0]['exit_code'] == 0 @@ -863,7 +760,7 @@ def _run(status, worker, iter, codes=()): failure_codes=list(codes), ) - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '1') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '1') # Status 25: written when a run is stopped through its keepalive file, so # the simulator exits 0 and the fault is visible only in the status file. @@ -885,7 +782,7 @@ def _run(status, worker, iter, codes=()): # The record is written before the abort, so an aborted study still says # on disk what stopped it rather than leaving only the traceback. - recorded = objective_mod.read_failure_records(tmp_path) + recorded = failures_mod.read_failure_records(tmp_path) # Ordered by (worker, iter), so the status-25 run at iter 0 comes first. assert [r['status'] for r in recorded] == [25, objective_mod.STATUS_MISSING] @@ -896,13 +793,13 @@ def _run(status, worker, iter, codes=()): # Boundedness: the failure score sits far below anything a completed run # can reach, so the optimiser is not drawn toward the excluded region. assert excluded.item() < -10.0 - assert objective_mod.read_failure_records(tmp_path)[-1]['category'] == ( + assert failures_mod.read_failure_records(tmp_path)[-1]['category'] == ( objective_mod.CATEGORY_EXCLUDED ) # Discrimination against a regression that raises unconditionally: with # the setting off, the same error status is scored and the study goes on. - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') scored = _run(25, worker=2, iter=0) assert scored.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) assert scored.item() < -10.0 @@ -915,7 +812,7 @@ def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): ('planet evaporated') and no current code path writes it, so it must not be scored as a failure by an off-by-one in the range bound. """ - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} ) @@ -958,7 +855,7 @@ def test_J_separates_an_excluded_outcome_from_a_failed_run(monkeypatch, tmp_path nothing usable. Reporting the first as the second sends the user looking for a bug in a run that did exactly what it was configured to do. """ - monkeypatch.setenv(objective_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} ) @@ -995,7 +892,7 @@ def _score(status, worker): # The record is kept for the end-of-study tally, labelled so the tally can # count it apart from the runs that genuinely failed. - recorded = objective_mod.read_failure_records(tmp_path) + recorded = failures_mod.read_failure_records(tmp_path) assert [(r['status'], r['category']) for r in recorded] == [ (11, objective_mod.CATEGORY_EXCLUDED) ] @@ -1012,7 +909,7 @@ def _score(status, worker): assert 'failed for worker=1' in warnings[0].getMessage() assert 'stopped in a failure state' in warnings[0].getMessage() assert 'status 21' in warnings[0].getMessage() - recorded = objective_mod.read_failure_records(tmp_path) + recorded = failures_mod.read_failure_records(tmp_path) assert [(r['status'], r['category']) for r in recorded] == [ (11, objective_mod.CATEGORY_EXCLUDED), (21, objective_mod.CATEGORY_FAILURE), @@ -1022,7 +919,7 @@ def _score(status, worker): # scored on its observables and leaves no record at all. The exact match on # a linear observable has the closed form -log10(0 + 1e-10) = 10. assert _score(13, worker=2) == pytest.approx(10.0, rel=1e-9) - assert len(objective_mod.read_failure_records(tmp_path)) == 2 + assert len(failures_mod.read_failure_records(tmp_path)) == 2 # ============================================================================ @@ -1051,77 +948,3 @@ def test_run_output_dir_names_the_folder_the_simulator_is_given(monkeypatch, tmp rel_init, _ = objective_mod.run_output_dir('study', -1, 7) assert rel_init.as_posix() == 'study/workers/w_-1/i_7' assert rel_init != rel - - -@pytest.mark.unit -def test_failure_records_round_trip_without_colliding_between_workers(tmp_path): - """Records are written one file per evaluation so that concurrent workers - never contend for the same file, and are read back ordered by worker then - iteration. The status description is stored rather than recomputed, so the - summary does not have to re-derive it from the code. - """ - first = objective_mod.ProteusRunFailure( - reason='the simulator exited with an error', - worker=0, - iter=5, - out_dir='/study/workers/w_0/i_5', - exit_code=1, - status=21, - parameters={'planet.mass_tot': 3.0}, - ) - # Same iteration, different worker: the pair is what makes the name unique. - second = objective_mod.ProteusRunFailure( - reason='exceeded the 3600.0 s timeout', - worker=1, - iter=5, - out_dir='/study/workers/w_1/i_5', - status=objective_mod.STATUS_MISSING, - parameters={'planet.mass_tot': 4.0}, - ) - - assert objective_mod.record_failure(tmp_path, second) is not None - assert objective_mod.record_failure(tmp_path, first) is not None - written = sorted(p.name for p in (tmp_path / objective_mod.FAILURE_RECORD_DIR).iterdir()) - assert written == ['w0_i5.json', 'w1_i5.json'] - - records = objective_mod.read_failure_records(tmp_path) - # Ordering guard: written second-then-first, read back in worker order. - assert [r['worker'] for r in records] == [0, 1] - assert records[0]['status'] == 21 - assert records[0]['status_desc'] == first.status_desc - assert records[0]['parameters']['planet.mass_tot'] == pytest.approx(3.0) - # A run that never wrote a status file is stored as such, not as a generic - # error, so the summary can separate start-up deaths from model faults. - assert records[1]['status'] == objective_mod.STATUS_MISSING - assert 'no readable status file' in records[1]['status_desc'] - - # Edge case: a corrupt record is skipped rather than aborting the summary, - # which would hide the records that did parse. - (tmp_path / objective_mod.FAILURE_RECORD_DIR / 'w2_i0.json').write_text('{not json') - assert len(objective_mod.read_failure_records(tmp_path)) == 2 - - -@pytest.mark.unit -def test_recording_a_failure_never_masks_the_failure_it_records(tmp_path): - """Bookkeeping must not bring down a study. When the record cannot be - written the writer reports that it could not, and the caller still has the - failure in hand to log and to score. - """ - blocked = tmp_path / 'not_a_directory' - blocked.write_text('this is a file, so no folder can be made beneath it') - failure = objective_mod.ProteusRunFailure( - reason='the simulator exited with an error', - worker=0, - iter=0, - out_dir=str(tmp_path), - exit_code=1, - status=21, - ) - - assert objective_mod.record_failure(blocked, failure) is None - # Discrimination: the same failure records fine against a usable folder, so - # the None above came from the blocked path and not from a writer that - # always fails. - assert objective_mod.record_failure(tmp_path / 'study', failure) is not None - # Reading a study that never created the folder is empty, not an error. - assert objective_mod.read_failure_records(tmp_path / 'never_ran') == [] diff --git a/tests/inference/test_utils_branches.py b/tests/inference/test_utils_branches.py index c4072fe9c..2ada8a980 100644 --- a/tests/inference/test_utils_branches.py +++ b/tests/inference/test_utils_branches.py @@ -16,7 +16,6 @@ import logging import numpy as np -import pandas as pd import pytest # The Bayesian-optimisation stack ships as the optional `inference` extra, @@ -362,217 +361,3 @@ def test_print_results_counts_unscored_runs_and_refuses_a_study_with_no_fit(tmp_ RuntimeError, match='None of the 2 optimisation evaluations produced a fit quality' ): print_results(D_dead, logs, config, str(tmp_path), n_init=1) - - -@pytest.mark.unit -def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp_path, caplog): - """The end-of-study tally turns the per-run records into one table and one - breakdown by cause, and escalates to a warning once most of the study - failed. Without the escalation, a posterior built on a handful of real - evaluations reads the same as one built on all of them. - """ - from proteus.inference.objective import ProteusRunFailure, record_failure - from proteus.inference.utils import summarise_failures - - # Two runs that died the same way and one that died differently, so the - # breakdown has something to group. - for worker, status in ((0, 21), (1, 21), (2, 24)): - record_failure( - tmp_path, - ProteusRunFailure( - reason='the simulator exited with an error', - worker=worker, - iter=0, - out_dir=f'/study/workers/w_{worker}/i_0', - exit_code=1, - status=status, - parameters={'planet.mass_tot': 1.0 + worker}, - ), - ) - - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - n_failed = summarise_failures(str(tmp_path), n_attempted=20) - - assert n_failed == 3 - messages = '\n'.join(r.message for r in caplog.records) - assert '3 of 20 evaluations' in messages - # Grouped by cause, so two runs that died the same way count as one line. - assert 'Interior model' in messages - # Below the escalation threshold (3/20 = 15%), the tally is reported but - # not warned about. - assert not [r for r in caplog.records if r.levelname == 'WARNING'] - - # The table carries the swept parameter alongside the diagnosis, so the - # failing region can be located without opening each run folder. - table = pd.read_csv(tmp_path / 'failures.csv') - assert len(table) == 3 - assert list(table['worker']) == [0, 1, 2] - assert sorted(table['status']) == [21, 21, 24] - assert table['planet.mass_tot'].max() == pytest.approx(3.0) - - # Discrimination: the same three failures against a smaller study cross the - # threshold and are warned about. A tally without the escalation would log - # identically in both cases. - caplog.clear() - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - summarise_failures(str(tmp_path), n_attempted=4) - warnings = [r for r in caplog.records if r.levelname == 'WARNING'] - assert len(warnings) == 1 - # The count of evaluations that are real is what the reader needs. - assert '1 real evaluations' in warnings[0].message - - -@pytest.mark.unit -def test_summarise_failures_counts_excluded_outcomes_apart_from_failures(tmp_path, caplog): - """A run that completed on a status the study excludes is tallied, but not - as a fault. Folding the two together would tell the user that a study whose - runs all reached their clock limit, exactly as configured, is a study full - of broken simulations. - """ - from proteus.inference.objective import ( - CATEGORY_EXCLUDED, - ProteusRunFailure, - record_failure, - ) - from proteus.inference.utils import summarise_failures - - record_failure( - tmp_path, - ProteusRunFailure( - reason='the simulator exited with an error', - worker=0, - iter=0, - out_dir='/study/workers/w_0/i_0', - exit_code=1, - status=21, - parameters={'planet.mass_tot': 1.0}, - ), - ) - for worker in (1, 2): - record_failure( - tmp_path, - ProteusRunFailure( - reason='completed on a status this study excludes', - worker=worker, - iter=0, - out_dir=f'/study/workers/w_{worker}/i_0', - exit_code=0, - status=11, - parameters={'planet.mass_tot': 1.0 + worker}, - category=CATEGORY_EXCLUDED, - ), - ) - - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - n_unscored = summarise_failures(str(tmp_path), n_attempted=20) - - # Both kinds are unscored, so both count toward how much of the study was - # real, but the breakdown names them apart. - assert n_unscored == 3 - messages = '\n'.join(r.message for r in caplog.records) - assert '1 did not produce a usable result' in messages - assert '2 completed on a status this study excludes' in messages - # The clock-limit outcome is labelled in the cause table rather than being - # listed beside the interior-model error as if it were one. - assert 'Completed (maximum clock runtime) [excluded]' in messages - assert 'Error (Interior model) [excluded]' not in messages - - # Carried into the table too, so the excluded rows can be filtered out when - # looking for the region that actually breaks the simulator. - table = pd.read_csv(tmp_path / 'failures.csv') - assert sorted(table['category']) == ['excluded', 'excluded', 'failure'] - assert sorted(table.loc[table['category'] == 'excluded', 'status']) == [11, 11] - - -@pytest.mark.unit -def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_path, caplog): - """A study in which nothing failed says so and writes no table. An empty - failures.csv would suggest the accounting had run and found nothing to - say about a study that in fact had nothing to report. - """ - from proteus.inference.utils import summarise_failures - - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - n_failed = summarise_failures(str(tmp_path), n_attempted=12) - - assert n_failed == 0 - assert not (tmp_path / 'failures.csv').exists() - messages = '\n'.join(r.message for r in caplog.records) - assert 'none' in messages and '12 evaluations' in messages - assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] - - -@pytest.mark.unit -def test_summarise_failures_labels_the_logfile_sample_and_counts_the_whole_study( - tmp_path, caplog -): - """The tally covers every evaluation attempted, initial samples included, - while the warning raised alongside the best fit covers the optimisation - steps alone. The logfile lines are a sample of at most three, so they are - labelled with how many of the total they show and printed above the pointer - to the full table; unlabelled, three paths below a "Full list" line read as - the complete set. - """ - from proteus.inference.objective import ProteusRunFailure, record_failure - from proteus.inference.utils import summarise_failures - - # Four records, the first of which has no logfile: the run died before the - # child wrote one. The sample must skip it and still offer three paths. - for worker in range(4): - record_failure( - tmp_path, - ProteusRunFailure( - reason='the simulator exited with an error', - worker=worker, - iter=0, - out_dir=f'/study/workers/w_{worker}/i_0', - exit_code=1, - status=21, - log_path=None if worker == 0 else f'/study/w_{worker}/proteus_00.log', - parameters={'planet.mass_tot': 1.0 + worker}, - ), - ) - - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - summarise_failures(str(tmp_path), n_attempted=20) - - lines = [r.message for r in caplog.records] - messages = '\n'.join(lines) - # The denominator of the tally is the whole study, stated in the line - # itself so it cannot be confused with the optimisation-only warning. - assert '4 of 20 evaluations' in messages - assert 'initial samples included' in messages - # Three shown out of four unscored, not four out of four. - assert 'Logfiles (3 of 4 shown):' in messages - shown = [line.strip() for line in lines if line.strip().endswith('proteus_00.log')] - assert len(shown) == 3 - # The record without a logfile is skipped rather than truncating the - # sample to the two paths that follow it in the first three records. - assert '/study/w_1/proteus_00.log' in shown - assert '/study/w_3/proteus_00.log' in shown - - # The pointer to the complete table comes after the sample, so the sample - # cannot be read as a continuation of it. - i_sample = next(i for i, line in enumerate(lines) if line.startswith('Logfiles (')) - i_full = next(i for i, line in enumerate(lines) if line.startswith('Full list:')) - assert i_sample < i_full - - # Discrimination: with no logfile recorded anywhere, no sample block is - # emitted at all, so the label tracks the data rather than always printing. - caplog.clear() - (tmp_path / 'failures').rename(tmp_path / 'failures_old') - record_failure( - tmp_path, - ProteusRunFailure( - reason='the simulator exited with an error', - worker=0, - iter=1, - out_dir='/study/workers/w_0/i_1', - exit_code=1, - status=21, - parameters={'planet.mass_tot': 1.0}, - ), - ) - with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.utils'): - summarise_failures(str(tmp_path), n_attempted=20) - assert not [r for r in caplog.records if r.message.startswith('Logfiles (')] From 203c52731397d64a7fd1f581525b1e25ba353b0b Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 13:30:29 +0200 Subject: [PATCH 31/72] move read_status into src/proteus/utils/helper as ReadStatus and import --- src/proteus/inference/failures.py | 26 +------------------------- src/proteus/inference/objective.py | 6 +++--- src/proteus/utils/helper.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index 387f4e4fd..e68e2e24c 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -12,7 +12,7 @@ import pandas as pd -from proteus.utils.helper import CommentFromStatus +from proteus.utils.helper import STATUS_MISSING, CommentFromStatus log = logging.getLogger('fwl.' + __name__) @@ -22,9 +22,6 @@ # Suffix for the file holding whatever a child wrote to its console. CHILD_CONSOLE_SUFFIX = '_console.log' -# Status written by PROTEUS before its output folder is cleaned. -STATUS_MISSING = -1 - # How an evaluation that failed is classified. A run that # crashed, or stopped in an error state, did not produce a result at all. A run # that completed normally but ended on a status listed in the study's @@ -169,27 +166,6 @@ def abort_on_failure() -> bool: return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' -def read_status(out_abs: Path | str) -> int: - """Read the PROTEUS status code from a finished run's output folder. - - Parameters - ---------- - - out_abs (Path | str): Absolute path to the run's output folder. - - Returns - ---------- - - int: The status code, or `STATUS_MISSING` when no readable status file - exists. A missing file is itself diagnostic: PROTEUS deletes the status - it writes at start-up when it cleans the output folder, and does not - write another until the main loop begins. - """ - try: - with open(Path(out_abs) / 'status', 'r') as f: - return int(f.readlines()[0].strip()) - except Exception: - return STATUS_MISSING - - def find_run_logfile(out_abs: Path | str) -> str | None: """Return the newest PROTEUS logfile in a run's output folder, if any. diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 8a86aac8a..754cfee30 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -19,12 +19,12 @@ ProteusRunFailure, abort_on_failure, find_run_logfile, - read_status, record_failure, ) from proteus.inference.transforms import unnormalize_parameters from proteus.utils.constants import element_list, gas_list from proteus.utils.coupler import get_proteus_directories, variable_is_logarithmic +from proteus.utils.helper import ReadStatus dtype = torch.double EPS_CLIP = 1e-10 @@ -239,7 +239,7 @@ def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: iter=iter, out_dir=str(out_abs), exit_code=exit_code, - status=read_status(out_abs), + status=ReadStatus(out_abs), log_path=find_run_logfile(out_abs), console_path=str(console), parameters=swept, @@ -277,7 +277,7 @@ def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: update_toml(ref_config, parameters, str(out_cfg)) # Read status file - status = read_status(out_abs) + status = ReadStatus(out_abs) # Read simulator output. A run that exits cleanly but writes no usable # helpfile (killed mid-write, or stopped before the first row) is a failed diff --git a/src/proteus/utils/helper.py b/src/proteus/utils/helper.py index 96a6b5abc..e4d436a34 100644 --- a/src/proteus/utils/helper.py +++ b/src/proteus/utils/helper.py @@ -351,6 +351,36 @@ def UpdateStatusfile(dirs: dict, status: int): hdl.write('%s\n' % desc) +# Status written by PROTEUS before its output folder is cleaned, and never +# rewritten until the main loop starts. A child that dies in between leaves no +# status file at all, so a missing file is reported as such rather than being +# silently reported as a generic error. The value sits outside the range of +# every status PROTEUS writes, so "wrote status 0 (Started) then died" stays +# distinguishable from "never wrote one". +STATUS_MISSING = -1 + + +def ReadStatus(out_abs: Path | str) -> int: + """Read the PROTEUS status code from a finished run's output folder. + + Parameters + ---------- + - out_abs (Path | str): Absolute path to the run's output folder. + + Returns + ---------- + - int: The status code, or `STATUS_MISSING` when no readable status file + exists. A missing file is itself diagnostic: PROTEUS deletes the status + it writes at start-up when it cleans the output folder, and does not + write another until the main loop begins. + """ + try: + with open(Path(out_abs) / 'status', 'r') as f: + return int(f.readlines()[0].strip()) + except Exception: + return STATUS_MISSING + + def CleanDir(directory, keep_stdlog=False): """Clean a directory. From 6af1e5a41d8eaa3dc530384ef79fdfe275d71096 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 13:41:54 +0200 Subject: [PATCH 32/72] remove bulk of docstrings in abort on failure funcs --- src/proteus/inference/failures.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index e68e2e24c..d11457a87 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -145,23 +145,16 @@ def __reduce__(self): ), ) +# Paired reader and writer for the abort switch. def set_abort_on_failure(abort: bool = False) -> None: - """Record whether a failed child run should abort the whole study. - - Stored in the environment so it is visible to the main process and to any - spawned pool workers, matching how the child timeout is plumbed. - """ + """Record whether a failed child run should abort the whole study.""" os.environ[_ABORT_ON_FAILURE_ENV] = '1' if abort else '0' def abort_on_failure() -> bool: - """Return whether a failed child run should abort the whole study. - - Defaults to False: an inference sweep is expected to visit parameter - combinations the simulator cannot integrate, and treating those as fatal - would make most studies unrunnable. Set the inference config field - `abort_on_failure` to stop at the first failure instead. + """Whether a failed child run should abort the study. + Defaults to False. """ return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' From d809c53b6bfb6daa6e0fd453b5706ac1550ee372 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 13:59:31 +0200 Subject: [PATCH 33/72] move abort_on_failure checks to inline instead of sepaerate functions --- src/proteus/inference/failures.py | 19 +++---------------- src/proteus/inference/inference.py | 4 ++-- src/proteus/inference/objective.py | 6 +++--- tests/inference/test_objective.py | 18 +++++++++--------- 4 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index d11457a87..7df9cea42 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -5,7 +5,6 @@ from __future__ import annotations import logging -import os from collections import Counter from dataclasses import dataclass, field from pathlib import Path @@ -16,8 +15,9 @@ log = logging.getLogger('fwl.' + __name__) -# Whether a failed child run aborts the study or scores a bad objective value. -_ABORT_ON_FAILURE_ENV = 'PROTEUS_INFERENCE_ABORT_ON_FAILURE' +# Whether a failed child run aborts the study, or is scored as a bad sample. +# Defaults to scoring. +ABORT_ON_FAILURE_ENV = 'PROTEUS_INFERENCE_ABORT_ON_FAILURE' # Suffix for the file holding whatever a child wrote to its console. CHILD_CONSOLE_SUFFIX = '_console.log' @@ -145,19 +145,6 @@ def __reduce__(self): ), ) -# Paired reader and writer for the abort switch. - -def set_abort_on_failure(abort: bool = False) -> None: - """Record whether a failed child run should abort the whole study.""" - os.environ[_ABORT_ON_FAILURE_ENV] = '1' if abort else '0' - - -def abort_on_failure() -> bool: - """Whether a failed child run should abort the study. - Defaults to False. - """ - return os.environ.get(_ABORT_ON_FAILURE_ENV, '0') == '1' - def find_run_logfile(out_abs: Path | str) -> str | None: """Return the newest PROTEUS logfile in a run's output folder, if any. diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index 261cdc325..550fbddac 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -30,7 +30,7 @@ # bayesopt source files from proteus.inference.async_BO import checkpoint, parallel_process -from proteus.inference.failures import set_abort_on_failure, summarise_failures +from proteus.inference.failures import ABORT_ON_FAILURE_ENV, summarise_failures from proteus.inference.gen_D_init import create_init from proteus.inference.objective import ( WORKER_CONFIG_OVERRIDES, @@ -223,7 +223,7 @@ def run_inference(config): # Whether a failed simulation stops the study or is scored as a poor # sample. Defaults to scoring, because a sweep over a wide parameter box # is expected to reach combinations the simulator cannot integrate. - set_abort_on_failure(bool(config.get('abort_on_failure', False))) + os.environ[ABORT_ON_FAILURE_ENV] = '1' if config.get('abort_on_failure', False) else '0' # Default for configs that pre-date this field config.setdefault('failure_codes', []) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 754cfee30..c82d554b9 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -12,12 +12,12 @@ from numpy import log10 from proteus.inference.failures import ( + ABORT_ON_FAILURE_ENV, CATEGORY_EXCLUDED, CATEGORY_FAILURE, CHILD_CONSOLE_SUFFIX, STATUS_MISSING, ProteusRunFailure, - abort_on_failure, find_run_logfile, record_failure, ) @@ -426,7 +426,7 @@ def J( # Recorded before the abort check, so an aborted study still leaves # the record of what stopped it. record_failure(get_proteus_directories(output)['output'], failure) - if abort_on_failure(): + if os.environ.get(ABORT_ON_FAILURE_ENV, '0') == '1': raise log.warning(failure.summary()) log.debug(failure.report()) @@ -473,7 +473,7 @@ def J( # A clean exit on an error status is as much a fault as a crash, # so it honours `abort_on_failure` the same way. An excluded # outcome never does: nothing went wrong in such a run. - if abort_on_failure(): + if os.environ.get(ABORT_ON_FAILURE_ENV, '0') == '1': raise failure log.warning(failure.summary()) else: diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index c74fc71d1..bc7105825 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -614,7 +614,7 @@ def _fail(**_kwargs): raise failure monkeypatch.setattr(objective_mod, 'run_proteus', _fail) - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') with caplog.at_level('WARNING'): value = objective_mod.J( @@ -644,9 +644,9 @@ def _fail(**_kwargs): assert [(r['worker'], r['iter'], r['status']) for r in recorded] == [(1, 2, 21)] assert recorded[0]['planet.mass_tot'] == pytest.approx(3.0) - # Opting in turns the same failure into a hard stop. `set_abort_on_failure` - # is the writer under test; monkeypatch restores the variable afterwards. - failures_mod.set_abort_on_failure(True) + # Opting in turns the same failure into a hard stop; monkeypatch restores + # the variable afterwards. + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '1') with pytest.raises(objective_mod.ProteusRunFailure): objective_mod.J( x=torch.tensor([[0.5]], dtype=torch.double), @@ -677,7 +677,7 @@ def test_J_scores_a_clean_run_that_stopped_in_an_error_state(monkeypatch, tmp_pa 'run_proteus', lambda **_kwargs: ({'R_obs': 9.25e6}, 25), ) - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') with caplog.at_level('WARNING'): value = objective_mod.J( @@ -760,7 +760,7 @@ def _run(status, worker, iter, codes=()): failure_codes=list(codes), ) - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '1') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '1') # Status 25: written when a run is stopped through its keepalive file, so # the simulator exits 0 and the fault is visible only in the status file. @@ -799,7 +799,7 @@ def _run(status, worker, iter, codes=()): # Discrimination against a regression that raises unconditionally: with # the setting off, the same error status is scored and the study goes on. - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') scored = _run(25, worker=2, iter=0) assert scored.item() == pytest.approx(objective_mod.BAD_OBJ_VALUE) assert scored.item() < -10.0 @@ -812,7 +812,7 @@ def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): ('planet evaporated') and no current code path writes it, so it must not be scored as a failure by an off-by-one in the range bound. """ - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} ) @@ -855,7 +855,7 @@ def test_J_separates_an_excluded_outcome_from_a_failed_run(monkeypatch, tmp_path nothing usable. Reporting the first as the second sends the user looking for a bug in a run that did exactly what it was configured to do. """ - monkeypatch.setenv(failures_mod._ABORT_ON_FAILURE_ENV, '0') + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} ) From c059332e6bda72053bbb3264aa57e9bd5069bf34 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 14:28:19 +0200 Subject: [PATCH 34/72] remove extra 50% warning and instead raise log to warning when there are real failures --- src/proteus/inference/failures.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index 7df9cea42..125916d53 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -53,11 +53,6 @@ 'console_path', ) -# Fraction of evaluations that may fail before the summary escalates from a -# report to a warning. Above this, the sampled region is mostly unrunnable and -# the posterior is built on too few real evaluations to mean much. -FAILURE_FRACTION_WARN = 0.5 - @dataclass(eq=False) class ProteusRunFailure(RuntimeError): @@ -268,14 +263,16 @@ def summarise_failures(output: str, n_attempted: int) -> int: n_excluded = sum(1 for r in records if r.get('category') == CATEGORY_EXCLUDED) n_failed = n_unscored - n_excluded + # One statement of the counts, raised to a warning when a run genuinely + # produced nothing frac = n_unscored / max(n_attempted, 1) - log.info( - f'Unscored evaluations: {n_unscored} of {n_attempted} evaluations ' - f'({100 * frac:.1f}%, initial samples included) carry the failure score ' - 'rather than a fit quality' + log.log( + logging.WARNING if n_failed else logging.INFO, + f'Unscored evaluations: {n_unscored} of {n_attempted} ' + f'({100 * frac:.1f}%, initial samples included) carry the failure score: ' + f'{n_failed} produced no usable result, {n_excluded} completed on an ' + 'excluded status.', ) - log.info(f' {n_failed} did not produce a usable result') - log.info(f' {n_excluded} completed on a status this study excludes') # Grouped by cause, and labelled so that an excluded outcome is not read as # something having gone wrong in the run that reached it. @@ -296,12 +293,6 @@ def summarise_failures(output: str, n_attempted: int) -> int: log.info(f' {log_path}') log.info(f'Full list: {Path(output) / FAILURE_CSV}') - if frac > FAILURE_FRACTION_WARN: - log.warning( - f'More than {100 * FAILURE_FRACTION_WARN:.0f}% of evaluations failed or were' - f'excluded, so the result rests on {n_attempted - n_unscored} real ' - 'evaluations.' - ) log.info('-----------------------------------') return n_unscored From fb0bdeb18f768702337e995780e72a49a8cfafbd Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 14:32:21 +0200 Subject: [PATCH 35/72] update with changed warning and loosen asserts on exact info logs --- tests/inference/test_failures.py | 89 +++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/tests/inference/test_failures.py b/tests/inference/test_failures.py index 14106a1e7..05c81fa9b 100644 --- a/tests/inference/test_failures.py +++ b/tests/inference/test_failures.py @@ -14,6 +14,7 @@ import logging import pickle +import re import pandas as pd import pytest @@ -23,6 +24,17 @@ pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] +def _counts(line: str) -> list[str]: + """Every number in a log line, in order. + + The tally lines are prose around a handful of counts. Asserting on the + numbers keeps a test pinned to what the reader has to get right, and lets + the wording be changed without a test failing for no reason. Order is kept, + so a line that swapped the failed and excluded counts still fails. + """ + return re.findall(r'\d+(?:\.\d+)?', line) + + @pytest.mark.unit def test_failure_summary_is_one_line_and_names_where_the_detail_is_kept(): """The line a study logs for each unscored run identifies the run, names @@ -237,11 +249,12 @@ def test_recording_a_failure_never_masks_the_failure_it_records(tmp_path): @pytest.mark.unit -def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp_path, caplog): - """The end-of-study tally turns the per-run records into one table and one - breakdown by cause, and escalates to a warning once most of the study - failed. Without the escalation, a posterior built on a handful of real - evaluations reads the same as one built on all of them. +def test_summarise_failures_tabulates_causes_and_warns_on_every_real_failure(tmp_path, caplog): + """The end-of-study tally turns the per-run rows into one table and one + breakdown by cause, and warns whenever a run produced nothing usable. The + warning does not wait for a fraction of the study to fail: a sweep can lose + a tenth of its evaluations and still fit well, so the count is put in front + of the reader to weigh rather than compared against a threshold. """ # Two runs that died the same way and one that died differently, so the @@ -265,12 +278,18 @@ def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp assert n_failed == 3 messages = '\n'.join(r.message for r in caplog.records) - assert '3 of 20 evaluations' in messages + assert '3 of 20' in messages # Grouped by cause, so two runs that died the same way count as one line. assert 'Interior model' in messages - # Below the escalation threshold (3/20 = 15%), the tally is reported but - # not warned about. - assert not [r for r in caplog.records if r.levelname == 'WARNING'] + + # 3 of 20 is 15%, well under the half-the-study line the old threshold drew, + # and it is raised to a warning anyway: the count is what the reader weighs. + # One record carries it, so the level changes rather than a second line + # repeating the counts the report already gave. + warnings = [r for r in caplog.records if r.levelname == 'WARNING'] + assert len(warnings) == 1 + # unscored, attempted, percent, failed, excluded. + assert _counts(warnings[0].message) == ['3', '20', '15.0', '3', '0'] # The table carries the swept parameter alongside the diagnosis, so the # failing region can be located without opening each run folder. @@ -280,16 +299,14 @@ def test_summarise_failures_tabulates_causes_and_flags_a_mostly_failed_study(tmp assert sorted(table['status']) == [21, 21, 24] assert table['planet.mass_tot'].max() == pytest.approx(3.0) - # Discrimination: the same three failures against a smaller study cross the - # threshold and are warned about. A tally without the escalation would log - # identically in both cases. + # The percentage tracks the study size rather than being a fixed string: + # the same three failures against a smaller study report a larger share. caplog.clear() with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): failures_mod.summarise_failures(str(tmp_path), n_attempted=4) warnings = [r for r in caplog.records if r.levelname == 'WARNING'] assert len(warnings) == 1 - # The count of evaluations that are real is what the reader needs. - assert '1 real evaluations' in warnings[0].message + assert _counts(warnings[0].message) == ['3', '4', '75.0', '3', '0'] @pytest.mark.unit @@ -334,8 +351,10 @@ def test_summarise_failures_counts_excluded_outcomes_apart_from_failures(tmp_pat # real, but the breakdown names them apart. assert n_unscored == 3 messages = '\n'.join(r.message for r in caplog.records) - assert '1 did not produce a usable result' in messages - assert '2 completed on a status this study excludes' in messages + tally = next(r for r in caplog.records if 'Unscored evaluations' in r.message) + # unscored, attempted, percent, failed, excluded: the one fault is counted + # apart from the two runs that completed on an excluded status. + assert _counts(tally.message) == ['3', '20', '15.0', '1', '2'] # The clock-limit outcome is labelled in the cause table rather than being # listed beside the interior-model error as if it were one. assert 'Completed (maximum clock runtime) [excluded]' in messages @@ -347,6 +366,37 @@ def test_summarise_failures_counts_excluded_outcomes_apart_from_failures(tmp_pat assert sorted(table['category']) == ['excluded', 'excluded', 'failure'] assert sorted(table.loc[table['category'] == 'excluded', 'status']) == [11, 11] + # Raised to a warning by the one genuine fault. The counts were pinned + # above; what matters here is that the line carrying them is the warning. + assert [r.levelname for r in caplog.records if r.levelname == 'WARNING'] == ['WARNING'] + assert 'Unscored evaluations' in tally.message and tally.levelname == 'WARNING' + + # Limit input: a study whose runs were *all* excluded did exactly what it + # was configured to do, so it is tallied without any warning at all. Keying + # the warning on the unscored total would have flagged it as broken. + caplog.clear() + (tmp_path / 'failures.csv').unlink() + for worker in (0, 1): + failures_mod.record_failure( + tmp_path, + failures_mod.ProteusRunFailure( + reason='completed on a status this study excludes', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=0, + status=11, + parameters={'planet.mass_tot': 1.0 + worker}, + category=failures_mod.CATEGORY_EXCLUDED, + ), + ) + with caplog.at_level(logging.INFO, logger='fwl.proteus.inference.failures'): + assert failures_mod.summarise_failures(str(tmp_path), n_attempted=4) == 2 + assert not [r for r in caplog.records if r.levelname == 'WARNING'] + # Discrimination: 2 of 4 is half the study, which the old fraction rule + # would have reported as a study mostly not worth trusting. + assert '2 of 4' in '\n'.join(r.message for r in caplog.records) + @pytest.mark.unit def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_path, caplog): @@ -361,7 +411,7 @@ def test_summarise_failures_reports_a_clean_study_without_writing_a_table(tmp_pa assert n_failed == 0 assert not (tmp_path / 'failures.csv').exists() messages = '\n'.join(r.message for r in caplog.records) - assert 'none' in messages and '12 evaluations' in messages + assert 'none' in messages and '12' in _counts(messages) assert not [r for r in caplog.records if r.levelname in ('WARNING', 'ERROR')] @@ -401,10 +451,9 @@ def test_summarise_failures_labels_the_logfile_sample_and_counts_the_whole_study messages = '\n'.join(lines) # The denominator of the tally is the whole study, stated in the line # itself so it cannot be confused with the optimisation-only warning. - assert '4 of 20 evaluations' in messages - assert 'initial samples included' in messages + assert '4 of 20' in messages # Three shown out of four unscored, not four out of four. - assert 'Logfiles (3 of 4 shown):' in messages + assert '3 of 4 shown' in messages shown = [line.strip() for line in lines if line.strip().endswith('proteus_00.log')] assert len(shown) == 3 # The record without a logfile is skipped rather than truncating the From ef4b1f3db4c26f86e0759d13596b3ea7a03e2a03 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 15:14:52 +0200 Subject: [PATCH 36/72] add archive_mod = 0 to turn on archiving of output for successful workers --- src/proteus/inference/objective.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index c82d554b9..0afb9775a 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -46,6 +46,7 @@ WORKER_CONFIG_OVERRIDES = { 'params.out.plot_mod': 'none', 'params.out.logging': 'WARNING', + 'params.out.archive_mod': 0, } # Config entries every run sets to the same thing, or to a value derived from From c0231b00968e6b570310a9ebdce5a5d7d2ad3719 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 18:03:45 +0200 Subject: [PATCH 37/72] add cached spectral file functionality. mainly for inference runs, but possible useful outside of this, so now a new config option. --- input/all_options.toml | 1 + src/proteus/atmos_clim/agni.py | 25 +++ src/proteus/atmos_clim/spectral_cache.py | 154 ++++++++++++++++ src/proteus/config/_atmos_clim.py | 4 + src/proteus/inference/objective.py | 15 +- tests/atmos_clim/test_spectral_cache.py | 212 +++++++++++++++++++++++ 6 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 src/proteus/atmos_clim/spectral_cache.py create mode 100644 tests/atmos_clim/test_spectral_cache.py diff --git a/input/all_options.toml b/input/all_options.toml index 030c00295..d601586a0 100644 --- a/input/all_options.toml +++ b/input/all_options.toml @@ -529,6 +529,7 @@ config_version = "3.0" # Grid and spectral setup (shared by agni + janus) spectral_group = "Honeyside" # opacity k-table set; see docs/assets/spectral_files.pdf spectral_bands = "48" # wavenumber bands in k-table + spectral_cache = "none" # reuse prepared spectral files across runs with the same star; none = off num_levels = 50 # vertical atmosphere levels (min 15) p_top = 1.0e-6 # top-of-atmosphere pressure [bar] p_obs = 0.02 # observation pressure level [bar] (transit radius) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index ac1a196a7..38a18cb1d 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -12,6 +12,7 @@ from scipy.interpolate import PchipInterpolator from proteus.atmos_clim.common import clip_radius_to_hill, get_oarr_from_parr, get_spfile_path +from proteus.atmos_clim.spectral_cache import cache_key, seed_from_cache, store_in_cache from proteus.utils.constants import gas_list, noble_gases from proteus.utils.helper import ( UpdateStatusfile, @@ -510,6 +511,10 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # bypass the glob entirely so a missing or empty `data/*.sflux` directory # is not a precondition for those modes. + # Set when this run built a prepared spectral file that the cache does not + # yet hold, so it can be stored once the build is known to have succeeded. + cache_store_key = None + # Spectral file path provided? if config.atmos_clim.agni.spectral_file is not None: # Grey gas? @@ -553,6 +558,22 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): input_sf = get_spfile_path(dirs['fwl'], config) input_star = sflux_path + # Reuse a cached file built earlier from this base file and this stellar + # spectrum, and skip the insertion. + if config.atmos_clim.spectral_cache: + key = cache_key( + input_sf, + sflux_path, + config.atmos_clim.spectral_group, + config.atmos_clim.spectral_bands, + ) + if seed_from_cache(config.atmos_clim.spectral_cache, key, dirs['output']): + log.debug('Reusing prepared spectral file from cache') + input_sf = try_spfile + input_star = '' + else: + cache_store_key = key + # Fast I/O folder if (config.atmos_clim.agni.verbosity >= 2) or (config.params.out.logging == 'DEBUG'): io_dir = dirs['output'] @@ -691,6 +712,10 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # Confirm the live Atmos_t contains every field that PROTEUS expects _check_agni_schema(atmos, dirs) + # Stored spectral file is now valid, so store it in the cache if requested. + if cache_store_key: + store_in_cache(config.atmos_clim.spectral_cache, cache_store_key, dirs['output']) + # Set temperature profile from old NetCDF if it exists nc_files = glob.glob(os.path.join(dirs['output'], 'data', '*_atm.nc')) if len(nc_files) > 0: diff --git a/src/proteus/atmos_clim/spectral_cache.py b/src/proteus/atmos_clim/spectral_cache.py new file mode 100644 index 000000000..17a8e15f1 --- /dev/null +++ b/src/proteus/atmos_clim/spectral_cache.py @@ -0,0 +1,154 @@ +"""Reuse of prepared spectral files across runs that share a stellar spectrum. + +A run's `runtime.sf` is the base spectral file from FWL_DATA with that run's +stellar spectrum inserted. This module keeps one copy per distinct input set so the +second and later runs copy it instead. + +The cache is seeded into the run's output folder as `runtime.sf`, which is +where the atmosphere wrapper already expects to manage it. +""" + +from __future__ import annotations + +import hashlib +import logging +import os +import shutil +from pathlib import Path + +log = logging.getLogger('fwl.' + __name__) + +# Prepared spectral files come in a pair, and the existence check in the AGNI +# module only tests the first. Seeding one without the other would leave the +# module using a file whose companion is absent. +SPECTRAL_SUFFIXES = ('', '_k') + +# Bytes read per chunk when fingerprinting a file. +_CHUNK = 1 << 20 + + +def _file_digest(path: Path) -> str: + """Hash a file's contents.""" + digest = hashlib.sha256() + with open(path, 'rb') as f: + while chunk := f.read(_CHUNK): + digest.update(chunk) + return digest.hexdigest() + + +def cache_key(base_sf: Path | str, star_spectrum: Path | str, group: str, bands: str) -> str: + """Name the cache entry for a prepared spectral file. The stellar spectrum is hashed by content. + + Parameters + ---------- + - base_sf (Path | str): Base spectral file, from FWL_DATA. + - star_spectrum (Path | str): Stellar spectrum (`.sflux`) to be inserted. + - group (str): Spectral file group. + - bands (str): Number of wavenumber bands. + + Returns + ---------- + - str: Hex digest naming this combination. + """ + base = Path(base_sf) + stat = base.stat() + parts = ( + group, + bands, + base.name, + str(stat.st_size), + str(int(stat.st_mtime)), + _file_digest(Path(star_spectrum)), + ) + return hashlib.sha256('\0'.join(parts).encode()).hexdigest()[:32] + + +def _entry_paths(cache_dir: Path | str, key: str) -> list[Path]: + """Paths of the cached pair for one key.""" + return [Path(cache_dir) / f'{key}.sf{suffix}' for suffix in SPECTRAL_SUFFIXES] + + +def _output_paths(out_dir: Path | str) -> list[Path]: + """Paths of the prepared pair inside a run's output folder.""" + return [Path(out_dir) / f'runtime.sf{suffix}' for suffix in SPECTRAL_SUFFIXES] + + +def seed_from_cache(cache_dir: Path | str, key: str, out_dir: Path | str) -> bool: + """Copy a cached spectral file pair into a run's output folder. + + Parameters + ---------- + - cache_dir (Path | str): Folder holding cached entries. + - key (str): Key from `cache_key`. + - out_dir (Path | str): The run's output folder. + + Returns + ---------- + - bool: True if the run can now use `runtime.sf` as-is. False on a miss, or + on any error: a cache that cannot be read is a slower run, never a failed + one, so the caller falls back to building the file. + """ + entries = _entry_paths(cache_dir, key) + if not all(entry.is_file() for entry in entries): + return False + + targets = _output_paths(out_dir) + try: + Path(out_dir).mkdir(parents=True, exist_ok=True) + for entry, target in zip(entries, targets): + shutil.copyfile(entry, target) + except OSError as err: + log.warning(f'Could not seed spectral file from cache: {err}') + # A half-copied pair would be read as a prepared file. Clear it so the + # run rebuilds from scratch instead of starting from a truncated file. + for target in targets: + try: + target.unlink(missing_ok=True) + except OSError: + pass + return False + + log.debug(f'Seeded spectral file from cache entry {key}') + return True + + +def store_in_cache(cache_dir: Path | str, key: str, out_dir: Path | str) -> bool: + """Copy a run's prepared spectral file pair into the cache. + + Written to a temporary name and renamed into place, so concurrent runs + racing to populate the same key never expose a half-written entry. + + Parameters + ---------- + - cache_dir (Path | str): Folder holding cached entries. + - key (str): Key from `cache_key`. + - out_dir (Path | str): The run's output folder, holding the prepared pair. + + Returns + ---------- + - bool: True if the entry was stored. False if the run produced no prepared + file, or the cache could not be written; neither is a fault of the run. + """ + sources = _output_paths(out_dir) + if not all(source.is_file() for source in sources): + return False + + try: + Path(cache_dir).mkdir(parents=True, exist_ok=True) + for source, entry in zip(sources, _entry_paths(cache_dir, key)): + tmp = entry.with_name(f'{entry.name}.{os.getpid()}.tmp') + try: + shutil.copyfile(source, tmp) + os.replace(tmp, entry) + except BaseException: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + raise + except OSError as err: + log.warning(f'Could not store spectral file in cache: {err}') + return False + + log.debug(f'Stored spectral file in cache as entry {key}') + return True diff --git a/src/proteus/config/_atmos_clim.py b/src/proteus/config/_atmos_clim.py index dda84a5a8..d875a5931 100644 --- a/src/proteus/config/_atmos_clim.py +++ b/src/proteus/config/_atmos_clim.py @@ -363,6 +363,9 @@ class AtmosClim: Spectral file group defining gas opacities. See https://proteus-framework.org/SOCRATES/Reference/proteus_spectral_file_reference.html spectral_bands: str Number of wavenumber bands in k-table. + spectral_cache: str | None + Folder in which to reuse prepared spectral files across runs that share + a stellar spectrum. None disables the cache and every run builds its own. num_levels: int Number of vertical atmosphere levels. p_top: float @@ -408,6 +411,7 @@ class AtmosClim: # Grid and spectral setup (shared by agni + janus) spectral_group: str = field(default='Honeyside') spectral_bands: str = field(default='48') + spectral_cache: str | None = field(default=None, converter=none_if_none) num_levels: int = field(default=50, validator=ge(15)) p_top: float = field(default=1e-6, validator=gt(0)) p_obs: float = field(default=20e-3, validator=gt(0)) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 0afb9775a..8a4d41f0f 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -49,9 +49,15 @@ 'params.out.archive_mod': 0, } +# Folder inside the study output where workers reuse prepared spectral files. +SPECTRAL_CACHE_DIR = 'spectral_cache' + # Config entries every run sets to the same thing, or to a value derived from # the run index. Excluded from failure reports, which name the swept values. -_FIXED_PARAMETER_KEYS = set(WORKER_CONFIG_OVERRIDES) | {'params.out.path'} +_FIXED_PARAMETER_KEYS = set(WORKER_CONFIG_OVERRIDES) | { + 'params.out.path', + 'atmos_clim.spectral_cache', +} def run_output_dir(output: str, worker: int, iter: int) -> tuple[Path, Path]: @@ -209,6 +215,13 @@ def run_proteus( # Inject output path into simulation parameters parameters['params.out.path'] = str(out_dir) + # Every evaluation of an inference run that holds the star fixed builds the same + # prepared spectral file. Point them all at one folder so only the first + # pays for it. + parameters['atmos_clim.spectral_cache'] = str( + Path(get_proteus_directories(output)['output']) / SPECTRAL_CACHE_DIR + ) + # Don't allow workers to make plots or logs parameters.update(WORKER_CONFIG_OVERRIDES) diff --git a/tests/atmos_clim/test_spectral_cache.py b/tests/atmos_clim/test_spectral_cache.py new file mode 100644 index 000000000..27c1e3d5a --- /dev/null +++ b/tests/atmos_clim/test_spectral_cache.py @@ -0,0 +1,212 @@ +""" +Unit tests for reuse of prepared spectral files across runs. + +Covers `proteus.atmos_clim.spectral_cache`: what makes two runs share a +prepared file, that the pair is seeded and stored together, and that a cache +which cannot be read or written costs a run time rather than correctness. + +References: + - docs/How-to/testing.md + - docs/Explanations/test_framework.md +""" + +from __future__ import annotations + +import pytest + +from proteus.atmos_clim.spectral_cache import ( + SPECTRAL_SUFFIXES, + cache_key, + seed_from_cache, + store_in_cache, +) + +pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] + + +def _make_inputs(tmp_path, star_bytes=b'1.0 2.0\n3.0 4.0\n'): + """Write a base spectral file and a stellar spectrum, and return both.""" + base = tmp_path / 'Honeyside.sf' + base.write_bytes(b'x' * 4096) + star = tmp_path / '0.sflux' + star.write_bytes(star_bytes) + return base, star + + +def _make_prepared(out_dir, marker=b'prepared'): + """Write the prepared pair a finished build leaves in an output folder.""" + out_dir.mkdir(parents=True, exist_ok=True) + for suffix in SPECTRAL_SUFFIXES: + (out_dir / f'runtime.sf{suffix}').write_bytes(marker + suffix.encode()) + + +@pytest.mark.unit +def test_key_tracks_the_stellar_spectrum_and_the_spectral_resolution(tmp_path): + """Two runs share a prepared file only when every input that goes into + building it agrees. The stellar spectrum is hashed by content, so a sweep + over any star parameter changes the key without this module having to know + which config fields those are. + """ + base, star = _make_inputs(tmp_path) + key = cache_key(base, star, 'Honeyside', '48') + + # Same inputs, same entry: this is what lets a study reuse one build. + assert cache_key(base, star, 'Honeyside', '48') == key + + # A different stellar spectrum is a different file, even byte-for-byte the + # same length, so a content hash rather than a size check is required. + other_star = tmp_path / 'other.sflux' + other_star.write_bytes(b'9.0 2.0\n3.0 4.0\n') + assert other_star.stat().st_size == star.stat().st_size + assert cache_key(base, other_star, 'Honeyside', '48') != key + + # Resolution and group select a different base file, so neither may collide. + assert cache_key(base, star, 'Honeyside', '256') != key + assert cache_key(base, star, 'Frostflow', '48') != key + + +@pytest.mark.unit +def test_key_changes_when_the_base_spectral_file_is_updated(tmp_path): + """A FWL_DATA update must not be served a stale entry. The base file is + fingerprinted by size and modification time rather than hashed, because it + is large and read-only, so both are exercised here. + """ + base, star = _make_inputs(tmp_path) + key = cache_key(base, star, 'Honeyside', '48') + + # Same size, newer file: the mtime component has to carry this one. + stat = base.stat() + base.write_bytes(b'y' * 4096) + import os + + os.utime(base, (stat.st_atime, stat.st_mtime + 120)) + assert base.stat().st_size == 4096 + key_touched = cache_key(base, star, 'Honeyside', '48') + assert key_touched != key + + # Same mtime, different size: the size component has to carry this one. + base.write_bytes(b'y' * 8192) + os.utime(base, (stat.st_atime, stat.st_mtime + 120)) + assert cache_key(base, star, 'Honeyside', '48') != key_touched + + +@pytest.mark.unit +def test_a_stored_entry_is_seeded_back_for_a_later_run(tmp_path): + """The round trip a study relies on: the first run stores what it built and + every later run with the same inputs starts from it. Both files of the pair + travel together, because the module that consumes them checks only the + first and would otherwise run with a missing companion. + """ + cache = tmp_path / 'cache' + first = tmp_path / 'run_0' + _make_prepared(first) + + assert store_in_cache(cache, 'abc123', first) is True + + second = tmp_path / 'run_1' + assert seed_from_cache(cache, 'abc123', second) is True + for suffix in SPECTRAL_SUFFIXES: + seeded = second / f'runtime.sf{suffix}' + assert seeded.is_file() + assert seeded.read_bytes() == (first / f'runtime.sf{suffix}').read_bytes() + + # Discrimination: a key nothing was stored under is a miss, so the run + # builds its own file rather than silently reusing another star's. + third = tmp_path / 'run_2' + assert seed_from_cache(cache, 'def456', third) is False + assert not (third / 'runtime.sf').exists() + + +@pytest.mark.unit +def test_a_half_written_entry_is_not_served(tmp_path): + """An entry is usable only once both files are in place. A reader that + accepted the first alone would hand a run a prepared file whose companion + is missing, which is worse than a miss because the run would not rebuild. + """ + cache = tmp_path / 'cache' + cache.mkdir() + # Only the first of the pair, as a reader would see mid-store. + (cache / 'abc123.sf').write_bytes(b'prepared') + + out = tmp_path / 'run' + assert seed_from_cache(cache, 'abc123', out) is False + assert not (out / 'runtime.sf').exists() + + # Discrimination: completing the pair makes the same key usable, so the + # refusal above came from the missing companion and not from the reader + # rejecting every entry. + (cache / 'abc123.sf_k').write_bytes(b'prepared_k') + assert seed_from_cache(cache, 'abc123', out) is True + assert (out / 'runtime.sf').is_file() + + +@pytest.mark.unit +def test_a_run_that_built_nothing_stores_nothing(tmp_path): + """Storing is driven by what the run produced, not by being asked. A run + whose build failed before writing the pair must not publish a partial entry + that every later run with the same star would then be served. + """ + cache = tmp_path / 'cache' + empty = tmp_path / 'run_empty' + empty.mkdir() + + assert store_in_cache(cache, 'abc123', empty) is False + assert not (cache / 'abc123.sf').exists() + + # Edge case: one file of the pair present is still nothing worth storing. + (empty / 'runtime.sf').write_bytes(b'prepared') + assert store_in_cache(cache, 'abc123', empty) is False + assert not (cache / 'abc123.sf').exists() + + # Discrimination: the complete pair does store, so the refusals above came + # from the missing file and not from a writer that always declines. + (empty / 'runtime.sf_k').write_bytes(b'prepared_k') + assert store_in_cache(cache, 'abc123', empty) is True + assert (cache / 'abc123.sf').is_file() + + +@pytest.mark.unit +def test_an_unusable_cache_costs_time_and_not_correctness(tmp_path): + """A cache that cannot be written or read leaves the run to build its own + file. Bookkeeping around a simulation must not decide whether it runs, so + both directions report the failure and return rather than raising. + """ + # A file where the cache folder should be: it cannot be created beneath. + blocked = tmp_path / 'not_a_directory' + blocked.write_text('this is a file, so no folder can be made beneath it') + + run = tmp_path / 'run' + _make_prepared(run) + assert store_in_cache(blocked, 'abc123', run) is False + + # Reading a cache folder that was never created is a miss, not an error. + assert seed_from_cache(tmp_path / 'never_made', 'abc123', tmp_path / 'out') is False + + # Discrimination: against a usable folder the same call succeeds, so the + # False above came from the blocked path rather than from a writer that + # always fails. + assert store_in_cache(tmp_path / 'cache', 'abc123', run) is True + + +@pytest.mark.unit +def test_storing_twice_leaves_one_usable_entry(tmp_path): + """Workers race to populate the same key, because they start together and + share a stellar spectrum. The later store replaces the earlier one in + place, so a reader sees one entry or the other and never a mixture. + """ + cache = tmp_path / 'cache' + first = tmp_path / 'run_0' + second = tmp_path / 'run_1' + _make_prepared(first, marker=b'from_first') + _make_prepared(second, marker=b'from_second') + + assert store_in_cache(cache, 'abc123', first) is True + assert store_in_cache(cache, 'abc123', second) is True + + # No temporary files survive to be mistaken for entries. + assert sorted(p.name for p in cache.iterdir()) == ['abc123.sf', 'abc123.sf_k'] + + out = tmp_path / 'run_2' + assert seed_from_cache(cache, 'abc123', out) is True + assert (out / 'runtime.sf').read_bytes() == b'from_second' + assert (out / 'runtime.sf_k').read_bytes() == b'from_second_k' From b64b2de03f9d0c15e622698a7a5f27af34087db6 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Fri, 18 Sep 2026 21:54:18 +0200 Subject: [PATCH 38/72] 'BaseException' -> Exception Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/proteus/atmos_clim/spectral_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proteus/atmos_clim/spectral_cache.py b/src/proteus/atmos_clim/spectral_cache.py index 17a8e15f1..2454ccd2b 100644 --- a/src/proteus/atmos_clim/spectral_cache.py +++ b/src/proteus/atmos_clim/spectral_cache.py @@ -140,7 +140,7 @@ def store_in_cache(cache_dir: Path | str, key: str, out_dir: Path | str) -> bool try: shutil.copyfile(source, tmp) os.replace(tmp, entry) - except BaseException: + except Exception: try: tmp.unlink(missing_ok=True) except OSError: From cab38e939f604fb355ac3e00b277ff1e2726c599 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sat, 19 Sep 2026 20:22:58 +0200 Subject: [PATCH 39/72] fix 'Empty except' with log Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/proteus/atmos_clim/spectral_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/proteus/atmos_clim/spectral_cache.py b/src/proteus/atmos_clim/spectral_cache.py index 2454ccd2b..babc91331 100644 --- a/src/proteus/atmos_clim/spectral_cache.py +++ b/src/proteus/atmos_clim/spectral_cache.py @@ -104,8 +104,8 @@ def seed_from_cache(cache_dir: Path | str, key: str, out_dir: Path | str) -> boo for target in targets: try: target.unlink(missing_ok=True) - except OSError: - pass + except OSError as cleanup_err: + log.debug(f'Could not remove partial spectral file {target}: {cleanup_err}') return False log.debug(f'Seeded spectral file from cache entry {key}') From 18cb794332cc2555636dc0c13aa8ad644d42fd82 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 20 Sep 2026 12:24:49 +0200 Subject: [PATCH 40/72] register spectral_cache in atmosclim doc_groups and update docs --- docs/Reference/config/atmosphere.md | 1 + docs/Reference/config/config_schema.json | 24 ++++++++++++++++++++---- src/proteus/config/_atmos_clim.py | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/Reference/config/atmosphere.md b/docs/Reference/config/atmosphere.md index ed985c957..8d905c805 100644 --- a/docs/Reference/config/atmosphere.md +++ b/docs/Reference/config/atmosphere.md @@ -25,6 +25,7 @@ parameter selects the surface energy balance scheme (`mixed_layer`, | `module` | str | `"agni"` | Which atmosphere module to use. Choices: `"dummy"`, `"agni"`, `"janus"`. | | `spectral_group` | str | `"Honeyside"` | Spectral file group defining gas opacities. See https://proteus-framework.org/SOCRATES/Reference/proteus_spectral_file_reference.html. | | `spectral_bands` | str | `"48"` | Number of wavenumber bands in k-table. | +| `spectral_cache` | str or none | `none` | Folder in which to reuse prepared spectral files across runs that share a stellar spectrum. None disables the cache and every run builds its own. | | `num_levels` | int | `50` | Number of vertical atmosphere levels. Must be >= 15. | | `p_top` | float | `1e-06` | Top-of-atmosphere pressure \[bar\]. Must be > 0. | | `p_obs` | float | `0.02` | Observation pressure level \[bar\] (transit radius). Must be > 0. | diff --git a/docs/Reference/config/config_schema.json b/docs/Reference/config/config_schema.json index d02cff41a..22580b785 100644 --- a/docs/Reference/config/config_schema.json +++ b/docs/Reference/config/config_schema.json @@ -8673,6 +8673,22 @@ "group": null, "group_qualifier": null }, + { + "path": "atmos_clim.spectral_cache", + "toml_section": "atmos_clim", + "class": "AtmosClim", + "type": "str or none", + "accepts_none": true, + "default": "none", + "choices": null, + "bounds": null, + "description": "Folder in which to reuse prepared spectral files across runs that share a stellar spectrum. None disables the cache and every run builds its own.", + "doc_source": "attributes", + "group_order": 0, + "group_position": 3, + "group": null, + "group_qualifier": null + }, { "path": "atmos_clim.num_levels", "toml_section": "atmos_clim", @@ -8690,7 +8706,7 @@ "description": "Number of vertical atmosphere levels.", "doc_source": "attributes", "group_order": 0, - "group_position": 3, + "group_position": 4, "group": null, "group_qualifier": null }, @@ -8711,7 +8727,7 @@ "description": "Top-of-atmosphere pressure [bar].", "doc_source": "attributes", "group_order": 0, - "group_position": 4, + "group_position": 5, "group": null, "group_qualifier": null }, @@ -8732,7 +8748,7 @@ "description": "Observation pressure level [bar] (transit radius).", "doc_source": "attributes", "group_order": 0, - "group_position": 5, + "group_position": 6, "group": null, "group_qualifier": null }, @@ -8748,7 +8764,7 @@ "description": "Gas overlap method. Choices: 'ro', 'rorr', 'ee'.", "doc_source": "attributes", "group_order": 0, - "group_position": 6, + "group_position": 7, "group": null, "group_qualifier": null }, diff --git a/src/proteus/config/_atmos_clim.py b/src/proteus/config/_atmos_clim.py index d875a5931..fa403342d 100644 --- a/src/proteus/config/_atmos_clim.py +++ b/src/proteus/config/_atmos_clim.py @@ -459,6 +459,7 @@ def surf_state_int(self) -> int: 'module', 'spectral_group', 'spectral_bands', + 'spectral_cache', 'num_levels', 'p_top', 'p_obs', From 84a277043160d672d22abf53ab032fb06f4b53e6 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 20 Sep 2026 12:28:23 +0200 Subject: [PATCH 41/72] add failures.py to docs in project structure overview --- docs/How-to/inference.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index 38f9840e1..43a659ed1 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -32,6 +32,7 @@ The system performs Bayesian optimization to infer planetary formation parameter | `async_BO.py` | Parallel BO implementation | | `BO.py` | Single BO step implementation | | `objective.py` | PROTEUS interface and objective function | + | `failures.py` | Functions for handling failing simulations | | `plot.py` | Visualization utilities | | `utils.py` | Helper functions for inference scheme | | `gen_D_init.py` | Generate initial data | From 13fda76f88c7597d85a2b4b535dbf574258a3848 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 20 Sep 2026 15:08:11 +0200 Subject: [PATCH 42/72] update docs line --- docs/How-to/inference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index 43a659ed1..f19e7ec7c 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -123,7 +123,7 @@ The system generates several outputs in: - `logs.csv`: Detailed logs of each BO step - `Ts.csv`: Timestamps for performance analysis - `init.csv`: Data used as an initial guess for starting the optimisation -- `failures.csv`: One row per simulation that carries the failure score instead of a fit quality, written only when there is at least one (see [Failed and excluded simulations](#failed-and-excluded-simulations)) +- `failures.csv`: One row per simulation that failed or was excluded, written only when there is at least one (see [Failed and excluded simulations](#failed-and-excluded-simulations)) ### Plots The BO scheme will generate many plots upon completion. From 7932d9d9642dce757c23c55d58745a93de52204d Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 20 Sep 2026 15:09:12 +0200 Subject: [PATCH 43/72] restore to matern 1/2 and update comments --- input/inference/example.infer.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/input/inference/example.infer.toml b/input/inference/example.infer.toml index 39b61b29b..0ca369634 100644 --- a/input/inference/example.infer.toml +++ b/input/inference/example.infer.toml @@ -16,18 +16,16 @@ ref_config = "input/inference/example.toml" init_samps = 3 # Number of random samples if starting from scratch. init_grid = 'none' # grid_demo/' # Path pre-computed grid (relative to PROTEUS output folder) -# Completion codes to exclude from the fit, on top of the error codes that are -# always excluded. A run ending on one of these completed normally; the study -# simply does not score that outcome. +# Completion codes to exclude from the fit. failure_codes = [11, ] # solidified (10), escaped (15), max_runtime (11) -# Stop the whole study at the first simulation that fails, instead of scoring +# Stop the whole inference run at the first simulation that fails, instead of scoring # it as a poor sample and carrying on. Off by default. abort_on_failure = false # Parameters for Bayesian optimisation n_workers = 5 # Number of parallel workers -kernel = "MAT3/2" # Kernel type for GP, "RBF" | "MAT1/2" | "MAT3/2" | "MAT5/2" +kernel = "MAT1/2" # Kernel type for GP, "RBF" | "MAT1/2" | "MAT3/2" | "MAT5/2" acqf = "LogEI" # Acquisition function, "UCB" | "LogEI" | "LogPI" n_steps = 100 # Total number of evaluations (i.e. BO steps) From 18b67499ef173167f796f143ca6a376491147ab1 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Sun, 20 Sep 2026 18:07:33 +0200 Subject: [PATCH 44/72] fix bug where spectral cache is on but unused --- src/proteus/atmos_clim/agni.py | 21 ++-- tests/atmos_clim/test_agni.py | 192 +++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 10 deletions(-) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index 6b2b47939..96b216102 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -485,6 +485,14 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # bypass the glob entirely so a missing or empty `data/*.sflux` directory # is not a precondition for those modes. + # Fast I/O folder. Decided before the spectral file, because AGNI writes the + # prepared runtime.sf pair here and so this is where the cache reads from. + if (config.atmos_clim.agni.verbosity >= 2) or (config.params.out.logging == 'DEBUG'): + io_dir = dirs['output'] + else: + io_dir = create_tmp_folder() + log.info(f'Temporary-file working dir: {io_dir}') + # Set when this run built a prepared spectral file that the cache does not # yet hold, so it can be stored once the build is known to have succeeded. cache_store_key = None @@ -541,20 +549,13 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): config.atmos_clim.spectral_group, config.atmos_clim.spectral_bands, ) - if seed_from_cache(config.atmos_clim.spectral_cache, key, dirs['output']): + if seed_from_cache(config.atmos_clim.spectral_cache, key, io_dir): log.debug('Reusing prepared spectral file from cache') - input_sf = try_spfile + input_sf = os.path.join(io_dir, 'runtime.sf') input_star = '' else: cache_store_key = key - # Fast I/O folder - if (config.atmos_clim.agni.verbosity >= 2) or (config.params.out.logging == 'DEBUG'): - io_dir = dirs['output'] - else: - io_dir = create_tmp_folder() - log.info(f'Temporary-file working dir: {io_dir}') - # composition vol_dict = _construct_voldict(config, hf_row, dirs) @@ -688,7 +689,7 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # Stored spectral file is now valid, so store it in the cache if requested. if cache_store_key: - store_in_cache(config.atmos_clim.spectral_cache, cache_store_key, dirs['output']) + store_in_cache(config.atmos_clim.spectral_cache, cache_store_key, io_dir) # Set temperature profile from old NetCDF if it exists nc_files = glob.glob(os.path.join(dirs['output'], 'data', '*_atm.nc')) diff --git a/tests/atmos_clim/test_agni.py b/tests/atmos_clim/test_agni.py index 4f68bb933..cb431d2b7 100644 --- a/tests/atmos_clim/test_agni.py +++ b/tests/atmos_clim/test_agni.py @@ -5,6 +5,9 @@ - Aerosol discovery (_determine_aerosols) - Condensate species determination (_determine_condensates) - AGNI atmosphere initialization (init_agni_atmos) +- Reuse of prepared spectral files across runs, covering the folder AGNI builds + them in, the hit path that skips the stellar insertion, and the contract that + an unusable cache slows a run down without changing its result - Temperature-profile carry-over between iterations (_validate_stored_profile, update_agni_atmos), covering pressure and temperature positivity, profile monotonicity under interpolation, and the atmosphere-failure contract when @@ -18,6 +21,7 @@ from __future__ import annotations import logging +from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -33,6 +37,7 @@ init_agni_atmos, write_atmos_ncdf, ) +from proteus.atmos_clim.spectral_cache import cache_key from proteus.utils.constants import noble_gases pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] @@ -417,6 +422,193 @@ def test_init_agni_atmos_greygas_bypasses_spectral_copy(monkeypatch, tmp_path): assert fake_agni.last_setup_kwargs['κ_grey_sw'] == pytest.approx(0.2) +class _SpectralWritingAGNI(_FakeAGNI): + """Fake AGNI that writes the runtime spectral pair where the real one does. + + AGNI builds `/runtime.sf` and its `_k` companion inside `allocate!`, + and only when a stellar spectrum is supplied; an empty spectrum means the + spectral file it was handed is already prepared and is used untouched. + """ + + def _allocate_b(self, atmos, input_star, **kwargs): + if input_star: + io_dir = Path(self.last_setup_kwargs['IO_DIR']) + io_dir.mkdir(parents=True, exist_ok=True) + star_name = Path(input_star).name + (io_dir / 'runtime.sf').write_text(f'prepared from {star_name}', encoding='utf-8') + (io_dir / 'runtime.sf_k').write_text(f'ktable from {star_name}', encoding='utf-8') + return super()._allocate_b(atmos, input_star, **kwargs) + + +def _setup_cached_spectral_run(monkeypatch, tmp_path, cache_dir, verbosity=1, log_level='INFO'): + """Wire up an init_agni_atmos call that goes through the spectral-file cache. + + The base spectral file and the stellar spectrum are real files, because the + cache key fingerprints both. `verbosity` and `log_level` select which folder + AGNI works in: the output folder only when verbose or debug-logged. + """ + fake_agni = _SpectralWritingAGNI() + fake_jl = SimpleNamespace(AGNI=fake_agni, Dict=dict, Char=str) + + output_dir = tmp_path / 'out' + data_dir = output_dir / 'data' + data_dir.mkdir(parents=True) + sflux = data_dir / '100.sflux' + sflux.write_text('400.0 1.0\n500.0 2.0\n', encoding='utf-8') + + fwl_dir = tmp_path / 'fwl' + fwl_dir.mkdir(parents=True, exist_ok=True) + base_sf = fwl_dir / 'Honeyside.sf' + base_sf.write_text('base spectral file, no star inserted', encoding='utf-8') + + scratch = tmp_path / 'scratch' + + def _fake_tmp_folder(): + scratch.mkdir(parents=True, exist_ok=True) + return str(scratch) + + config = _build_greygas_config() + config.atmos_clim.agni.spectral_file = None + config.atmos_clim.agni.verbosity = verbosity + config.atmos_clim.spectral_group = 'Honeyside' + config.atmos_clim.spectral_bands = '16' + config.atmos_clim.spectral_cache = str(cache_dir) + config.params.out.logging = log_level + + monkeypatch.setattr(agni_mod, 'jl', fake_jl) + monkeypatch.setattr(agni_mod, 'convert', lambda _typ, value: value) + monkeypatch.setattr(agni_mod, '_construct_voldict', lambda *_a, **_k: {'H2O': 1.0}) + monkeypatch.setattr(agni_mod, 'sync_log_files', lambda *_a, **_k: None) + monkeypatch.setattr(agni_mod, 'get_spfile_path', lambda *_a, **_k: str(base_sf)) + monkeypatch.setattr(agni_mod, 'create_tmp_folder', _fake_tmp_folder) + + return SimpleNamespace( + fake_agni=fake_agni, + dirs={'output': str(output_dir), 'agni': '/fake/agni', 'fwl': str(fwl_dir)}, + config=config, + hf_row={ + 'F_ins': 1000.0, + 'albedo_pl': 0.2, + 'T_surf': 900.0, + 'gravity': 9.8, + 'R_int': 6.4e6, + 'P_surf': 1.0, + 'axial_period': 86400.0, + 'longitude': 0.0, + 'latitude': 0.0, + }, + output_dir=output_dir, + scratch=scratch, + base_sf=base_sf, + sflux=sflux, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + ('verbosity', 'log_level', 'work_dir'), + [(1, 'INFO', 'scratch'), (2, 'INFO', 'output')], + ids=['quiet-run-works-in-scratch', 'verbose-run-works-in-output'], +) +def test_spectral_cache_is_filled_from_the_folder_agni_wrote_in( + monkeypatch, tmp_path, verbosity, log_level, work_dir +): + """A run that builds a spectral file leaves a cache entry, in either work folder. + + AGNI writes the prepared file into its own working folder, which is the run's + output folder only when the run is verbose or debug-logged. A quiet run, the + default and the one inference workers use, works in a scratch folder instead; + harvesting the entry from the output folder there stores nothing at all, so + the cache stays empty and every later run repeats the insertion. + """ + cache = tmp_path / 'cache' + ctx = _setup_cached_spectral_run( + monkeypatch, tmp_path, cache, verbosity=verbosity, log_level=log_level + ) + + atmos = init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) + assert atmos is not None + + # Cache miss: this run did the insertion itself, so allocate saw the spectrum. + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + + built_in = ctx.scratch if work_dir == 'scratch' else ctx.output_dir + assert (built_in / 'runtime.sf').is_file() + + key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16') + assert sorted(p.name for p in cache.iterdir()) == [f'{key}.sf', f'{key}.sf_k'] + assert (cache / f'{key}.sf').read_text() == (built_in / 'runtime.sf').read_text() + assert (cache / f'{key}.sf_k').read_text() == (built_in / 'runtime.sf_k').read_text() + + # Discriminating guard: in the quiet run the output folder holds no prepared + # file, so a harvest pointed there would find nothing and cache nothing. + quiet_output_is_empty = not (ctx.output_dir / 'runtime.sf').is_file() + assert quiet_output_is_empty == (work_dir == 'scratch') + + +@pytest.mark.unit +def test_a_cached_spectral_file_is_reused_without_reinserting_the_spectrum( + monkeypatch, tmp_path +): + """A cache hit hands AGNI the prepared file and skips the stellar insertion. + + The seeded pair has to land in the folder AGNI reads from, and the path + handed to setup has to be that copy: pointing at the output folder in a quiet + run names a file that was never created there. + """ + cache = tmp_path / 'cache' + cache.mkdir() + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) + + key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16') + (cache / f'{key}.sf').write_text('cached prepared file', encoding='utf-8') + (cache / f'{key}.sf_k').write_text('cached ktable', encoding='utf-8') + + atmos = init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) + assert atmos is not None + + # Empty spectrum: AGNI takes the file as already prepared and does not rebuild. + assert ctx.fake_agni.last_allocate_input_star == '' + + # setup_b positional args: [dirs['agni'], dirs['output'], input_sf, ...] + assert ctx.fake_agni.last_setup_args[2] == str(ctx.scratch / 'runtime.sf') + assert (ctx.scratch / 'runtime.sf').read_text() == 'cached prepared file' + assert (ctx.scratch / 'runtime.sf_k').read_text() == 'cached ktable' + + # Guard: the companion must travel with its file. A seeded pair that AGNI + # cannot find is the failure mode the path-choice above exists to avoid. + assert not (ctx.output_dir / 'runtime.sf').exists() + + +@pytest.mark.unit +def test_a_cache_that_cannot_be_written_costs_time_and_not_correctness( + monkeypatch, tmp_path, caplog +): + """An unusable cache folder degrades to a normal build instead of failing. + + A cache path occupied by a regular file cannot hold entries. The run must + still initialise, still insert the spectrum itself, and leave the occupying + file untouched. + """ + blocked = tmp_path / 'blocked' + blocked.write_text('not a folder', encoding='utf-8') + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, blocked) + + with caplog.at_level(logging.WARNING, logger='fwl.proteus.atmos_clim.spectral_cache'): + atmos = init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) + + assert atmos is not None + assert 'Could not store spectral file in cache' in caplog.text + + # The run built its own file, exactly as it would with the cache switched off. + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + assert (ctx.scratch / 'runtime.sf').is_file() + + # Nothing was written over the occupying file. + assert blocked.is_file() + assert blocked.read_text() == 'not a folder' + + @pytest.mark.unit def test_init_agni_atmos_loads_the_row_matched_profile(monkeypatch, tmp_path): """init_agni_atmos seeds AGNI from the atmosphere written for this row. From 125bdc7d3085296956501aef4a7e705254b135eb Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 11:20:14 +0200 Subject: [PATCH 45/72] Keep per-run config entries out of inference failure reports by using a copy --- src/proteus/inference/objective.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 8a4d41f0f..be8331708 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -212,30 +212,30 @@ def run_proteus( # Ensure output directory exists out_abs.mkdir(parents=True, exist_ok=True) + # Swept parameter values only. Add `updates` as a copy for the failure report + # so that the original `parameters` dict is unchanged. + swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} + updates = dict(parameters) + # Inject output path into simulation parameters - parameters['params.out.path'] = str(out_dir) + updates['params.out.path'] = str(out_dir) # Every evaluation of an inference run that holds the star fixed builds the same - # prepared spectral file. Point them all at one folder so only the first - # pays for it. - parameters['atmos_clim.spectral_cache'] = str( + # prepared spectral file. + updates['atmos_clim.spectral_cache'] = str( Path(get_proteus_directories(output)['output']) / SPECTRAL_CACHE_DIR ) # Don't allow workers to make plots or logs - parameters.update(WORKER_CONFIG_OVERRIDES) + updates.update(WORKER_CONFIG_OVERRIDES) # Generate config - update_toml(ref_config, parameters, str(out_cfg)) + update_toml(ref_config, updates, str(out_cfg)) # Generate environment env = dict(**os.environ) env['OMP_NUM_THREADS'] = '1' - # Swept parameter values only, for the failure report. The output path and - # the worker overrides are fixed for every run and add no diagnostic value. - swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} - # A run that dies before its logger is configured leaves no logfile behind, so # this stream records it. console = out_abs.parent / f'{out_abs.name}{CHILD_CONSOLE_SUFFIX}' @@ -288,7 +288,7 @@ def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: stream.close() # Re-write config in case simulator mutates or removes it - update_toml(ref_config, parameters, str(out_cfg)) + update_toml(ref_config, updates, str(out_cfg)) # Read status file status = ReadStatus(out_abs) From ffff6abc8f82cadffb8d65574a04c671949ed046 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 11:25:40 +0200 Subject: [PATCH 46/72] Test failure recording through the real run_proteus --- tests/inference/test_objective.py | 105 +++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index bc7105825..9283f9d73 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -8,6 +8,7 @@ from __future__ import annotations +import csv import logging import subprocess @@ -112,7 +113,8 @@ def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): """``run_proteus`` handles the escaped-atmosphere case (P_surf=0): the observable dictionary is populated with zeros instead of NaN, and ``update_toml`` is invoked exactly twice (once per simulator pass) - so the inversion harness sees a numeric value. + so the inversion harness sees a numeric value. The per-run config entries + are added to what is written, never to the parameters passed in. """ out_abs = tmp_path / 'sim' out_abs.mkdir(parents=True) @@ -133,7 +135,7 @@ def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): ) monkeypatch.setattr(objective_mod.subprocess, 'run', lambda *args, **kwargs: None) - parameters = {} + parameters = {'planet.mass_tot': 2.0} obs, status = objective_mod.run_proteus( parameters=parameters, worker=1, @@ -146,6 +148,13 @@ def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): assert obs['P_surf'] == pytest.approx(0.0) assert obs['atm_kg_per_mol'] == pytest.approx(0.0) assert len(updates) == 2 + # The fixed entries reach the config that is written, but not the caller's + # dict: `J` reuses that dict for the failure report, which formats every + # value as a number and names only the swept parameters. + assert updates[0][1]['params.out.path'] == 'dummy_output/workers/w_1/i_2' + assert updates[0][1]['params.out.plot_mod'] == 'none' + assert list(parameters) == ['planet.mass_tot'] + assert parameters['planet.mass_tot'] == pytest.approx(2.0) # No status file was written, which is reported as such rather than as a # generic error: a run that dies during start-up and a run that reaches # the main loop and fails there call for different investigations. @@ -948,3 +957,95 @@ def test_run_output_dir_names_the_folder_the_simulator_is_given(monkeypatch, tmp rel_init, _ = objective_mod.run_output_dir('study', -1, 7) assert rel_init.as_posix() == 'study/workers/w_-1/i_7' assert rel_init != rel + + +@pytest.mark.unit +def test_J_records_clean_exit_failures_through_the_real_simulator_wrapper( + monkeypatch, tmp_path +): + """Only the simulator call is replaced, so `J` drives the real `run_proteus`, + config writing, status reading and failure table. Four evaluations cover + the outcomes a study meets: a crash, a clean exit on an error status (25), + a clean exit on a status the study excludes (11), and a completed run (13). + + The two clean-exit outcomes build their report from the swept values held + by `J`. Those must stay numeric and limited to the swept keys, or the report + cannot be formatted and the table gains columns partway through, after + which it no longer reads back and the summary counts every run as usable. + """ + monkeypatch.setenv('PROTEUS_OUTPUT_PATH', str(tmp_path)) + monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') + ref_config = tmp_path / 'reference.toml' + ref_config.write_text('[planet]\nmass_tot = 1.0\n\n[params.out]\npath = "unset"\n') + + # Status each worker's run records, and whether it then crashes. + outcomes = {0: (21, True), 1: (25, False), 2: (11, False), 3: (13, False)} + calls = [] + + def _fake_run(command, **_kwargs): + cfg = toml.load(command[3]) + calls.append(cfg) + out_abs = tmp_path / cfg['params']['out']['path'] + worker = int(out_abs.parent.name.removeprefix('w_')) + status, crashes = outcomes[worker] + (out_abs / 'status').write_text(f'{status}\n') + pd.DataFrame([{'P_surf': 1e5, 'R_obs': 9.25e6}]).to_csv( + out_abs / 'runtime_helpfile.csv', sep=' ', index=False + ) + if crashes: + raise subprocess.CalledProcessError(returncode=1, cmd=command) + + monkeypatch.setattr(objective_mod.subprocess, 'run', _fake_run) + + # A distinct swept value per worker, so each row can be matched to its run. + scores = { + w: objective_mod.J( + x=torch.tensor([[1.5 + w]], dtype=torch.double), + parameters=['planet.mass_tot'], + true_observables={'R_obs': 9.25e6}, + worker=w, + iter=0, + output='study', + ref_config=str(ref_config), + failure_codes=[11], + ).item() + for w in outcomes + } + + assert [scores[w] for w in (0, 1, 2)] == pytest.approx([objective_mod.BAD_OBJ_VALUE] * 3) + # Discrimination: the completed run goes through the same wrapper and is + # scored on its observables, -log10(0 + 1e-10) = 10 for an exact match. + assert scores[3] == pytest.approx(10.0, rel=1e-9) + + # The fixed entries still reach the simulator config; keeping them out of + # the report must not keep them out of the run. + assert len(calls) == 4 + assert calls[1]['params']['out']['path'] == 'study/workers/w_1/i_0' + assert calls[1]['params']['out']['plot_mod'] == 'none' + assert calls[1]['atmos_clim']['spectral_cache'].endswith('spectral_cache') + assert calls[1]['planet']['mass_tot'] == pytest.approx(2.5) + + # One header and one row per unscored run, all the same width: the fixed + # columns plus the single swept parameter, and none of the fixed entries. + # Parsed as CSV rather than split on commas: status descriptions such as + # the one for status 25 contain a comma and are written quoted. + with open(tmp_path / 'study' / failures_mod.FAILURE_CSV, newline='') as f: + table = list(csv.reader(f)) + header = table[0] + assert header == [*failures_mod._FAILURE_COLUMNS, 'planet.mass_tot'] + assert 'params.out.path' not in header + assert len(table) == 4 + assert {len(row) for row in table} == {len(header)} + + records = failures_mod.read_failure_records(tmp_path / 'study') + assert [(r['worker'], r['status'], r['category']) for r in records] == [ + (0, 21, objective_mod.CATEGORY_FAILURE), + (1, 25, objective_mod.CATEGORY_FAILURE), + (2, 11, objective_mod.CATEGORY_EXCLUDED), + ] + assert [r['planet.mass_tot'] for r in records] == pytest.approx([1.5, 2.5, 3.5]) + assert [r['exit_code'] for r in records] == [1, 0, 0] + + # The study tally counts the three unscored runs. An unreadable table would + # report zero here and describe every evaluation as usable. + assert failures_mod.summarise_failures(str(tmp_path / 'study'), n_attempted=4) == 3 From 5b9dda74744731739cb56c025fcdd59b216435ee Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 11:49:57 +0200 Subject: [PATCH 47/72] stop optimising on abort_on_failure set to true and with a failure --- src/proteus/inference/async_BO.py | 44 ++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index 524666972..2e6d2dc2a 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -26,6 +26,7 @@ import torch from proteus.inference.BO import BO_step, init_locs +from proteus.inference.failures import ProteusRunFailure from proteus.inference.utils import get_kernel, load_dataset_csv, save_dataset_csv from proteus.utils.coupler import get_proteus_directories from proteus.utils.logs import attach_worker_logfile @@ -95,6 +96,8 @@ def worker( output_dir: str, logpath: str | None = None, log_level: int = logging.INFO, + stop=None, + aborts=None, ) -> None: """Worker subprocess that performs asynchronous BO steps. @@ -103,7 +106,8 @@ def worker( 2. Calls BO_step to propose and evaluate a new point. 3. Logs timing and performance metrics. 4. Updates shared data, busy points, and checkpoints. - Runs until the total number of observations reaches max_len. + Runs until the total number of observations reaches max_len, or until + `stop` is set because a worker's run failed under `abort_on_failure`. Parameters ---------- @@ -122,6 +126,11 @@ def worker( - logpath (str | None): Inference run logfile to reopen when this process has no logging configuration of its own. - log_level (int): Numeric level to log at, read from the parent. + - stop (Manager.Event | None): Set by the worker whose run failed under + `abort_on_failure`, and checked by every worker before it starts + another evaluation. + - aborts (Manager.list | None): Receives the failure that set `stop`, so + the parent can raise it once every worker has exited. Returns ---------- @@ -148,7 +157,16 @@ def worker( worker_id, log_list, output_dir, + stop, ) + except ProteusRunFailure as failure: + # Only raised out of the objective under `abort_on_failure`. + if aborts is not None: + aborts.append(failure) + if stop is not None: + stop.set() + log.exception(f'Worker {worker_id} stopped the study after a failed run') + raise except BaseException: # A worker that dies takes its traceback with it: multiprocessing # prints it to the parent's stderr without consulting the logging @@ -178,6 +196,7 @@ def _worker_loop( worker_id: int, log_list, output_dir: str, + stop=None, ) -> None: """Run BO iterations until the evaluation budget is reached. @@ -202,6 +221,12 @@ def _worker_loop( log.info(f'Worker {worker_id} exiting') break + # Another worker's run failed under `abort_on_failure`. Checked + # between evaluations only. + if stop is not None and stop.is_set(): + log.info(f'Worker {worker_id} exiting: the study is stopping on a failed run') + break + # For the first iteration, use provided initial point x_in = x_init if task_id == 0 else None @@ -336,6 +361,10 @@ def parallel_process( n_init = len(D_shared['X']) lock = mgr.Lock() + # Set by the first worker whose run fails under `abort_on_failure`, + # which also leaves that failure in `aborts` for the parent to re-raise. + stop = mgr.Event() + aborts = mgr.list() log_list = mgr.list([None] * n_init) # no logs from init data # Generate initial candidate locations and busy-map @@ -377,6 +406,8 @@ def parallel_process( output_abspath, worker_logpath, worker_log_level, + stop, + aborts, ), ) p.start() @@ -386,6 +417,17 @@ def parallel_process( for p in procs: p.join() + # A failed run under `abort_on_failure` ends the study here. + if len(aborts): + failure = aborts[0] + n_done = len(D_shared['X']) + log.error( + 'Study stopped: a run failed and abort_on_failure is set. ' + f'{n_done} evaluation{"" if n_done == 1 else "s"}, initial samples ' + 'included, completed before it stopped.' + ) + raise failure + # Collect final results D_final = dict(D_shared) logs = list(log_list) From dc0b16b8178fcd5f4d2ac61ac3abcf894dbb3c18 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 11:51:00 +0200 Subject: [PATCH 48/72] assert that optimisation stops under failure with abort_on_failure --- tests/inference/test_async_bo.py | 190 +++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index 7a0bfcf12..e3a77ed06 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging +import threading import pandas as pd import pytest @@ -21,6 +22,7 @@ pytest.importorskip('gpytorch') import proteus.inference.async_BO as async_mod # noqa: E402 +from proteus.inference.failures import ProteusRunFailure # noqa: E402 pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] @@ -219,6 +221,9 @@ def list(self, data=None): def Lock(self): return _DummyLock() + def Event(self): + return threading.Event() + created_processes = [] class FakeProcess: @@ -310,6 +315,9 @@ def list(self, data=None): def Lock(self): return _DummyLock() + def Event(self): + return threading.Event() + (tmp_path / 'init.csv').write_text('x_0,y\n0.1,0.2\n', encoding='utf-8') monkeypatch.setattr( async_mod, 'get_proteus_directories', lambda _output: {'output': str(tmp_path)} @@ -732,3 +740,185 @@ def exploding_process_fun(**_kwargs): assert not (tmp_path / 'infer.log').exists() reported = '\n'.join(record.getMessage() for record in caplog.records) assert 'Worker 0 stopped early' in reported + + +# ============================================================================ +# Stopping the study on a failed run under abort_on_failure +# ============================================================================ + + +def _run_failure(worker_id): + """The failure the objective raises under `abort_on_failure`.""" + return ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker_id, + iter=0, + out_dir=f'/study/workers/w_{worker_id}/i_0', + exit_code=1, + status=21, + parameters={'a': 0.25}, + ) + + +@pytest.mark.unit +def test_worker_signals_the_study_to_stop_on_a_failed_run(tmp_path): + """A failed run that reaches the worker, which happens only under + `abort_on_failure`, sets the shared stop signal and hands the failure to + the parent before the worker exits. Any other error ends only the worker + that raised it, as before, so the rest of the study carries on. + """ + B = {0: torch.tensor([[0.3]], dtype=torch.double)} + stop = threading.Event() + aborts = [] + + def failing_process_fun(**_kwargs): + raise _run_failure(0) + + def _call(process_fun): + async_mod.worker( + process_fun=process_fun, + build_obj=lambda **kwargs: lambda x: x, + D_shared={ + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[0.2]], dtype=torch.double), + }, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.3]], dtype=torch.double), + n_init=1, + lock=_DummyLock(), + max_len=4, + worker_id=0, + log_list=[], + output_dir=str(tmp_path), + stop=stop, + aborts=aborts, + ) + + with pytest.raises(ProteusRunFailure): + _call(failing_process_fun) + assert stop.is_set() + assert [(f.worker, f.status) for f in aborts] == [(0, 21)] + # The claimed point is still released on this path. + assert 0 not in B + + # Discrimination: a worker that dies of anything else does not stop the + # study. Without this the test would pass against a worker that set the + # signal on every exception. + stop.clear() + aborts.clear() + + def exploding_process_fun(**_kwargs): + raise RuntimeError('objective evaluation failed') + + with pytest.raises(RuntimeError, match='objective evaluation failed'): + _call(exploding_process_fun) + assert not stop.is_set() + assert aborts == [] + + +@pytest.mark.unit +def test_worker_starts_no_evaluation_once_the_study_is_stopping(tmp_path, caplog): + """A worker that finds the stop signal set exits before its next + evaluation, although the budget is far from reached, and says why. + """ + stop = threading.Event() + stop.set() + B = {1: torch.tensor([[0.6]], dtype=torch.double)} + + with caplog.at_level('INFO', logger='fwl.proteus.inference.async_BO'): + async_mod.worker( + process_fun=lambda **_kwargs: pytest.fail('no evaluation should run'), + build_obj=lambda **kwargs: lambda x: x, + D_shared={ + 'X': torch.tensor([[0.1]], dtype=torch.double), + 'Y': torch.tensor([[0.2]], dtype=torch.double), + }, + B=B, + T=[], + T0=0.0, + x_init=torch.tensor([[0.6]], dtype=torch.double), + n_init=1, + lock=_DummyLock(), + # One sample of 50: exiting here is the stop signal's doing, not + # the budget's. + max_len=50, + worker_id=1, + log_list=[], + output_dir=str(tmp_path), + stop=stop, + aborts=[], + ) + + assert 1 not in B + messages = [r.getMessage() for r in caplog.records] + assert 'Worker 1 exiting: the study is stopping on a failed run' in messages + # Not reported as the ordinary exit on a spent budget. + assert 'Worker 1 exiting' not in messages + + +@pytest.mark.unit +def test_parallel_process_stops_the_study_when_a_run_fails_under_abort( + monkeypatch, tmp_path, caplog +): + """Under `abort_on_failure` one failed run ends the optimisation phase: + the other workers start no further evaluations, and the failure is raised + once they have all exited instead of a partial study being returned for a + best-fit summary. The real worker runs in-process here, one after another, + so worker 1 starts after worker 0 has failed. + """ + + class FakeProcess: + def __init__(self, target, args): + self.target = target + self.args = args + self.exitcode = None + + def start(self): + try: + self.target(*self.args) + self.exitcode = 0 + except BaseException: + self.exitcode = 1 + + def join(self): + return None + + _mocked_parallel_process_env(monkeypatch, tmp_path, FakeProcess) + monkeypatch.setattr(async_mod, '_parent_logfile', lambda: None) + + evaluated = [] + + def fake_bo_step(f, D, B, x_in, lock, worker_id, **_kwargs): + evaluated.append(worker_id) + if worker_id == 0: + raise _run_failure(0) + x = torch.tensor([[0.5]], dtype=torch.double) + return x, torch.tensor([[0.7]], dtype=torch.double), 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + + monkeypatch.setattr(async_mod, 'BO_step', fake_bo_step) + + with caplog.at_level('ERROR'), pytest.raises(ProteusRunFailure) as excinfo: + async_mod.parallel_process( + objective_builder=lambda **kwargs: lambda x: x, + kernel='MAT3/2', + acqf='LogEI', + n_workers=2, + max_len=6, + output='dummy', + seed=1, + ref_config='ref.toml', + observables={'obs': 1.0}, + parameters={'a': [0.0, 1.0]}, + failure_codes=[], + ) + + # The failure raised is the one that stopped the study. + assert (excinfo.value.worker, excinfo.value.status) == (0, 21) + # Worker 1 started no evaluation, although 5 steps of the budget remained. + assert evaluated == [0] + reported = '\n'.join(r.getMessage() for r in caplog.records) + assert 'abort_on_failure is set' in reported + # Only the initial sample was in the dataset when the study stopped. + assert '1 evaluation, initial samples included, completed' in reported From ae2df064f8dab2e88524ffaf95cdcee405e42fbe Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:07:38 +0200 Subject: [PATCH 49/72] reject log-scaled params with bounds that include 0 or below --- src/proteus/inference/inference.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index 550fbddac..cbabc0043 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -39,7 +39,7 @@ set_child_timeout, ) from proteus.inference.utils import print_results, str_time -from proteus.utils.coupler import get_proteus_directories +from proteus.utils.coupler import get_proteus_directories, variable_is_logarithmic from proteus.utils.helper import safe_rm from proteus.utils.logs import setup_logger @@ -87,7 +87,8 @@ def parameter_bounds(parameters: dict) -> dict[str, tuple[float, float]]: - dict[str, tuple[float, float]]: Same keys, bounds as (min, max) floats. Raises: - ValueError: If a range is not a pair of numbers, or does not increase. + ValueError: If a range is not a pair of numbers, does not increase, or + includes zero or a negative value for a log-scaled parameter. """ bounds: dict[str, tuple[float, float]] = {} for key, value in parameters.items(): @@ -112,6 +113,12 @@ def parameter_bounds(parameters: dict) -> dict[str, tuple[float, float]]: raise ValueError( f"Bounds for inference parameter '{key}' must increase, got [{low:g}, {high:g}]" ) + # Reject a log-scale parameter bound that includes zero or a negative value. + if variable_is_logarithmic(key) and low <= 0: + raise ValueError( + f"Inference parameter '{key}' is swept on a log scale, so both bounds " + f'must be > 0, got [{low:g}, {high:g}]' + ) bounds[key] = (low, high) return bounds From 821c44e27682cce15f0599574f43843a68323f3c Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:08:30 +0200 Subject: [PATCH 50/72] test that log-scaled bounds with 0 or below are rejected but linear ones are not --- tests/inference/test_inference.py | 55 ++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/inference/test_inference.py b/tests/inference/test_inference.py index a75997d9b..d96893eb4 100644 --- a/tests/inference/test_inference.py +++ b/tests/inference/test_inference.py @@ -185,6 +185,38 @@ def test_parameter_bounds_converts_pairs_and_rejects_malformed_ranges(): inference_mod.parameter_bounds({'planet.mass_tot': [float('nan'), 3.0]}) +@pytest.mark.unit +def test_parameter_bounds_rejects_a_log_scaled_range_that_reaches_zero(): + """A parameter swept on a log scale cannot have a bound at or below zero: + the optimiser samples it in log10 space. The range is rejected here, while + the config is being read, not in every worker after the previous study + has been removed. `planet.elements.H_budget` is log-scaled; a linear + parameter with the same bounds is accepted. + """ + with pytest.raises(ValueError, match='log scale') as excinfo: + inference_mod.parameter_bounds({'planet.elements.H_budget': [0.0, 2e4]}) + # The message names the offending parameter and the range as written. + assert "'planet.elements.H_budget'" in str(excinfo.value) + assert '[0, 20000]' in str(excinfo.value) + with pytest.raises(ValueError, match='log scale'): + inference_mod.parameter_bounds({'planet.elements.H_budget': [-10.0, 2e4]}) + # A range that is negative throughout is rejected on the same grounds. + with pytest.raises(ValueError, match='log scale'): + inference_mod.parameter_bounds({'orbit.semimajoraxis': [-2.0, -1.0]}) + + # Edge case: the smallest positive lower bound is accepted, so the check is + # `> 0` and not a threshold further from zero. + tiny = inference_mod.parameter_bounds({'planet.elements.H_budget': [5e-324, 2e4]}) + assert tiny['planet.elements.H_budget'][0] > 0.0 + + # Discrimination: the same bounds on a linear parameter are valid, so the + # rejection comes from the log scale, not from the value itself. + linear = inference_mod.parameter_bounds({'outgas.fO2_shift_IW': [-4.0, 0.0]}) + assert linear['outgas.fO2_shift_IW'] == (pytest.approx(-4.0), pytest.approx(0.0)) + assert inference_mod.variable_is_logarithmic('outgas.fO2_shift_IW') is False + assert inference_mod.variable_is_logarithmic('planet.elements.H_budget') is True + + @pytest.mark.unit def test_validate_reference_config_accepts_a_runnable_sweep(): """A reference config that PROTEUS accepts, swept over parameters that stay @@ -274,11 +306,24 @@ def test_validate_reference_config_rejects_a_faulty_reference_file(tmp_path): @pytest.mark.unit -def test_run_inference_validates_reference_config_before_emptying_output(monkeypatch, tmp_path): +@pytest.mark.parametrize( + ('parameters', 'error', 'match'), + [ + ({'planet.mass_tott': [0.7, 3.0]}, UnknownConfigKeyError, 'planet.mass_tott'), + # Accepted by the schema, and rejected by the optimiser only once the + # workers start sampling it in log10 space. + ({'planet.elements.H_budget': [0.0, 2e4]}, ValueError, 'log scale'), + ], + ids=['misspelt_parameter', 'log_scaled_range_reaching_zero'], +) +def test_run_inference_validates_reference_config_before_emptying_output( + monkeypatch, tmp_path, parameters, error, match +): """``run_inference`` validates the reference config before it empties the study output folder and before it generates any initial design. Re-running - a finished study with a typo'd parameter name must cost the user neither - simulation time nor the previous study's results. + a finished study with a typo'd parameter name, or with a range the + optimiser cannot sample, must cost the user neither simulation time nor + the previous study's results. """ config = { 'output': 'unit_inference', @@ -290,7 +335,7 @@ def test_run_inference_validates_reference_config_before_emptying_output(monkeyp 'acqf': 'LogEI', 'seed': 1, 'observables': {'P_surf': 1.0}, - 'parameters': {'planet.mass_tott': [0.7, 3.0]}, + 'parameters': parameters, } # Stand in for a completed earlier study occupying the same output folder. output_root = tmp_path / 'output' @@ -314,7 +359,7 @@ def test_run_inference_validates_reference_config_before_emptying_output(monkeyp inference_mod, 'create_init', lambda *a, **kw: create_init_calls.append((a, kw)) ) - with pytest.raises(UnknownConfigKeyError, match='planet.mass_tott'): + with pytest.raises(error, match=match): inference_mod.run_inference(config) # Ordering: the guard must fire before the initial design is generated. assert create_init_calls == [] From d57e93f36d8de5db45343cfff2c739bd564d070d Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:19:06 +0200 Subject: [PATCH 51/72] add status 29 (evaporated) to an always_excluded category: should never be fit against but it is not a crash --- src/proteus/inference/objective.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index be8331708..20acdf078 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -30,6 +30,8 @@ EPS_CLIP = 1e-10 LOG_CLIP = 1e-20 BAD_OBJ_VALUE = -20.0 +# Completion statuses excluded from every fit, whatever `failure_codes` holds. +ALWAYS_EXCLUDED_STATUSES = frozenset({29}) log = logging.getLogger('fwl.' + __name__) # Per-child PROTEUS run timeout for inference workers. A single wedged child @@ -453,11 +455,10 @@ def J( failed = (20 <= sim_status <= 28) or (sim_status in (0, 1, STATUS_MISSING)) # Runs that completed normally on an outcome this study does not fit - # against, named by the `failure_codes` field of the inference config: a - # run stopped by its clock limit (status 11) or one whose volatiles all - # escaped (status 15), for instance. Nothing went wrong in such a run, so - # it is scored as a poor sample but is not reported as a fault. - excluded = (not failed) and (sim_status in failure_codes) + # against, in `failure_codes` or in ALWAYS_EXCLUDED_STATUSES. + excluded = (not failed) and ( + sim_status in failure_codes or sim_status in ALWAYS_EXCLUDED_STATUSES + ) # Either way the evaluation carries the failure score instead of a fit # quality, and is recorded so that the end-of-study tally covers it. From f76c8779f4425d45f505dcedf607601cc88187af Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:20:17 +0200 Subject: [PATCH 52/72] assert status 29 is always excluded --- tests/inference/test_objective.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 9283f9d73..b1a743bd9 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -817,16 +817,13 @@ def _run(status, worker, iter, codes=()): @pytest.mark.unit def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): """The failure range covers the error statuses the simulator can record. - Code 28 is the highest error the status table defines; 29 is a completion - ('planet evaporated') and no current code path writes it, so it must not - be scored as a failure by an off-by-one in the range bound. """ monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} ) - def _score(status): + def _score(status, worker=0): monkeypatch.setattr( objective_mod, 'run_proteus', @@ -836,18 +833,29 @@ def _score(status): x=torch.tensor([[0.5]], dtype=torch.double), parameters=['planet.mass_tot'], true_observables={'R_obs': 9.25e6}, - worker=0, + worker=worker, iter=0, output='dummy_output', ref_config='reference.toml', + failure_codes=[], ).item() # Highest defined error code, and the escape-model error below it. assert _score(28) == pytest.approx(objective_mod.BAD_OBJ_VALUE) assert _score(21) == pytest.approx(objective_mod.BAD_OBJ_VALUE) - # Completion codes are scored on their observables. - assert _score(29) == pytest.approx(10.0, rel=1e-9) + # An evaporated planet is kept out of the fit with no `failure_codes` entry. + assert _score(29, worker=1) == pytest.approx(objective_mod.BAD_OBJ_VALUE) + recorded = {r['worker']: r for r in failures_mod.read_failure_records(tmp_path)} + assert (recorded[1]['status'], recorded[1]['category']) == ( + 29, + objective_mod.CATEGORY_EXCLUDED, + ) + assert recorded[0]['category'] == objective_mod.CATEGORY_FAILURE + # Discrimination: other completion codes are still scored on their + # observables, -log10(0 + 1e-10) = 10 for an exact match, so status 29 is + # not excluded by a range that also catches 13. assert _score(13) == pytest.approx(10.0, rel=1e-9) + assert _score(18) == pytest.approx(10.0, rel=1e-9) # A run that never updated its status past 'Running' died mid-flight. assert _score(1) == pytest.approx(objective_mod.BAD_OBJ_VALUE) # An unreadable status file is treated as a failure, because the run's own From ebbe8f8f0aa64ffd5e59aab8d9941354f9d4023c Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:20:59 +0200 Subject: [PATCH 53/72] status 29 is always excluded --- input/inference/example.infer.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input/inference/example.infer.toml b/input/inference/example.infer.toml index 0ca369634..639f11b3d 100644 --- a/input/inference/example.infer.toml +++ b/input/inference/example.infer.toml @@ -16,7 +16,7 @@ ref_config = "input/inference/example.toml" init_samps = 3 # Number of random samples if starting from scratch. init_grid = 'none' # grid_demo/' # Path pre-computed grid (relative to PROTEUS output folder) -# Completion codes to exclude from the fit. +# Completion codes to exclude from the fit. Evaporated (29) is always excluded. failure_codes = [11, ] # solidified (10), escaped (15), max_runtime (11) # Stop the whole inference run at the first simulation that fails, instead of scoring From 08a7de332f64b83277b16e56a6b61d090ee18bc9 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:21:30 +0200 Subject: [PATCH 54/72] mention status 29 is always excluded --- docs/How-to/inference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index f19e7ec7c..bdc836045 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -153,7 +153,7 @@ The system prints the final results including: ### Failed and excluded simulations -During the inference run, some PROTEUS simulations might crash or fail, or stop on a status that is excluded in the inference configuration (e.g. maximum runtime reached). The run carries on when there are failures unless `abort_on_failure` is set to `true` in the inference config. At the end of the study all failures are written to `failures.csv` in the output folder, and summarised. +During the inference run, some PROTEUS simulations might crash or fail, or stop on a status that is excluded in the inference configuration (e.g. maximum runtime reached). Status 29 (planet evaporated) is always excluded. The run carries on when there are failures unless `abort_on_failure` is set to `true` in the inference config. At the end of the study all failures are written to `failures.csv` in the output folder, and summarised. ## Customization From 71f0d122f0a0e2592513361ecd8fe210cda774ee Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:30:19 +0200 Subject: [PATCH 55/72] add lock so that concurrent failures do not lose rows in failures.csv --- src/proteus/inference/failures.py | 28 +++++++++++++++++----------- src/proteus/inference/plot.py | 19 +++++++++++++------ 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index 125916d53..b7874330b 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -4,7 +4,9 @@ from __future__ import annotations +import fcntl import logging +import os from collections import Counter from dataclasses import dataclass, field from pathlib import Path @@ -156,11 +158,12 @@ def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | N """Append one row to the study's failure table. Workers are separate processes with no shared state, so each appends its - own row rather than handing the failure back to the parent. The first - worker to fail creates the file with its header through an exclusive - create, which exactly one caller can win, and every later row is a single - append. A row is one `write` call of well under a pipe buffer, which the - kernel adds whole, so no lock is needed on a local filesystem. + own row rather than handing the failure back to the parent. The file is + locked from the check for an empty table through the write, so exactly one + writer adds the header and no row can land ahead of it, which would leave + the table unreadable. The lock is advisory and per open file, so it holds + between the pool workers of initial sampling and the optimisation workers + alike, which share nothing else. Parameters ---------- @@ -182,12 +185,15 @@ def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | N line = _csv_row(ordered.values()) try: target.parent.mkdir(parents=True, exist_ok=True) - try: - with open(target, 'x') as f: - f.write(_csv_row(ordered.keys()) + line) - except FileExistsError: - with open(target, 'a') as f: - f.write(line) + with open(target, 'a') as f: + fcntl.flock(f, fcntl.LOCK_EX) + # Sized after the lock is taken: another writer may have added the + # header between this one opening the file and acquiring the lock. + if os.fstat(f.fileno()).st_size == 0: + line = _csv_row(ordered.keys()) + line + f.write(line) + # Written out before the lock is released on close. + f.flush() except OSError as err: log.warning( f'Could not record the failure of worker={failure.worker} ' diff --git a/src/proteus/inference/plot.py b/src/proteus/inference/plot.py index bf6418e17..0e8099506 100644 --- a/src/proteus/inference/plot.py +++ b/src/proteus/inference/plot.py @@ -570,8 +570,13 @@ def plot_result_correlation(pars: dict, obs: dict, directory): # Extract parameters and observables X, Y = [], [] for c in cases: - # Read config - conf = toml.load(c / 'init_coupler.toml') + # A case that died during start-up has a folder but did not get as far + # as writing its resolved config, and so has nothing to plot. + conf_path = c / 'init_coupler.toml' + if not conf_path.is_file(): + log.warning(f'Missing init_coupler.toml for {c}') + continue + conf = toml.load(conf_path) # Check success hf_path = c / 'runtime_helpfile.csv' @@ -580,7 +585,7 @@ def plot_result_correlation(pars: dict, obs: dict, directory): continue # Read helpfile for observables - help = pd.read_csv(c / 'runtime_helpfile.csv', delimiter=r'\s+') + help = pd.read_csv(hf_path, delimiter=r'\s+') # Get parameters and observables xx = [recursive_get(conf, k.split('.')) for k in par_keys] @@ -589,13 +594,15 @@ def plot_result_correlation(pars: dict, obs: dict, directory): # Store these X.append(xx) Y.append(yy) - X = np.array(X, dtype=float) - Y = np.array(Y, dtype=float) - # Axes n_par = len(par_keys) n_obs = len(obs_keys) + # Shaped explicitly, so that a study in which no case produced output still + # has a column per parameter and observable to index. + X = np.array(X, dtype=float).reshape(-1, n_par) + Y = np.array(Y, dtype=float).reshape(-1, n_obs) + # Make plot fig, axs = plt.subplots(n_obs, n_par, figsize=(2.7 * n_par, 2.7 * n_obs)) for i in range(n_par): From 8395ad4c6f44383c7bec1c4004f8d1c45f7478e3 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 12:30:52 +0200 Subject: [PATCH 56/72] test new lock for failures.csv --- tests/inference/test_failures.py | 62 +++++++++++++++++++++++++++++++ tests/inference/test_plot.py | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/tests/inference/test_failures.py b/tests/inference/test_failures.py index 05c81fa9b..ee69ad992 100644 --- a/tests/inference/test_failures.py +++ b/tests/inference/test_failures.py @@ -12,9 +12,11 @@ from __future__ import annotations +import fcntl import logging import pickle import re +import threading import pandas as pd import pytest @@ -222,6 +224,66 @@ def test_failure_records_round_trip_into_one_table(tmp_path): assert failures_mod.read_failure_records(tmp_path) == [] +@pytest.mark.unit +def test_a_row_cannot_land_ahead_of_the_header_another_writer_is_adding(tmp_path): + """Two workers that fail together must both end up in the table. The first + to open the file finds it empty and adds the header. A second writer that + arrives in between waits for the lock and appends below it, instead of + adding its row first, which would put a data row where the header belongs + and make the whole table unreadable. + + The test takes the place of the first writer: it holds the lock on an + empty table while a second writer is started in a thread, then writes the + header and its own row and releases the lock. + """ + + def _failure(worker): + return failures_mod.ProteusRunFailure( + reason='the simulator exited with an error', + worker=worker, + iter=0, + out_dir=f'/study/workers/w_{worker}/i_0', + exit_code=1, + status=21, + parameters={'planet.mass_tot': 1.0 + worker}, + ) + + first = _failure(0) + first_row = {key: getattr(first, key) for key in failures_mod._FAILURE_COLUMNS} + first_row['planet.mass_tot'] = 1.0 + table = tmp_path / failures_mod.FAILURE_CSV + + with open(table, 'a') as held: + fcntl.flock(held, fcntl.LOCK_EX) + second = threading.Thread( + target=failures_mod.record_failure, args=(tmp_path, _failure(1)) + ) + second.start() + second.join(timeout=0.1) + # The second writer is waiting on the lock, not writing past it. + assert second.is_alive() + assert table.read_text() == '' + held.write( + failures_mod._csv_row(first_row.keys()) + failures_mod._csv_row(first_row.values()) + ) + held.flush() + second.join(timeout=5) + assert not second.is_alive() + + lines = table.read_text().splitlines() + assert lines[0].startswith('worker,iter,') + # One header, and both rows below it. + assert sum(line.startswith('worker,iter,') for line in lines) == 1 + records = failures_mod.read_failure_records(tmp_path) + assert [(r['worker'], r['planet.mass_tot']) for r in records] == [(0, 1.0), (1, 2.0)] + + # Edge case: an empty table left by a writer that was killed before its + # first row gets the header from the next writer, so it reads back. + table.write_text('') + failures_mod.record_failure(tmp_path, _failure(2)) + assert [r['worker'] for r in failures_mod.read_failure_records(tmp_path)] == [2] + + @pytest.mark.unit def test_recording_a_failure_never_masks_the_failure_it_records(tmp_path): """Bookkeeping must not bring down a study. When the record cannot be diff --git a/tests/inference/test_plot.py b/tests/inference/test_plot.py index f914cb194..a55ad56b8 100644 --- a/tests/inference/test_plot.py +++ b/tests/inference/test_plot.py @@ -627,6 +627,70 @@ def test_plot_result_correlation_ignores_stray_console_log_file(monkeypatch, tmp fig.savefig.assert_called_once() +def test_plot_result_correlation_skips_a_case_that_died_during_start_up( + monkeypatch, tmp_path, caplog +): + """A case that died before writing its resolved config has a folder but no + ``init_coupler.toml``. It is skipped with a warning naming it, and the + finished cases are still plotted. A study in which every case died still + produces a figure, with no points on it, rather than failing on an empty + array. + """ + workers = tmp_path / 'workers' + case_ok = workers / 'w_0' / 'i_0' + case_ok.mkdir(parents=True) + (case_ok / 'init_coupler.toml').write_text( + toml.dumps({'planet': {'mass_tot': 1.5}}), + encoding='utf-8', + ) + pd.DataFrame([{'P_surf': 2.0}]).to_csv( + case_ok / 'runtime_helpfile.csv', sep=' ', index=False + ) + # What `run_proteus` leaves for a child that exits before PROTEUS writes + # anything of its own: the folder and the config it was handed. + case_dead = workers / 'w_1' / 'i_0' + case_dead.mkdir(parents=True) + (case_dead / 'input.toml').write_text( + toml.dumps({'planet': {'mass_tot': 2.5}}), encoding='utf-8' + ) + + axis = MagicMock() + axis.__getitem__.return_value = axis + fig = MagicMock() + mock_plt = MagicMock() + mock_plt.subplots.return_value = (fig, axis) + monkeypatch.setattr(plot_mod, 'plt', mock_plt) + monkeypatch.setattr(plot_mod, 'variable_is_logarithmic', lambda _k: False) + + def _plot(): + plot_mod.plot_result_correlation( + pars={'planet.mass_tot': [0.7, 3.0]}, + obs={'P_surf': 1.0}, + directory=str(tmp_path), + ) + + with caplog.at_level('WARNING'): + _plot() + + fig.savefig.assert_called_once() + # Only the finished case is plotted: its parameter against its observable. + xx, yy = axis.scatter.call_args.args[:2] + assert list(xx) == pytest.approx([1.5]) + assert list(yy) == pytest.approx([2.0]) + warnings = [r.getMessage() for r in caplog.records if r.levelname == 'WARNING'] + assert warnings == [f'Missing init_coupler.toml for {case_dead}'] + + # Edge case: no case produced output. The figure is still written, with + # empty data of the right width rather than an IndexError on X[:, 0]. + (case_ok / 'init_coupler.toml').unlink() + axis.reset_mock() + fig.reset_mock() + _plot() + fig.savefig.assert_called_once() + xx, yy = axis.scatter.call_args.args[:2] + assert (len(xx), len(yy)) == (0, 0) + + def test_plot_result_correlation_two_par_two_obs_uses_2d_axes(monkeypatch, tmp_path): """n_par > 1 and n_obs > 1 takes the ``axs[j, i]`` 2D indexing branch. From 57d02456b3784f92071748a4c50194c7cdbcee87 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 13:25:51 +0200 Subject: [PATCH 57/72] refactor: shorten comments, module docstrings and doctrings to improve readability, and improve code efficiency --- src/proteus/inference/BO.py | 18 +---- src/proteus/inference/async_BO.py | 16 +--- src/proteus/inference/failures.py | 52 ++++--------- src/proteus/inference/gen_D_init.py | 28 ++----- src/proteus/inference/inference.py | 4 +- src/proteus/inference/objective.py | 109 ++++++++++++---------------- src/proteus/inference/utils.py | 45 ++++-------- 7 files changed, 84 insertions(+), 188 deletions(-) diff --git a/src/proteus/inference/BO.py b/src/proteus/inference/BO.py index e7535a122..6dbb9e1dd 100644 --- a/src/proteus/inference/BO.py +++ b/src/proteus/inference/BO.py @@ -1,13 +1,5 @@ -"""Bayesian optimization core functions. - -This module provides utility functions and the core BO_step function -for fitting Gaussian processes, optimizing acquisition functions, -and plotting results during Bayesian optimization. - -Functions: - unit_bounds: Generate unit hypercube bounds for acquisition optimization. - plot_iter: Visualize GP posterior and acquisition function at each iteration. - BO_step: Execute a single Bayesian optimization step with timing and logging. +"""Bayesian optimisation steps: fit a GP, optimise the acquisition function, and +evaluate the objective at the chosen point. """ from __future__ import annotations @@ -42,9 +34,7 @@ def unit_bounds(d): ---------- - torch.Tensor: Tensor of shape (2, d) where row 0 is zeros and row 1 is ones. """ - # Build bounds [[0,...,0], [1,...,1]] - bounds = torch.tensor([[0] * d, [1] * d], dtype=dtype) - return bounds + return torch.tensor([[0] * d, [1] * d], dtype=dtype) def BO_step(D, B, f, k, acqf, lock, worker_id, x_in=None): @@ -272,8 +262,6 @@ def plot_iter(gp, acqf, X, Y, next_x, busys, dir, name): ax[0].plot(xs.detach().flatten(), mu, color='black', linewidth=0.5, label='post mean') ax[0].fill_between(xs.detach().flatten(), mu - 2 * sig, mu + 2 * sig, alpha=0.25) ax[0].scatter(X.flatten(), Y.flatten(), s=5, label='data', color='blue') - # fs = objective(xs).flatten() - # ax[0].plot(xs.detach().flatten(), fs, color = "grey", label = "f", linewidth = 0.5) ax[1].plot(xs.detach().flatten(), ac, color='cornflowerblue', label='acqf') a = acqf(next_x).detach().flatten() diff --git a/src/proteus/inference/async_BO.py b/src/proteus/inference/async_BO.py index 2e6d2dc2a..d462b5bed 100644 --- a/src/proteus/inference/async_BO.py +++ b/src/proteus/inference/async_BO.py @@ -1,17 +1,5 @@ -"""Asynchronous Bayesian optimization pipeline utilities. - -This module implements functions to: - - * Save and checkpoint optimization state to disk. - * Generate initial sample locations via a Halton low-discrepancy sequence. - * Run a worker process that performs BO steps, logs metrics, and checkpoints. - * Coordinate multiple worker processes for asynchronous Bayesian optimization. - -Functions: - checkpoint: Persist shared data, logs, and timestamps to files. - init_locs: Create initial candidate points in the unit hypercube. - worker: Execute BO iterations in a subprocess, updating shared state. - parallel_process: Set up shared resources, spawn workers, and collect results. +"""Asynchronous Bayesian optimisation: worker processes that each run BO steps +against shared data, checkpointing as they go, and the orchestration around them. """ from __future__ import annotations diff --git a/src/proteus/inference/failures.py b/src/proteus/inference/failures.py index b7874330b..ddeb62c34 100644 --- a/src/proteus/inference/failures.py +++ b/src/proteus/inference/failures.py @@ -8,7 +8,7 @@ import logging import os from collections import Counter -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path import pandas as pd @@ -24,24 +24,17 @@ # Suffix for the file holding whatever a child wrote to its console. CHILD_CONSOLE_SUFFIX = '_console.log' -# How an evaluation that failed is classified. A run that -# crashed, or stopped in an error state, did not produce a result at all. A run -# that completed normally but ended on a status listed in the study's -# `failure_codes` did produce a result, but the study does not fit against -# that outcome. Only the first is a fault. +# A failure produced no result and is a fault. An excluded run completed on a +# status the study does not fit against (`failure_codes`, or status 29). CATEGORY_FAILURE = 'failure' CATEGORY_EXCLUDED = 'excluded' -# Table inside the study output holding one row per unscored evaluation. -# Appended by the workers as they fail and read back once at the end, so that -# the summary covers initial sampling and optimisation alike without the two -# paths having to share any state while they run. +# One row per unscored evaluation, appended by each worker as it fails and read +# back once at the end, so initial sampling and optimisation share no state. FAILURE_CSV = 'failures.csv' -# Fixed columns of that table, in order. The swept parameter values follow, one -# column each. The two paths are what the user opens after: -# the logfile for a run that got far enough to configure its logger, -# the console capture for one that did not. +# Fixed columns of that table; one column per swept parameter follows. The +# console capture is the only record of a run that died before its logger started. _FAILURE_COLUMNS = ( 'worker', 'iter', @@ -122,25 +115,9 @@ def __str__(self) -> str: return self.report() def __reduce__(self): - # A failure raised inside a pool worker is pickled to be re-raised in - # the parent. BaseException.__reduce__ rebuilds from `self.args`, - # which a dataclass __init__ leaves empty, so the default would fail - # to reconstruct this class. Rebuild from the fields instead. - return ( - self.__class__, - ( - self.reason, - self.worker, - self.iter, - self.out_dir, - self.exit_code, - self.status, - self.log_path, - self.console_path, - self.parameters, - self.category, - ), - ) + # Pickled to cross from a pool worker to the parent. The default rebuilds + # from `self.args`, which the dataclass __init__ leaves empty. + return (self.__class__, tuple(getattr(self, f.name) for f in fields(self))) def find_run_logfile(out_abs: Path | str) -> str | None: @@ -176,13 +153,10 @@ def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | N Recording is best-effort: a study must not be brought down by a fault in its own bookkeeping, so the failure being reported still reaches the log. """ - row = {key: getattr(failure, key) for key in _FAILURE_COLUMNS if key != 'status_desc'} - row['status_desc'] = failure.status_desc - row.update(failure.parameters) - ordered = {key: row[key] for key in (*_FAILURE_COLUMNS, *failure.parameters)} + row = {key: getattr(failure, key) for key in _FAILURE_COLUMNS} | failure.parameters target = Path(study_abs) / FAILURE_CSV - line = _csv_row(ordered.values()) + line = _csv_row(row.values()) try: target.parent.mkdir(parents=True, exist_ok=True) with open(target, 'a') as f: @@ -190,7 +164,7 @@ def record_failure(study_abs: Path | str, failure: ProteusRunFailure) -> str | N # Sized after the lock is taken: another writer may have added the # header between this one opening the file and acquiring the lock. if os.fstat(f.fileno()).st_size == 0: - line = _csv_row(ordered.keys()) + line + line = _csv_row(row.keys()) + line f.write(line) # Written out before the lock is released on close. f.flush() diff --git a/src/proteus/inference/gen_D_init.py b/src/proteus/inference/gen_D_init.py index 35866b379..9eb132e13 100644 --- a/src/proteus/inference/gen_D_init.py +++ b/src/proteus/inference/gen_D_init.py @@ -1,9 +1,5 @@ -"""Generate and save initial dataset for Bayesian optimization. - -This script sets up parameter bounds and true observables for the PROTEUS simulator, -builds the objective function via `prot_builder`, generates a small random sample -of points in the normalized input space, evaluates the objective to obtain outputs, -and saves the resulting dataset to disk for use as the initial data in the BO pipeline. +"""Initial dataset for Bayesian optimisation, from Halton samples of the parameter +box or from a precomputed grid, saved as `init.csv` in the study output. """ from __future__ import annotations @@ -241,9 +237,6 @@ def sample_from_bounds( n_workers = os.cpu_count() - 1 log.warning(f'Number of workers reduced to {n_workers}') - # Build the PROTEUS-based objective function with fixed context - # This will be used to evaluate the objective function to provide initial samples - # Determine problem dimension (number of parameters) dims = len(params) @@ -256,20 +249,11 @@ def sample_from_bounds( failure_codes=failure_codes, ) - # Generate n random points in [0,1]^d and evaluate the objective - # Each of the parameters are evaluated in space 0-1, normalised to the bounds - # This variable is 2D, with shape [nsamp, dims] - + # Halton points in [0, 1]^d, shape [nsamp, dims], each axis normalised to its bounds sampler = Halton(d=dims, rng=np.random.default_rng(seed), scramble=True) - X = sampler.random(n=nsamp) - X = torch.tensor(X, dtype=dtype) - - # X = torch.rand(nsamp, dims, - # generator=torch.manual_seed(seed), dtype=dtype) - - # Evaluate the objective function for each of the samples - # This variable is 1D, with shape [nsamp] + X = torch.tensor(sampler.random(n=nsamp), dtype=dtype) + # Evaluate the objective at each sample, in parallel aug_args = [(x[None, :], i, builder_args) for i, x in enumerate(X)] t0 = time.perf_counter() @@ -285,8 +269,6 @@ def sample_from_bounds( Y = torch.vstack(results) - # Y = torch.stack([f(x[None, :]) for x in X]).reshape(nsamp, 1) - log.info(f'Generated initial dataset with {nsamp} points in {dims}-dim space') # Save dataset for use in BO pipeline diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index cbabc0043..59b8f5e2f 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -281,9 +281,7 @@ def run_inference(config): log.info(f'This took: {t_1 - t_0:.2f} seconds') log.info('-----------------------------------') - # Account for the simulations that did not produce a usable result. Runs - # before the best-fit summary, so the reader sees how much of the study was - # real before reading what it concluded, and so the breakdown is still + # Before the best-fit summary, so the breakdown is read first, and is still # reported when every evaluation failed and the summary refuses to print. summarise_failures(dirs['output'], len(D_final['X'])) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 20acdf078..b0106204e 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -3,7 +3,6 @@ import logging import os import subprocess -from functools import partial from pathlib import Path import pandas as pd @@ -34,11 +33,9 @@ ALWAYS_EXCLUDED_STATUSES = frozenset({29}) log = logging.getLogger('fwl.' + __name__) -# Per-child PROTEUS run timeout for inference workers. A single wedged child -# run would otherwise hang the whole batch with no diagnostic. Tunable per -# study via the inference config field `child_timeout_s`. The value is plumbed -# to the worker processes (which may be spawned, and so do not inherit module -# state) through the environment. A value of 0 or below disables the timeout. +# Per-child run timeout, so one wedged run cannot hang the batch. Set per study +# by `child_timeout_s` (0 or below disables it) and passed to the workers, +# which may be spawned, through the environment. DEFAULT_CHILD_TIMEOUT_S = 6 * 3600.0 _CHILD_TIMEOUT_ENV = 'PROTEUS_INFERENCE_CHILD_TIMEOUT_S' @@ -214,8 +211,8 @@ def run_proteus( # Ensure output directory exists out_abs.mkdir(parents=True, exist_ok=True) - # Swept parameter values only. Add `updates` as a copy for the failure report - # so that the original `parameters` dict is unchanged. + # Swept values for the failure report. The fixed entries go into a copy, so + # the caller's dict is left unchanged. swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} updates = dict(parameters) @@ -300,13 +297,7 @@ def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: # sample, not a crash of the study. try: df_row = dict(pd.read_csv(out_csv, delimiter=r'\s+').iloc[-1]) - except ( - FileNotFoundError, - OSError, - pd.errors.EmptyDataError, - pd.errors.ParserError, - IndexError, - ) as err: + except (OSError, pd.errors.EmptyDataError, pd.errors.ParserError, IndexError) as err: # A truncated whitespace-delimited file usually presents as a ragged # row (ParserError) rather than an empty one, so both are caught. raise _failure( @@ -435,23 +426,11 @@ def J( output=output, ) except ProteusRunFailure as failure: - # A parameter combination the simulator cannot integrate is an - # expected outcome of sweeping a wide box, so it is scored as a poor - # sample and the study continues. Every such run is reported once, - # and the full report goes to the failure record. - # Recorded before the abort check, so an aborted study still leaves - # the record of what stopped it. - record_failure(get_proteus_directories(output)['output'], failure) - if os.environ.get(ABORT_ON_FAILURE_ENV, '0') == '1': - raise - log.warning(failure.summary()) - log.debug(failure.report()) - return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) + # Parameters the simulator cannot integrate are expected in a wide box. + return _handle_unscored(failure, output) - # Runs that exit cleanly but stop in an error state, such as a run halted - # through its keepalive file (status 25), or that never reach the main loop - # (status 0 and 1). An unreadable status counts here too: the run's own - # account of itself is missing, so its output cannot be trusted. + # A clean exit on an error status (e.g. 25, stopped via the keepalive file), + # one that never reached the main loop (0, 1), or with no readable status. failed = (20 <= sim_status <= 28) or (sim_status in (0, 1, STATUS_MISSING)) # Runs that completed normally on an outcome this study does not fit @@ -460,12 +439,8 @@ def J( sim_status in failure_codes or sim_status in ALWAYS_EXCLUDED_STATUSES ) - # Either way the evaluation carries the failure score instead of a fit - # quality, and is recorded so that the end-of-study tally covers it. if failed or excluded: _, out_abs = run_output_dir(output, worker, iter) - # Built once, so the entry left on disk and the exception raised under - # `abort_on_failure` describe the same run. failure = ProteusRunFailure( reason=( 'exited cleanly but stopped in a failure state' @@ -481,28 +456,43 @@ def J( parameters=raw, category=CATEGORY_FAILURE if failed else CATEGORY_EXCLUDED, ) - # Recorded before the abort check, so an aborted study still leaves - # the record of what stopped it. - record_failure(get_proteus_directories(output)['output'], failure) - if failed: - # A clean exit on an error status is as much a fault as a crash, - # so it honours `abort_on_failure` the same way. An excluded - # outcome never does: nothing went wrong in such a run. - if os.environ.get(ABORT_ON_FAILURE_ENV, '0') == '1': - raise failure - log.warning(failure.summary()) - else: - # Nothing went wrong in such a run, so it is reported at info - # level and, like a fault, on one line. - log.info(failure.summary()) - # The rest of the report is kept out of the study log - log.debug(failure.report()) - return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) + return _handle_unscored(failure, output) # Compute value of objective function given these results return eval_obj(sim_vals, true_observables) +def _handle_unscored(failure: ProteusRunFailure, output: str) -> torch.Tensor: + """Record an evaluation that has no fit quality, report it, and score it. + + The record is written before the abort check, so an aborted study still + leaves the record of what stopped it. Only a failure honours + `abort_on_failure`; an excluded outcome is reported at info level, since + nothing went wrong in that run. The full report goes to the debug log only. + + Parameters + ---------- + - failure (ProteusRunFailure): The failed or excluded evaluation. + - output (str): Study output folder, relative to the PROTEUS output root. + + Returns + ---------- + - torch.Tensor: The failure score, shape (1, 1). + + Raises: + ProteusRunFailure: The failure itself, when `abort_on_failure` is set. + """ + record_failure(get_proteus_directories(output)['output'], failure) + if failure.category == CATEGORY_FAILURE: + if os.environ.get(ABORT_ON_FAILURE_ENV, '0') == '1': + raise failure + log.warning(failure.summary()) + else: + log.info(failure.summary()) + log.debug(failure.report()) + return BAD_OBJ_VALUE * torch.ones((1, 1), dtype=dtype) + + def prot_builder( parameters: dict[str, list[float]], observables: dict[str, float], @@ -521,7 +511,7 @@ def prot_builder( - parameters (dict): Mapping of parameter keys to [low, high] bounds. - observables (dict): Target observable values. - worker (int): Worker identifier. - - iter (int): Iteration number (seed) for reproducibility. + - iter (int): Iteration identifier within that worker. - output (str): Path to output folder relative to PROTEUS output folder. - ref_config (str): Reference TOML config path. - failure_codes (list[int]): PROTEUS status codes that complete normally but @@ -531,12 +521,10 @@ def prot_builder( ---------- - callable: Function f(x_norm) -> y_objective. """ - # Build bounds tensor for unnormalization + # Bounds as (2, d): lower bounds in row 0, upper in row 1 param_keys = list(parameters.keys()) d = len(param_keys) - bounds = torch.tensor( - [[list(parameters.values())[i][j] for i in range(d)] for j in range(2)], dtype=dtype - ) + bounds = torch.tensor(list(parameters.values()), dtype=dtype).T def f(x_norm: torch.Tensor) -> torch.Tensor: """Inference objective function accepting normalized inputs. @@ -552,9 +540,8 @@ def f(x_norm: torch.Tensor) -> torch.Tensor: # Convert normalized to raw inputs x_raw = unnormalize_parameters(x_norm, bounds, param_keys) - # Partially apply J with fixed context - J_context = partial( - J, + J_eval = J( + x_raw, parameters=param_keys, true_observables=observables, worker=worker, @@ -564,8 +551,6 @@ def f(x_norm: torch.Tensor) -> torch.Tensor: failure_codes=failure_codes, ) - J_eval = J_context(x_raw) - # Check J is finite if not torch.isfinite(J_eval).all(): x_param = {param_keys[i]: x_raw[0, i].item() for i in range(d)} diff --git a/src/proteus/inference/utils.py b/src/proteus/inference/utils.py index bb6f843a3..b673f019a 100644 --- a/src/proteus/inference/utils.py +++ b/src/proteus/inference/utils.py @@ -1,16 +1,5 @@ -"""Utilities for configuration access and optimization result reporting. - -This module provides helper functions to: - - * Extract values from nested dictionaries using dot-separated keys. - * Flatten nested dictionaries into a single-level dict with compound keys. - * Identify the best Bayesian optimization run and display true vs. - simulated observables and inferred parameter values. - -Functions: - get_nested: Retrieve a nested value by a dot-separated key path. - flatten: Flatten a nested dict into a single-level dict with dot-separated keys. - print_results: Select the best run and print its observables and parameters. +"""Helpers for inference: nested-config access, dataset files, GP kernels and +acquisition functions, and the end-of-study results report. """ from __future__ import annotations @@ -42,6 +31,7 @@ def str_time(): + """Current local time, with its timezone, for log and file headers.""" return datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %Z') @@ -172,11 +162,9 @@ def print_results(D, logs, config, output, n_init): X = D['X'] Y = D['Y'] - # Count the evaluations that were never scored on fit quality, so a study - # built mostly on those is not read as a converged result. Such a run - # scores BAD_OBJ_VALUE, whether it failed outright or completed on a - # status the study excludes; the objective value alone cannot tell the two - # apart, so the wording here covers both and the tally above splits them. + # Evaluations with no fit quality, so a study built mostly on them is not + # read as converged. The score cannot tell a failure from an excluded + # outcome, so the wording covers both; the tally above splits them. optim_Y = Y[n_init:] n_optim = len(optim_Y) n_unscored = int((optim_Y == BAD_OBJ_VALUE).sum().item()) @@ -246,10 +234,7 @@ def print_results(D, logs, config, output, n_init): log.info(' ') # Log parameter statistics - d = len(param_keys) - bounds = torch.tensor( - [[list(config['parameters'].values())[i][j] for i in range(d)] for j in range(2)] - ) + bounds = torch.tensor(list(config['parameters'].values())).T # remove intial data X_samp = np.array(unnormalize_parameters(X, bounds, param_keys), copy=None, dtype=float)[ n_init:, : @@ -348,17 +333,13 @@ def get_kernel(kernel: str, d: int) -> MaternKernel | RBFKernel: ---------- - MaternKernel | RBFKernel: Configured kernel instance. """ - if kernel == 'RBF': - kernel = get_kernel_w_prior(ard_num_dims=d, use_rbf_kernel=True) - elif kernel == 'MAT1/2': - kernel = get_kernel_w_prior(ard_num_dims=d, use_rbf_kernel=False, nu=0.5) - elif kernel == 'MAT3/2': - kernel = get_kernel_w_prior(ard_num_dims=d, use_rbf_kernel=False, nu=1.5) - elif kernel == 'MAT5/2': - kernel = get_kernel_w_prior(ard_num_dims=d, use_rbf_kernel=False, nu=2.5) - else: + # Matern smoothness per kernel name; None selects the RBF kernel + nus = {'RBF': None, 'MAT1/2': 0.5, 'MAT3/2': 1.5, 'MAT5/2': 2.5} + if kernel not in nus: raise ValueError('Unknown kernel, choices are RBF or MAT{1/2, 3/2, 5/2}') - return kernel + return get_kernel_w_prior( + ard_num_dims=d, use_rbf_kernel=nus[kernel] is None, nu=nus[kernel] + ) def get_obs(out_csv, observables: list[str]): From 52250b3517e72312a141dc6e63425e6126cdfedd Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 13:26:26 +0200 Subject: [PATCH 58/72] shorten comments to max 3 lines --- tests/inference/test_async_bo.py | 12 +++----- tests/inference/test_bo.py | 7 ++--- tests/inference/test_bo_convergence.py | 41 ++++++++----------------- tests/inference/test_failures.py | 20 +++++------- tests/inference/test_gen_D_init.py | 19 +++++------- tests/inference/test_inference.py | 6 ++-- tests/inference/test_objective.py | 42 ++++++++++---------------- tests/inference/test_plot.py | 7 ++--- tests/inference/test_utils_branches.py | 8 ++--- 9 files changed, 58 insertions(+), 104 deletions(-) diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index e3a77ed06..70b5edf47 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -135,10 +135,8 @@ def test_parallel_process_rejects_unknown_kernel(): parameters={'a': [0.0, 1.0]}, failure_codes=[], ) - # Discrimination: the error message must surface the valid choices so - # callers can correct the misconfiguration; this guards against a - # regression that left only a bare "Unknown kernel" string with no - # remediation hint. + # Discrimination: the message names the valid choices, not just + # "Unknown kernel", so the user can correct the config. with pytest.raises(ValueError, match='RBF'): async_mod.parallel_process( objective_builder=lambda **kwargs: None, @@ -153,10 +151,8 @@ def test_parallel_process_rejects_unknown_kernel(): parameters={'a': [0.0, 1.0]}, failure_codes=[], ) - # Discrimination: the error message must surface the valid choices so - # callers can correct the misconfiguration; this guards against a - # regression that left only a bare "Unknown kernel" string with no - # remediation hint. + # Discrimination: the message names the valid choices, not just + # "Unknown kernel", so the user can correct the config. with pytest.raises(ValueError, match='RBF'): async_mod.parallel_process( objective_builder=lambda **kwargs: None, diff --git a/tests/inference/test_bo.py b/tests/inference/test_bo.py index 863c56ced..980d2ddee 100644 --- a/tests/inference/test_bo.py +++ b/tests/inference/test_bo.py @@ -460,10 +460,9 @@ def test_bo_step_identifies_busy_points_by_worker_id_not_position(monkeypatch): 'X': torch.tensor([[0.1]], dtype=torch.double), 'Y': torch.tensor([[1.0]], dtype=torch.double), } - # Worker 1 has stopped and released its point. Worker 2 is still running, - # and is the caller here: its own claim must be excluded, the others kept. - # Two other workers are present so the nearest is not also the furthest, - # which a single other point would make indistinguishable. + # Worker 1 has stopped and released its point; worker 2 is the caller, so + # its own claim is excluded. Two other points keep the nearest distinct + # from the furthest. B = { 0: torch.tensor([[0.1]], dtype=torch.double), 2: torch.tensor([[0.75]], dtype=torch.double), diff --git a/tests/inference/test_bo_convergence.py b/tests/inference/test_bo_convergence.py index 41991bc26..6c65b315a 100644 --- a/tests/inference/test_bo_convergence.py +++ b/tests/inference/test_bo_convergence.py @@ -41,9 +41,6 @@ pytestmark = [pytest.mark.slow, pytest.mark.timeout(3600)] -# --------------------------------------------------------------------------- -# Synthetic objectives -# --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # BO loop helper # --------------------------------------------------------------------------- @@ -64,10 +61,8 @@ def _run_bo_loop( """ assert d >= 2, 'use d >= 2 to avoid the d==1 plot side effect inside BO_step' - # Seed every RNG that botorch / scipy / Python could touch. torch alone - # is not enough: scipy.optimize internals and any numpy fall-back path - # in the acquisition optimiser would otherwise leave residual non- - # determinism that flakes the seed-determinism tests across hosts. + # Seed every RNG botorch or scipy could touch. torch alone leaves + # scipy.optimize and numpy fall-backs non-deterministic across hosts. torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) @@ -77,12 +72,9 @@ def _run_bo_loop( Y = torch.stack([objective(X[i : i + 1]) for i in range(n_init)]).reshape(-1, 1) D = {'X': X, 'Y': Y} - # BO_step computes the cdist between this worker's candidate and the - # OTHER workers' busy points; with only one entry the post-mask tensor - # is empty and torch.min raises. Use two workers (the test acts as - # worker 0; worker 1 is an inert placeholder so the cdist branch has - # something to compute against) to match how the production loop - # invokes BO_step. + # The test acts as worker 0. Worker 1 is an inert placeholder, so the + # distance to other workers' busy points has something to compute against, + # as in the production loop. B = { 0: torch.zeros((1, d), dtype=torch.double), 1: 0.5 * torch.ones((1, d), dtype=torch.double), @@ -215,11 +207,9 @@ def test_bo_converges_with_each_acquisition_function(acqf): f'Acquisition {acqf!r} did not improve the best Y by 30%. ' f'Y_init_best={Y_init_best:.4f}, Y_final_best={Y_final_best:.4f}.' ) - # Discrimination: the best-x must approach the target geometrically, not - # just lower its Y by exploiting some pathological GP fit. A regression - # that returned constant `(0.5, 0.5)` for every acqf would still satisfy - # the gap check on this centered target; the proximity bound is what - # catches that mode. + # Discrimination: best-x must approach the target in x, not only in Y. A + # constant (0.5, 0.5) for every acqf would pass the gap check on this + # centred target; the proximity bound catches it. initial_min_dist = _initial_max_distance(X[:4], target) final_min_dist = torch.min(torch.norm(X - target, dim=1)).item() assert final_min_dist < initial_min_dist, ( @@ -271,10 +261,9 @@ def test_bo_different_seeds_produce_different_trajectories(): 'BO loop produced identical initial X under different seeds; the ' 'random-seed plumbing is broken or seeds are silently overridden.' ) - # Discrimination: the BO-selected candidates (rows beyond the initial - # sample) must also diverge. A regression that re-seeded inside the - # acquisition optimiser would let the initial-X check pass while still - # collapsing the BO trajectory to a seed-independent path. + # Discrimination: the BO-selected rows must diverge too. Re-seeding inside + # the acquisition optimiser would pass the initial-X check but collapse + # the trajectory to a seed-independent path. assert not torch.allclose(X1[3:], X2[3:], rtol=1e-3), ( 'BO post-init trajectory is seed-independent; the acquisition ' 'optimiser is overriding the test seed.' @@ -294,12 +283,8 @@ def test_bo_step_rejects_unknown_acquisition(): objective = make_quadratic_objective(target) with pytest.raises(ValueError, match=r'Unsupported acquisition function: not-a-real-acqf'): _run_bo_loop(objective, d=2, n_init=3, n_iter=1, acqf='not-a-real-acqf', seed=4) - # Discrimination: a known-good acqf in the same harness must complete - # without raising AND grow the dataset by exactly one iteration. - # Without this paired call, a regression that raised - # `ValueError('Unsupported acquisition function')` for EVERY acqf would - # still pass the test above; without the shape pin, a regression that - # returned an empty X tensor would also pass. + # Discrimination: a known-good acqf completes and adds exactly one row, + # ruling out a harness that rejects every acqf or returns an empty X. X_ok, _ = _run_bo_loop(objective, d=2, n_init=3, n_iter=1, acqf='LogEI', seed=4) assert X_ok.shape == (4, 2) diff --git a/tests/inference/test_failures.py b/tests/inference/test_failures.py index ee69ad992..fc4df4247 100644 --- a/tests/inference/test_failures.py +++ b/tests/inference/test_failures.py @@ -131,10 +131,9 @@ def test_proteus_run_failure_survives_the_trip_back_from_a_pool_worker(): assert restored.console_path == original.console_path assert restored.category == failures_mod.CATEGORY_FAILURE - # The category rides along in the same tuple, and it decides whether the - # parent calls the run a fault. A field dropped from the reconstruction - # would fall back to the 'failure' default and go unnoticed on a failure, - # so the round trip is checked on the other value too. + # The category decides whether the parent calls the run a fault. A dropped + # field would fall back to the 'failure' default unnoticed, so the round + # trip is checked on the other value too. excluded = failures_mod.ProteusRunFailure( reason='completed on a status this study excludes', worker=2, @@ -216,10 +215,8 @@ def test_failure_records_round_trip_into_one_table(tmp_path): assert reread['reason'] == 'the simulator exited with an error, code 3' assert reread['planet.mass_tot'] == pytest.approx(5.0) - # Edge case: a table that cannot be parsed is reported and treated as empty - # rather than aborting the summary it feeds. A zero-length file is the - # reachable form of this: a worker killed between creating the table and - # writing its first row leaves exactly that behind. + # Edge case: an unparseable table is treated as empty, not an abort. A + # zero-length file is what a worker killed before its first row leaves. table.write_text('') assert failures_mod.read_failure_records(tmp_path) == [] @@ -344,10 +341,9 @@ def test_summarise_failures_tabulates_causes_and_warns_on_every_real_failure(tmp # Grouped by cause, so two runs that died the same way count as one line. assert 'Interior model' in messages - # 3 of 20 is 15%, well under the half-the-study line the old threshold drew, - # and it is raised to a warning anyway: the count is what the reader weighs. - # One record carries it, so the level changes rather than a second line - # repeating the counts the report already gave. + # 3 of 20 is 15%, well under half the study, and still a warning: the + # count is what the reader weighs. One record carries it, not a second + # line repeating the counts. warnings = [r for r in caplog.records if r.levelname == 'WARNING'] assert len(warnings) == 1 # unscored, attempted, percent, failed, excluded. diff --git a/tests/inference/test_gen_D_init.py b/tests/inference/test_gen_D_init.py index 3ec95479a..9948a0a1a 100644 --- a/tests/inference/test_gen_D_init.py +++ b/tests/inference/test_gen_D_init.py @@ -118,10 +118,9 @@ def fake_sample_from_grid(output, params, observables, grid_dir): } assert init_mod.create_init(config) == 6 - # The grid dir is resolved through get_proteus_directories(grid)['output'], - # i.e. the (possibly relocated) output root joined with the grid name, not a - # hard-coded /output. Pinning the full path discriminates a - # regression that ignored the grid name or reintroduced the hard-coded root. + # The grid dir is the (possibly relocated) output root joined with the grid + # name. The full path rules out ignoring the grid name or a hard-coded + # /output root. assert observed['grid_dir'] == str(tmp_path / 'output' / 'my_grid') @@ -179,10 +178,8 @@ def test_sample_from_bounds_rejects_invalid_worker_count(): n_workers=0, failure_codes=[], ) - # Discrimination: negative worker counts must also raise. A regression - # that only guarded the n_workers==0 boundary (e.g. `if n == 0`) would - # let -1 slip through and crash multiprocessing.Pool with an opaque - # error far from the user's misconfiguration. + # Discrimination: negative counts raise too. A guard on n_workers == 0 + # alone would let -1 reach multiprocessing.Pool and fail opaquely there. with pytest.raises(ValueError, match='at least 1'): init_mod.sample_from_bounds( output='out', @@ -194,10 +191,8 @@ def test_sample_from_bounds_rejects_invalid_worker_count(): n_workers=-1, failure_codes=[], ) - # Discrimination: negative worker counts must also raise. A regression - # that only guarded the n_workers==0 boundary (e.g. `if n == 0`) would - # let -1 slip through and crash multiprocessing.Pool with an opaque - # error far from the user's misconfiguration. + # Discrimination: negative counts raise too. A guard on n_workers == 0 + # alone would let -1 reach multiprocessing.Pool and fail opaquely there. with pytest.raises(ValueError, match='at least 1'): init_mod.sample_from_bounds( output='out', diff --git a/tests/inference/test_inference.py b/tests/inference/test_inference.py index d96893eb4..24be5fb06 100644 --- a/tests/inference/test_inference.py +++ b/tests/inference/test_inference.py @@ -69,10 +69,8 @@ def test_run_inference_rejects_too_many_workers(monkeypatch, tmp_path): ) with pytest.raises(RuntimeError, match='Not enough CPU cores'): inference_mod.run_inference(config) - # Discrimination: the CPU-count guard must fire before any expensive - # initial-design dispatch. A regression that allowed init sampling to - # start and only raised at parallel_process would have a non-empty - # call list here. + # Discrimination: the CPU-count guard fires before initial sampling. A + # guard that raised only at parallel_process would leave calls here. assert create_init_calls == [] diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index b1a743bd9..f98d664fb 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -192,10 +192,8 @@ def _fake_run(*args, **kwargs): ref_config='reference.toml', output='dummy_output', ) - # Cause-preservation guard: the original FileNotFoundError must be - # chained via __cause__. A regression that swallowed the cause and - # raised a bare RuntimeError would still match the 'command not found' - # text but lose the traceback the operator needs. + # Cause guard: the FileNotFoundError is chained via __cause__. A bare + # RuntimeError would match the text but lose the traceback. assert isinstance(excinfo.value.__cause__, FileNotFoundError) # Side-effect guard: subprocess.run must have been invoked exactly # once. A regression that short-circuited before dispatch would @@ -300,10 +298,8 @@ def test_run_proteus_raises_on_missing_observable(monkeypatch, tmp_path): ref_config='reference.toml', output='dummy_output', ) - # Identity guard: the raised KeyError must name the offending - # observable explicitly. A regression that emitted a generic - # 'Requested observable not found' without the field name would - # match the regex above but lose the diagnostic information. + # Identity guard: the KeyError names the missing observable. A generic + # message would match the regex above but not say which field. assert 'not_present' in str(excinfo.value) # Discrimination: a valid observable on the same helpfile must # complete normally. This rules out a regression that hard-raises @@ -339,20 +335,17 @@ def test_eval_obj_mixes_log_and_linear_variables(monkeypatch): expected_sq = ((1.0 - (-6.0 / -5.0)) ** 2) + ((1.0 - 2.0 / 1.0) ** 2) expected = -torch.log10(torch.tensor([[expected_sq + 1e-10]], dtype=torch.double)) assert value.item() == pytest.approx(expected.item()) - # Discrimination guard: a regression that treated P_surf as linear - # (1e-6 vs 1e-5: relative residual 0.9) would land at a very - # different objective than the log-mode (-6/-5 = 1.2: residual - # 0.04). Pin the magnitude with a wrong-mode counter-value. + # Discrimination: treating P_surf as linear (1e-6 vs 1e-5, residual 0.9) + # gives a very different objective from log mode (-6/-5 = 1.2, residual + # 0.04). Pin against that wrong-mode value. sim_lin = {'P_surf': 1e-6, 'R_obs': 2.0} expected_sq_wrong = ((1.0 - 1e-6 / 1e-5) ** 2) + ((1.0 - 2.0 / 1.0) ** 2) expected_wrong = -torch.log10( torch.tensor([[expected_sq_wrong + 1e-10]], dtype=torch.double) ) assert abs(value.item() - expected_wrong.item()) > 0.1 - # Sign / boundedness guard: the objective is -log10(sum_sq + 1e-10). - # With sum_sq > 0 (mismatched sim vs tru), the inner argument - # exceeds 1e-10 and the result is finite. A regression that - # produced NaN or inf would fail an isfinite check. + # Boundedness guard: the objective is -log10(sum_sq + 1e-10), finite for + # any sum_sq >= 0, so NaN or inf means a regression. assert torch.isfinite(value).all() # Identical sim == tru produces sum_sq = 0, hence -log10(1e-10) = 10. value_match = objective_mod.eval_obj(sim_lin, sim_lin) @@ -781,10 +774,9 @@ def _run(status, worker, iter, codes=()): # on the child having exited non-zero. assert caught.value.exit_code == 0 - # Boundary of the failure set: STATUS_MISSING is the lowest code treated - # as a fault, and the run's own account of itself is absent, so it cannot - # be scored. A range check written as `20 <= status <= 28` alone would - # miss it. + # Boundary of the failure set: STATUS_MISSING is a fault because the run's + # own account of itself is absent. A check on `20 <= status <= 28` alone + # would miss it. with pytest.raises(objective_mod.ProteusRunFailure) as missing: _run(objective_mod.STATUS_MISSING, worker=0, iter=1) assert missing.value.status == objective_mod.STATUS_MISSING @@ -816,8 +808,7 @@ def _run(status, worker, iter, codes=()): @pytest.mark.unit def test_J_treats_the_documented_error_codes_as_failures(monkeypatch, tmp_path): - """The failure range covers the error statuses the simulator can record. - """ + """The failure range covers the error statuses the simulator can record.""" monkeypatch.setenv(failures_mod.ABORT_ON_FAILURE_ENV, '0') monkeypatch.setattr( objective_mod, 'get_proteus_directories', lambda _path: {'output': str(tmp_path)} @@ -1033,10 +1024,9 @@ def _fake_run(command, **_kwargs): assert calls[1]['atmos_clim']['spectral_cache'].endswith('spectral_cache') assert calls[1]['planet']['mass_tot'] == pytest.approx(2.5) - # One header and one row per unscored run, all the same width: the fixed - # columns plus the single swept parameter, and none of the fixed entries. - # Parsed as CSV rather than split on commas: status descriptions such as - # the one for status 25 contain a comma and are written quoted. + # One header and one row per unscored run, all the fixed columns plus the + # swept parameter. Parsed as CSV, since the status 25 description holds a + # quoted comma. with open(tmp_path / 'study' / failures_mod.FAILURE_CSV, newline='') as f: table = list(csv.reader(f)) header = table[0] diff --git a/tests/inference/test_plot.py b/tests/inference/test_plot.py index a55ad56b8..d62d87f96 100644 --- a/tests/inference/test_plot.py +++ b/tests/inference/test_plot.py @@ -744,11 +744,8 @@ def _axs_lookup(key): # Legend lands on axs[0, 0] in the 2D branch. cells[(0, 0)].legend.assert_called_once() fig.savefig.assert_called_once() - # Outer rim x-labels go on axs[-1, i]: the test grid is 2x2 so [-1, 0] - # and [-1, 1] both get set_xlabel. With our mock, [-1, 0] resolves - # the same way [1, 0] does only if __getitem__ supports negative keys. - # Skip that exact assertion and instead pin: at least 4 set_xticklabels - # calls happened across the bottom row and right column hiders. + # The mock does not resolve negative keys like axs[-1, i], so instead of + # the outer-rim labels, pin the set_xticklabels calls across the grid. total_xtick_hide = sum(c.set_xticklabels.call_count for c in cells.values()) # Top row hides x-tick labels: j=0 for both i=0 and i=1. assert total_xtick_hide >= 2 diff --git a/tests/inference/test_utils_branches.py b/tests/inference/test_utils_branches.py index 2ada8a980..0f8f9eceb 100644 --- a/tests/inference/test_utils_branches.py +++ b/tests/inference/test_utils_branches.py @@ -171,10 +171,8 @@ def test_print_results_returns_best_input_toml_path_and_logs_summary(tmp_path, c """ from proteus.inference.utils import print_results - # Build a workspace with three runs: indices 0..2. Mark the first as - # the initial guess (n_init=1) so only entries 1 and 2 are eligible - # for the best-case selection. Entry 1 carries the largest Y and - # therefore must be picked. + # Three runs; the first is the initial guess (n_init=1), so only entries + # 1 and 2 are eligible. Entry 1 has the largest Y and must be picked. _make_worker_dir(tmp_path, worker=0, iteration=0, obs_value=0.1, param_value=0.5) best_dir = _make_worker_dir(tmp_path, worker=0, iteration=1, obs_value=0.9, param_value=1.0) _make_worker_dir(tmp_path, worker=0, iteration=2, obs_value=0.5, param_value=1.5) @@ -291,7 +289,7 @@ def test_get_kernel_raises_for_unknown_kernel_name(): with pytest.raises(ValueError, match='Unknown kernel'): get_kernel('', d=2) - # Edge: case-sensitive — 'rbf' is not 'RBF'. + # Edge: case-sensitive, 'rbf' is not 'RBF'. with pytest.raises(ValueError, match='Unknown kernel'): get_kernel('rbf', d=2) From 3ad70ef1c4578629fd7550fcd768f91c77e1e899 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 13:40:39 +0200 Subject: [PATCH 59/72] Swap BaseException with Exception --- tests/inference/test_async_bo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/inference/test_async_bo.py b/tests/inference/test_async_bo.py index 70b5edf47..48d2f9ea3 100644 --- a/tests/inference/test_async_bo.py +++ b/tests/inference/test_async_bo.py @@ -875,7 +875,7 @@ def start(self): try: self.target(*self.args) self.exitcode = 0 - except BaseException: + except Exception: self.exitcode = 1 def join(self): From f3ddaf3732870b03deb0542628ab8d9f65b36822 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 13:42:17 +0200 Subject: [PATCH 60/72] ruff fix --- src/proteus/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/proteus/cli.py b/src/proteus/cli.py index d2e1b02f1..39df72fe4 100644 --- a/src/proteus/cli.py +++ b/src/proteus/cli.py @@ -57,7 +57,6 @@ def _should_apply_deterministic(argv, environ) -> bool: else: os.execvp(sys.argv[0], sys.argv) -import logging # noqa: E402 import shutil # noqa: E402 import subprocess # noqa: E402 import tempfile # noqa: E402 From 426fc2100c832c04f995a24fe338c030df0d3d03 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:14:43 +0200 Subject: [PATCH 61/72] allow for rayleigh scattering and aerosols in spectral cache --- src/proteus/atmos_clim/agni.py | 17 +++++++++------- src/proteus/atmos_clim/spectral_cache.py | 26 +++++++++++++++++++++--- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index 96b216102..6f7df16d3 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -497,6 +497,14 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # yet hold, so it can be stored once the build is known to have succeeded. cache_store_key = None + # Aerosol species dictionary (set MMR to zero initially). Determined before + # the spectral file, whose cache key depends on it. + aerosol_species = {} + if config.atmos_clim.aerosols_enabled: + aerosol_species = {a: 0.0 for a in _determine_aerosols(dirs)} + if len(aerosol_species) == 0: + log.warning('No data found for aerosol species') + # Spectral file path provided? if config.atmos_clim.agni.spectral_file is not None: # Grey gas? @@ -548,6 +556,8 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): sflux_path, config.atmos_clim.spectral_group, config.atmos_clim.spectral_bands, + rayleigh=config.atmos_clim.rayleigh, + aerosols=list(aerosol_species) if config.atmos_clim.aerosols_enabled else None, ) if seed_from_cache(config.atmos_clim.spectral_cache, key, io_dir): log.debug('Reusing prepared spectral file from cache') @@ -587,13 +597,6 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): p_top = config.atmos_clim.p_top p_surf = max(p_surf, p_top * 1.1) # this will happen if the atmosphere is stripped - # Aerosol species dictionary (set MMR to zero initially) - aerosol_species = {} - if config.atmos_clim.aerosols_enabled: - aerosol_species = {a: 0.0 for a in _determine_aerosols(dirs)} - if len(aerosol_species) == 0: - log.warning('No data found for aerosol species') - # Build the AGNI setup! kwargs. The ``aerosol_species`` parameter is # only present on newer AGNI installs; if the installed AGNI predates # that addition, sending the kwarg raises a Julia MethodError. Detect diff --git a/src/proteus/atmos_clim/spectral_cache.py b/src/proteus/atmos_clim/spectral_cache.py index babc91331..89bd930eb 100644 --- a/src/proteus/atmos_clim/spectral_cache.py +++ b/src/proteus/atmos_clim/spectral_cache.py @@ -1,8 +1,8 @@ """Reuse of prepared spectral files across runs that share a stellar spectrum. A run's `runtime.sf` is the base spectral file from FWL_DATA with that run's -stellar spectrum inserted. This module keeps one copy per distinct input set so the -second and later runs copy it instead. +stellar spectrum inserted, plus Rayleigh and aerosol blocks when enabled. This +module keeps one copy per distinct input set so later runs copy it instead. The cache is seeded into the run's output folder as `runtime.sf`, which is where the atmosphere wrapper already expects to manage it. @@ -14,6 +14,7 @@ import logging import os import shutil +from collections.abc import Sequence from pathlib import Path log = logging.getLogger('fwl.' + __name__) @@ -36,15 +37,30 @@ def _file_digest(path: Path) -> str: return digest.hexdigest() -def cache_key(base_sf: Path | str, star_spectrum: Path | str, group: str, bands: str) -> str: +def cache_key( + base_sf: Path | str, + star_spectrum: Path | str, + group: str, + bands: str, + *, + rayleigh: bool, + aerosols: Sequence[str] | None, +) -> str: """Name the cache entry for a prepared spectral file. The stellar spectrum is hashed by content. + AGNI also writes Rayleigh scattering and aerosol blocks into the prepared + file, so both settings are part of the key. They are keyword-only and have + no default, so a caller cannot leave them out and reuse the wrong file. + Parameters ---------- - base_sf (Path | str): Base spectral file, from FWL_DATA. - star_spectrum (Path | str): Stellar spectrum (`.sflux`) to be inserted. - group (str): Spectral file group. - bands (str): Number of wavenumber bands. + - rayleigh (bool): Whether Rayleigh scattering is included. + - aerosols (Sequence[str] | None): Aerosol species included, or None when + aerosols are disabled. Order does not matter. Returns ---------- @@ -59,6 +75,10 @@ def cache_key(base_sf: Path | str, star_spectrum: Path | str, group: str, bands: str(stat.st_size), str(int(stat.st_mtime)), _file_digest(Path(star_spectrum)), + f'rayleigh={bool(rayleigh)}', + # Disabled and enabled with no species are distinct: AGNI still sets + # the aerosol flag when inserting blocks in the second case. + 'aerosols=off' if aerosols is None else 'aerosols=' + ','.join(sorted(aerosols)), ) return hashlib.sha256('\0'.join(parts).encode()).hexdigest()[:32] From 5652ac2d3774d92e8ca08c7191e54a339f549261 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:15:07 +0200 Subject: [PATCH 62/72] update tests for spectral cache with rayleigh and aerosol settings --- tests/atmos_clim/test_agni.py | 35 ++++++++++++++- tests/atmos_clim/test_spectral_cache.py | 57 +++++++++++++++++++++---- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/tests/atmos_clim/test_agni.py b/tests/atmos_clim/test_agni.py index cb431d2b7..97adb38cb 100644 --- a/tests/atmos_clim/test_agni.py +++ b/tests/atmos_clim/test_agni.py @@ -535,7 +535,7 @@ def test_spectral_cache_is_filled_from_the_folder_agni_wrote_in( built_in = ctx.scratch if work_dir == 'scratch' else ctx.output_dir assert (built_in / 'runtime.sf').is_file() - key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16') + key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=None) assert sorted(p.name for p in cache.iterdir()) == [f'{key}.sf', f'{key}.sf_k'] assert (cache / f'{key}.sf').read_text() == (built_in / 'runtime.sf').read_text() assert (cache / f'{key}.sf_k').read_text() == (built_in / 'runtime.sf_k').read_text() @@ -560,7 +560,7 @@ def test_a_cached_spectral_file_is_reused_without_reinserting_the_spectrum( cache.mkdir() ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) - key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16') + key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=None) (cache / f'{key}.sf').write_text('cached prepared file', encoding='utf-8') (cache / f'{key}.sf_k').write_text('cached ktable', encoding='utf-8') @@ -580,6 +580,37 @@ def test_a_cached_spectral_file_is_reused_without_reinserting_the_spectrum( assert not (ctx.output_dir / 'runtime.sf').exists() +@pytest.mark.unit +def test_a_cached_spectral_file_is_not_reused_across_rayleigh_settings(monkeypatch, tmp_path): + """A cache folder shared by runs that differ only in `atmos_clim.rayleigh` + must not hand one the other's file, because AGNI writes the Rayleigh block + into it. The entry built without Rayleigh scattering is left alone, and the + run with it builds and stores its own. + """ + cache = tmp_path / 'cache' + cache.mkdir() + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) + without = cache_key( + ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=None + ) + (cache / f'{without}.sf').write_text('built without rayleigh', encoding='utf-8') + (cache / f'{without}.sf_k').write_text('ktable without rayleigh', encoding='utf-8') + ctx.config.atmos_clim.rayleigh = True + + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) is not None + + # A miss: the spectrum was handed to AGNI for insertion, not a seeded file. + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + assert (ctx.scratch / 'runtime.sf').read_text() != 'built without rayleigh' + with_rayleigh = cache_key( + ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=True, aerosols=None + ) + assert sorted(p.name for p in cache.iterdir()) == sorted( + [f'{without}.sf', f'{without}.sf_k', f'{with_rayleigh}.sf', f'{with_rayleigh}.sf_k'] + ) + assert (cache / f'{without}.sf').read_text() == 'built without rayleigh' + + @pytest.mark.unit def test_a_cache_that_cannot_be_written_costs_time_and_not_correctness( monkeypatch, tmp_path, caplog diff --git a/tests/atmos_clim/test_spectral_cache.py b/tests/atmos_clim/test_spectral_cache.py index 27c1e3d5a..ab111c1fe 100644 --- a/tests/atmos_clim/test_spectral_cache.py +++ b/tests/atmos_clim/test_spectral_cache.py @@ -23,6 +23,10 @@ pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] +# Scattering settings for tests about the other key inputs: Rayleigh on, the +# config default, and aerosols disabled. +_PLAIN = {'rayleigh': True, 'aerosols': None} + def _make_inputs(tmp_path, star_bytes=b'1.0 2.0\n3.0 4.0\n'): """Write a base spectral file and a stellar spectrum, and return both.""" @@ -48,21 +52,58 @@ def test_key_tracks_the_stellar_spectrum_and_the_spectral_resolution(tmp_path): which config fields those are. """ base, star = _make_inputs(tmp_path) - key = cache_key(base, star, 'Honeyside', '48') + key = cache_key(base, star, 'Honeyside', '48', **_PLAIN) # Same inputs, same entry: this is what lets a study reuse one build. - assert cache_key(base, star, 'Honeyside', '48') == key + assert cache_key(base, star, 'Honeyside', '48', **_PLAIN) == key # A different stellar spectrum is a different file, even byte-for-byte the # same length, so a content hash rather than a size check is required. other_star = tmp_path / 'other.sflux' other_star.write_bytes(b'9.0 2.0\n3.0 4.0\n') assert other_star.stat().st_size == star.stat().st_size - assert cache_key(base, other_star, 'Honeyside', '48') != key + assert cache_key(base, other_star, 'Honeyside', '48', **_PLAIN) != key # Resolution and group select a different base file, so neither may collide. - assert cache_key(base, star, 'Honeyside', '256') != key - assert cache_key(base, star, 'Frostflow', '48') != key + assert cache_key(base, star, 'Honeyside', '256', **_PLAIN) != key + assert cache_key(base, star, 'Frostflow', '48', **_PLAIN) != key + + +@pytest.mark.unit +def test_key_tracks_the_scattering_blocks_agni_writes_into_the_file(tmp_path): + """AGNI writes Rayleigh and aerosol blocks into the prepared file, so runs + that differ in either setting must not share an entry. Aerosols disabled + and enabled with no species are different files, and the species list is + compared as a set, so the order the data folder lists them in is irrelevant. + """ + base, star = _make_inputs(tmp_path) + key = cache_key(base, star, 'Honeyside', '48', rayleigh=True, aerosols=None) + + assert cache_key(base, star, 'Honeyside', '48', rayleigh=False, aerosols=None) != key + with_aerosols = cache_key( + base, star, 'Honeyside', '48', rayleigh=True, aerosols=['H2SO4', 'SiO2'] + ) + assert with_aerosols != key + # Edge case: enabled with no species still sets the aerosol flag in AGNI. + no_species = cache_key(base, star, 'Honeyside', '48', rayleigh=True, aerosols=[]) + assert no_species not in (key, with_aerosols) + # A different species set is a different file; the same set in another + # order is the same file. + assert ( + cache_key(base, star, 'Honeyside', '48', rayleigh=True, aerosols=['H2SO4']) + != with_aerosols + ) + assert ( + cache_key(base, star, 'Honeyside', '48', rayleigh=True, aerosols=['SiO2', 'H2SO4']) + == with_aerosols + ) + + # Error contract: the settings have no default, so a caller cannot build a + # key that silently ignores them. + with pytest.raises(TypeError): + cache_key(base, star, 'Honeyside', '48') + with pytest.raises(TypeError): + cache_key(base, star, 'Honeyside', '48', True, None) @pytest.mark.unit @@ -72,7 +113,7 @@ def test_key_changes_when_the_base_spectral_file_is_updated(tmp_path): is large and read-only, so both are exercised here. """ base, star = _make_inputs(tmp_path) - key = cache_key(base, star, 'Honeyside', '48') + key = cache_key(base, star, 'Honeyside', '48', **_PLAIN) # Same size, newer file: the mtime component has to carry this one. stat = base.stat() @@ -81,13 +122,13 @@ def test_key_changes_when_the_base_spectral_file_is_updated(tmp_path): os.utime(base, (stat.st_atime, stat.st_mtime + 120)) assert base.stat().st_size == 4096 - key_touched = cache_key(base, star, 'Honeyside', '48') + key_touched = cache_key(base, star, 'Honeyside', '48', **_PLAIN) assert key_touched != key # Same mtime, different size: the size component has to carry this one. base.write_bytes(b'y' * 8192) os.utime(base, (stat.st_atime, stat.st_mtime + 120)) - assert cache_key(base, star, 'Honeyside', '48') != key_touched + assert cache_key(base, star, 'Honeyside', '48', **_PLAIN) != key_touched @pytest.mark.unit From 99cdfe3ca5619d7249a15bd7a6cb4b2fee8b99b7 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:48:06 +0200 Subject: [PATCH 63/72] Stop initial sampling at the first failed sample --- src/proteus/inference/gen_D_init.py | 35 +++++++++++++++++--- tests/inference/test_gen_D_init.py | 51 ++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/proteus/inference/gen_D_init.py b/src/proteus/inference/gen_D_init.py index 9eb132e13..f5d2db460 100644 --- a/src/proteus/inference/gen_D_init.py +++ b/src/proteus/inference/gen_D_init.py @@ -7,7 +7,7 @@ import logging import os import time -from multiprocessing import Pool +from multiprocessing import Event, Pool from pathlib import Path import numpy as np @@ -26,6 +26,10 @@ dtype = torch.double log = logging.getLogger('fwl.' + __name__) +# Stop signal shared by the initial-sampling pool, set by the first sample that +# raises. Installed in each pool worker by `_init_pool_worker`. +_stop = None + def create_init(config): """Create the initial BO dataset using sampling or a precomputed grid. @@ -161,9 +165,20 @@ def sample_from_grid(output: str, params: dict, observables: dict, grid_dir: str return len(Y.flatten()) +def _init_pool_worker(stop) -> None: + """Give an initial-sampling pool worker the batch's shared stop signal.""" + global _stop + _stop = stop + + def f_aug(x, iter, builder_args): """Evaluate a single initial sample using a temporary objective wrapper. + Any error raised here fails the whole batch, including a failed run under + `abort_on_failure`. The first one sets the stop signal, so samples not yet + started return at once instead of running a simulation whose result would + be discarded. Runs already in progress finish. + Parameters ---------- - x (torch.Tensor): Candidate input of shape (1, d) in normalized space. @@ -172,8 +187,11 @@ def f_aug(x, iter, builder_args): Returns ---------- - - torch.Tensor: Objective value tensor with shape (1, 1). + - torch.Tensor | None: Objective value tensor with shape (1, 1), or None + for a sample skipped because the batch is stopping. """ + if _stop is not None and _stop.is_set(): + return None f = prot_builder( parameters=builder_args['parameters'], observables=builder_args['observables'], @@ -183,8 +201,12 @@ def f_aug(x, iter, builder_args): output=builder_args['output'], failure_codes=builder_args['failure_codes'], ) - - return f(x) + try: + return f(x) + except Exception: + if _stop is not None: + _stop.set() + raise def _pool_timeout(n_tasks: int, n_workers: int) -> float | None: @@ -257,7 +279,10 @@ def sample_from_bounds( aug_args = [(x[None, :], i, builder_args) for i, x in enumerate(X)] t0 = time.perf_counter() - with Pool(processes=n_workers) as pool: + # A skipped sample only occurs once another has raised, and that error is + # what `get` raises, so no None reaches the dataset. + stop = Event() + with Pool(processes=n_workers, initializer=_init_pool_worker, initargs=(stop,)) as pool: async_result = pool.starmap_async(f_aug, aug_args) # Bound the whole batch so a worker that wedges outside the per-child # subprocess timeout cannot hang the run indefinitely. Leaving the Pool diff --git a/tests/inference/test_gen_D_init.py b/tests/inference/test_gen_D_init.py index 9948a0a1a..c2d53e089 100644 --- a/tests/inference/test_gen_D_init.py +++ b/tests/inference/test_gen_D_init.py @@ -8,6 +8,8 @@ from __future__ import annotations +import threading + import numpy as np import pandas as pd import pytest @@ -21,6 +23,7 @@ pytest.importorskip('gpytorch') import proteus.inference.gen_D_init as init_mod # noqa: E402 +from proteus.inference.failures import ProteusRunFailure # noqa: E402 pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] @@ -224,8 +227,9 @@ def random(self, n): return np.array([[0.1], [0.9]])[:n] class FakePool: - def __init__(self, processes): + def __init__(self, processes, initializer=None, initargs=()): captured['processes'] = processes + captured['initializer'] = initializer def __enter__(self): return self @@ -271,6 +275,8 @@ def fake_save_dataset_csv(X, Y, fpath): assert n == 2 assert captured['processes'] == 3 assert captured['task_count'] == 2 + # Each pool worker is handed the batch's stop signal. + assert captured['initializer'] is init_mod._init_pool_worker assert captured['saved_shape'] == ((2, 1), (2, 1)) assert captured['saved_path'].endswith('init.csv') # The batch is bounded so a wedged worker cannot hang it indefinitely: @@ -279,6 +285,49 @@ def fake_save_dataset_csv(X, Y, fpath): assert captured['pool_timeout'] > 0 +@pytest.mark.unit +def test_a_failed_initial_sample_stops_the_samples_not_yet_started(monkeypatch): + """Under `abort_on_failure` the first failed run fails the whole initial + batch, so the samples still queued must not each run a simulation first. + The failing sample sets the batch's stop signal, and a sample that finds it + set returns without building an objective or running anything. + """ + stop = threading.Event() + monkeypatch.setattr(init_mod, '_stop', None) + init_mod._init_pool_worker(stop) + args = {'parameters': {'a': [0.0, 1.0]}, 'observables': {'obs': 1.0}} + args |= {'ref_config': 'ref.toml', 'output': 'out', 'failure_codes': []} + x = torch.tensor([[0.5]], dtype=torch.double) + + # Discrimination: a sample that succeeds returns its score and leaves the + # signal clear, so the stop below is the failure's doing. + score = torch.tensor([[1.25]], dtype=torch.double) + monkeypatch.setattr(init_mod, 'prot_builder', lambda **_kw: lambda _x: score) + assert init_mod.f_aug(x, 0, args).item() == pytest.approx(1.25) + assert not stop.is_set() + + def _failing(_x): + raise ProteusRunFailure(reason='r', worker=-1, iter=1, out_dir='/o', status=21) + + monkeypatch.setattr(init_mod, 'prot_builder', lambda **_kw: _failing) + with pytest.raises(ProteusRunFailure): + init_mod.f_aug(x, 1, args) + assert stop.is_set() + + # A later sample runs nothing: building its objective would fail the test. + monkeypatch.setattr( + init_mod, 'prot_builder', lambda **_kw: pytest.fail('a skipped sample ran') + ) + assert init_mod.f_aug(x, 2, args) is None + + # Edge case: called outside a pool, with no signal installed, a failure + # propagates as before and nothing is skipped. + monkeypatch.setattr(init_mod, '_stop', None) + monkeypatch.setattr(init_mod, 'prot_builder', lambda **_kw: _failing) + with pytest.raises(ProteusRunFailure): + init_mod.f_aug(x, 3, args) + + @pytest.mark.unit def test_real_halton_accepts_rng_keyword_and_is_deterministic(): """Check halton sampler accepts rng and behaves pseudo-deterministically. From a1c2b5070d43fa2bf581e36d0241e51bdabc0e3e Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:53:33 +0200 Subject: [PATCH 64/72] update docs with new failures behaviour --- docs/How-to/inference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index bdc836045..3fbd2e6ed 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -153,7 +153,7 @@ The system prints the final results including: ### Failed and excluded simulations -During the inference run, some PROTEUS simulations might crash or fail, or stop on a status that is excluded in the inference configuration (e.g. maximum runtime reached). Status 29 (planet evaporated) is always excluded. The run carries on when there are failures unless `abort_on_failure` is set to `true` in the inference config. At the end of the study all failures are written to `failures.csv` in the output folder, and summarised. +During the inference run, some PROTEUS simulations might crash or fail, or stop on a status that is excluded in the inference configuration (e.g. maximum runtime reached). Status 29 (planet evaporated) is always excluded. The run carries on when there are failures unless `abort_on_failure` is set to `true` in the inference config. The study then starts no new simulations, but those already running finish first. Each failure is added to `failures.csv` in the output folder as it happens, and all failures are summarised at the end of the study. A study stopped by `abort_on_failure` keeps its `failures.csv` but prints no summary. ## Customization From 9e70b0a350cb9f91abffe2463692355f2dfe82a1 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:54:23 +0200 Subject: [PATCH 65/72] update example toml with abort on failure info --- input/inference/example.infer.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/input/inference/example.infer.toml b/input/inference/example.infer.toml index 639f11b3d..25b5fcc78 100644 --- a/input/inference/example.infer.toml +++ b/input/inference/example.infer.toml @@ -19,8 +19,9 @@ init_grid = 'none' # grid_demo/' # Path pre-computed grid (relative to PROTEU # Completion codes to exclude from the fit. Evaporated (29) is always excluded. failure_codes = [11, ] # solidified (10), escaped (15), max_runtime (11) -# Stop the whole inference run at the first simulation that fails, instead of scoring -# it as a poor sample and carrying on. Off by default. +# Stop the whole inference run at the first simulation that fails. +# Simulations already running finish first. +# Off by default. abort_on_failure = false # Parameters for Bayesian optimisation From a6f9b5df4116de45e47091dc1ae31f8dd08fd874 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 14:58:05 +0200 Subject: [PATCH 66/72] add explanatory comment on exception --- src/proteus/atmos_clim/spectral_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/proteus/atmos_clim/spectral_cache.py b/src/proteus/atmos_clim/spectral_cache.py index 89bd930eb..cdefa668e 100644 --- a/src/proteus/atmos_clim/spectral_cache.py +++ b/src/proteus/atmos_clim/spectral_cache.py @@ -163,8 +163,8 @@ def store_in_cache(cache_dir: Path | str, key: str, out_dir: Path | str) -> bool except Exception: try: tmp.unlink(missing_ok=True) - except OSError: - pass + except OSError as cleanup_err: + log.debug(f'Could not remove temporary cache file {tmp}: {cleanup_err}') raise except OSError as err: log.warning(f'Could not store spectral file in cache: {err}') From 9ad2918fdf0d8f60936d24eb879307c99769d5ba Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 15:06:21 +0200 Subject: [PATCH 67/72] invalid UTF-8 catch added --- src/proteus/inference/objective.py | 13 ++++++++++--- tests/inference/test_objective.py | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index b0106204e..97d229b56 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -293,11 +293,18 @@ def _failure(reason: str, exit_code: int | None) -> ProteusRunFailure: status = ReadStatus(out_abs) # Read simulator output. A run that exits cleanly but writes no usable - # helpfile (killed mid-write, or stopped before the first row) is a failed - # sample, not a crash of the study. + # helpfile (killed mid-write, stopped before the first row, or corrupted + # on disk) is a failed sample, not a crash of the study. + unreadable = ( + OSError, + UnicodeDecodeError, + pd.errors.EmptyDataError, + pd.errors.ParserError, + IndexError, + ) try: df_row = dict(pd.read_csv(out_csv, delimiter=r'\s+').iloc[-1]) - except (OSError, pd.errors.EmptyDataError, pd.errors.ParserError, IndexError) as err: + except unreadable as err: # A truncated whitespace-delimited file usually presents as a ragged # row (ParserError) rather than an empty one, so both are caught. raise _failure( diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index f98d664fb..4f22fa03b 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -539,8 +539,9 @@ def test_run_proteus_failure_points_at_the_simulator_logfile(monkeypatch, tmp_pa @pytest.mark.unit def test_run_proteus_reports_a_clean_exit_that_produced_no_output(monkeypatch, tmp_path): """A run that exits zero but writes no readable helpfile is reported as a - failed sample rather than crashing the study with a bare parser error. The - exit code is recorded as zero so the report does not suggest a crash. + failed sample rather than crashing the study with a bare parser error. That + covers a missing file, an empty one, and one corrupted into invalid UTF-8. + The exit code is recorded as zero so the report does not suggest a crash. """ out_abs = tmp_path / 'sim' out_abs.mkdir(parents=True) @@ -575,9 +576,24 @@ def test_run_proteus_reports_a_clean_exit_that_produced_no_output(monkeypatch, t output='dummy_output', ) + # Edge case: bytes that are not valid UTF-8 raise UnicodeDecodeError, a + # ValueError rather than a parser error, from pandas. + (out_abs / 'runtime_helpfile.csv').write_bytes(b'Time P_surf\n1.0 2.0\n3.0 \xff\xfe\n') + with pytest.raises(objective_mod.ProteusRunFailure) as excinfo: + objective_mod.run_proteus( + parameters={}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config='reference.toml', + output='dummy_output', + ) + assert isinstance(excinfo.value.__cause__, UnicodeDecodeError) + assert 'no readable output' in excinfo.value.reason + # Discrimination: a helpfile with a usable row completes normally, so the - # two failures above come from the output and not from an unconditional - # raise on this code path. + # failures above come from the output and not from an unconditional raise + # on this code path. pd.DataFrame([{'P_surf': 2.5}]).to_csv( out_abs / 'runtime_helpfile.csv', sep=' ', index=False ) From 27c5d9bb56e8f75c70e76ca53cac4ea17635d68c Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 15:08:55 +0200 Subject: [PATCH 68/72] shorten comment --- src/proteus/utils/helper.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/proteus/utils/helper.py b/src/proteus/utils/helper.py index ee0683641..10399647d 100644 --- a/src/proteus/utils/helper.py +++ b/src/proteus/utils/helper.py @@ -356,11 +356,7 @@ def UpdateStatusfile(dirs: dict, status: int): # Status written by PROTEUS before its output folder is cleaned, and never -# rewritten until the main loop starts. A child that dies in between leaves no -# status file at all, so a missing file is reported as such rather than being -# silently reported as a generic error. The value sits outside the range of -# every status PROTEUS writes, so "wrote status 0 (Started) then died" stays -# distinguishable from "never wrote one". +# rewritten until the main loop starts. STATUS_MISSING = -1 From e9c41df51cefbf9793c0443654a7dcba946dd9eb Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 15:29:31 +0200 Subject: [PATCH 69/72] make spectral cache explicit in inference run so that a reference config copied from all_options doesnt silently turn it off --- docs/How-to/inference.md | 1 + input/inference/example.infer.toml | 4 + src/proteus/inference/inference.py | 24 +++++- src/proteus/inference/objective.py | 40 ++++++++-- tests/inference/test_inference.py | 113 +++++++++++++++++++++++++++-- tests/inference/test_objective.py | 64 ++++++++++++++++ 6 files changed, 232 insertions(+), 14 deletions(-) diff --git a/docs/How-to/inference.md b/docs/How-to/inference.md index 3fbd2e6ed..a7134e034 100644 --- a/docs/How-to/inference.md +++ b/docs/How-to/inference.md @@ -171,3 +171,4 @@ During the inference run, some PROTEUS simulations might crash or fail, or stop - Set `n_workers` to be less than your CPU core count minus 1 - The system automatically limits thread usage to prevent oversubscription - PROTEUS evaluation time typically dominates total runtime +- Workers share prepared spectral files through a cache in the inference run's output folder. Set `spectral_cache = false` in the inference config to turn it off. The study log names the cache in use. diff --git a/input/inference/example.infer.toml b/input/inference/example.infer.toml index 25b5fcc78..0c3ee9a6a 100644 --- a/input/inference/example.infer.toml +++ b/input/inference/example.infer.toml @@ -24,6 +24,10 @@ failure_codes = [11, ] # solidified (10), escaped (15), max_runtime (11) # Off by default. abort_on_failure = false +# Share prepared spectral files between workers through a cache in the output folder. +# Replaces atmos_clim.spectral_cache from the reference config. +spectral_cache = true + # Parameters for Bayesian optimisation n_workers = 5 # Number of parallel workers kernel = "MAT1/2" # Kernel type for GP, "RBF" | "MAT1/2" | "MAT3/2" | "MAT5/2" diff --git a/src/proteus/inference/inference.py b/src/proteus/inference/inference.py index 59b8f5e2f..6591bdc04 100644 --- a/src/proteus/inference/inference.py +++ b/src/proteus/inference/inference.py @@ -33,10 +33,12 @@ from proteus.inference.failures import ABORT_ON_FAILURE_ENV, summarise_failures from proteus.inference.gen_D_init import create_init from proteus.inference.objective import ( + SPECTRAL_CACHE_ENV, WORKER_CONFIG_OVERRIDES, apply_nested_updates, prot_builder, set_child_timeout, + worker_spectral_cache, ) from proteus.inference.utils import print_results, str_time from proteus.utils.coupler import get_proteus_directories, variable_is_logarithmic @@ -123,7 +125,9 @@ def parameter_bounds(parameters: dict) -> dict[str, tuple[float, float]]: return bounds -def validate_reference_config(ref_config: str, parameters: dict) -> None: +def validate_reference_config( + ref_config: str, parameters: dict, spectral_cache: bool = True +) -> None: """Reject a reference config the workers could not run, before any run starts. The file is checked exactly as PROTEUS checks its own input, and then again @@ -144,6 +148,7 @@ def validate_reference_config(ref_config: str, parameters: dict) -> None: ---------- - ref_config (str): Path to the reference PROTEUS config file. - parameters (dict): Mapping of dot-separated config keys to [min, max]. + - spectral_cache (bool): The inference config's `spectral_cache` switch. Returns ---------- @@ -165,6 +170,9 @@ def validate_reference_config(ref_config: str, parameters: dict) -> None: updates = {key: pair[index] for key, pair in bounds.items()} updates.update(WORKER_CONFIG_OVERRIDES) updates['params.out.path'] = _VALIDATION_OUT_PATH + updates['atmos_clim.spectral_cache'] = worker_spectral_cache( + _VALIDATION_OUT_PATH, spectral_cache + ) candidate = apply_nested_updates(copy.deepcopy(raw), updates) _reject_bad_config(candidate, f'{ref_config} (parameters at their {label} bounds)') @@ -199,7 +207,13 @@ def run_inference(config): if not os.path.isfile(config['ref_config']): raise FileNotFoundError('Cannot find reference config: ' + config['ref_config']) - validate_reference_config(config['ref_config'], config['parameters']) + # Whether workers share prepared spectral files, which replaces the + # reference config's own atmos_clim.spectral_cache setting. + spectral_cache = config.get('spectral_cache', True) + if not isinstance(spectral_cache, bool): + raise ValueError(f'spectral_cache must be true or false, got {spectral_cache!r}') + + validate_reference_config(config['ref_config'], config['parameters'], spectral_cache) # Create output directory safe_rm(dirs['output']) @@ -232,6 +246,12 @@ def run_inference(config): # is expected to reach combinations the simulator cannot integrate. os.environ[ABORT_ON_FAILURE_ENV] = '1' if config.get('abort_on_failure', False) else '0' + os.environ[SPECTRAL_CACHE_ENV] = '1' if spectral_cache else '0' + if spectral_cache: + log.info(f'Spectral cache: {worker_spectral_cache(config["output"], True)}') + else: + log.info('Spectral cache: off (spectral_cache = false in the inference config)') + # Default for configs that pre-date this field config.setdefault('failure_codes', []) diff --git a/src/proteus/inference/objective.py b/src/proteus/inference/objective.py index 97d229b56..d1db2b792 100644 --- a/src/proteus/inference/objective.py +++ b/src/proteus/inference/objective.py @@ -48,8 +48,10 @@ 'params.out.archive_mod': 0, } -# Folder inside the study output where workers reuse prepared spectral files. +# Folder inside the study output where workers reuse prepared spectral files, +# and the switch for it, passed to the workers through the environment. SPECTRAL_CACHE_DIR = 'spectral_cache' +SPECTRAL_CACHE_ENV = 'PROTEUS_INFERENCE_SPECTRAL_CACHE' # Config entries every run sets to the same thing, or to a value derived from # the run index. Excluded from failure reports, which name the swept values. @@ -167,6 +169,33 @@ def update_toml(config_file: str, updates: dict, output_file: str) -> None: toml.dump(config, f) +def spectral_cache_enabled() -> bool: + """Whether workers share a spectral cache, as recorded by the parent process.""" + return os.environ.get(SPECTRAL_CACHE_ENV, '1') == '1' + + +def worker_spectral_cache(output: str, enabled: bool) -> str: + """The `atmos_clim.spectral_cache` value every worker runs with. + + Set by the inference config's `spectral_cache` switch, whatever the + reference config holds: a reference config copied from `all_options.toml` + sets "none" without meaning to. Every evaluation that holds the star fixed + builds the same prepared file, so enabled workers share one folder. + + Parameters + ---------- + - output (str): Study output folder, relative to the PROTEUS output root. + - enabled (bool): The inference config's `spectral_cache` switch. + + Returns + ---------- + - str: The study's cache folder, or "none" when the cache is off. + """ + if not enabled: + return 'none' + return str(Path(get_proteus_directories(output)['output']) / SPECTRAL_CACHE_DIR) + + def run_proteus( parameters: dict, worker: int, @@ -216,13 +245,10 @@ def run_proteus( swept = {k: v for k, v in parameters.items() if k not in _FIXED_PARAMETER_KEYS} updates = dict(parameters) - # Inject output path into simulation parameters + # Inject output path and spectral cache into simulation parameters updates['params.out.path'] = str(out_dir) - - # Every evaluation of an inference run that holds the star fixed builds the same - # prepared spectral file. - updates['atmos_clim.spectral_cache'] = str( - Path(get_proteus_directories(output)['output']) / SPECTRAL_CACHE_DIR + updates['atmos_clim.spectral_cache'] = worker_spectral_cache( + output, spectral_cache_enabled() ) # Don't allow workers to make plots or logs diff --git a/tests/inference/test_inference.py b/tests/inference/test_inference.py index 24be5fb06..b6e3d8215 100644 --- a/tests/inference/test_inference.py +++ b/tests/inference/test_inference.py @@ -10,6 +10,7 @@ from __future__ import annotations import multiprocessing as mp +import os from pathlib import Path import pytest @@ -24,6 +25,14 @@ import proteus.inference.inference as inference_mod # noqa: E402 from proteus.config import UnknownConfigKeyError # noqa: E402 +from proteus.inference.failures import ABORT_ON_FAILURE_ENV # noqa: E402 +from proteus.inference.objective import ( # noqa: E402 + _CHILD_TIMEOUT_ENV as CHILD_TIMEOUT_ENV, +) +from proteus.inference.objective import ( # noqa: E402 + SPECTRAL_CACHE_DIR, + SPECTRAL_CACHE_ENV, +) pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] @@ -215,6 +224,38 @@ def test_parameter_bounds_rejects_a_log_scaled_range_that_reaches_zero(): assert inference_mod.variable_is_logarithmic('planet.elements.H_budget') is True +@pytest.mark.unit +@pytest.mark.parametrize('enabled', [True, False], ids=['cache-on', 'cache-off']) +def test_validate_reference_config_checks_the_spectral_cache_the_workers_use( + monkeypatch, tmp_path, enabled +): + """The configs checked at startup carry the spectral cache the workers will + run with, which the inference switch sets whatever the reference config + holds. The reference config here sets a path of its own to show that. + """ + ref = toml.load(BASE_CONFIG) + ref.setdefault('atmos_clim', {})['spectral_cache'] = '/my/cache' + ref_config = tmp_path / 'ref.toml' + ref_config.write_text(toml.dumps(ref), encoding='utf-8') + + checked = [] + monkeypatch.setattr( + inference_mod, '_reject_bad_config', lambda raw, label: checked.append((label, raw)) + ) + inference_mod.validate_reference_config( + str(ref_config), {'planet.mass_tot': [0.7, 3.0]}, spectral_cache=enabled + ) + + # The file as written, then both bound variants. + assert len(checked) == 3 + assert checked[0][1]['atmos_clim']['spectral_cache'] == '/my/cache' + variants = [raw['atmos_clim']['spectral_cache'] for _label, raw in checked[1:]] + if enabled: + assert all(v.endswith(SPECTRAL_CACHE_DIR) for v in variants) + else: + assert variants == ['none', 'none'] + + @pytest.mark.unit def test_validate_reference_config_accepts_a_runnable_sweep(): """A reference config that PROTEUS accepts, swept over parameters that stay @@ -305,17 +346,19 @@ def test_validate_reference_config_rejects_a_faulty_reference_file(tmp_path): @pytest.mark.unit @pytest.mark.parametrize( - ('parameters', 'error', 'match'), + ('parameters', 'extra', 'error', 'match'), [ - ({'planet.mass_tott': [0.7, 3.0]}, UnknownConfigKeyError, 'planet.mass_tott'), + ({'planet.mass_tott': [0.7, 3.0]}, {}, UnknownConfigKeyError, 'planet.mass_tott'), # Accepted by the schema, and rejected by the optimiser only once the # workers start sampling it in log10 space. - ({'planet.elements.H_budget': [0.0, 2e4]}, ValueError, 'log scale'), + ({'planet.elements.H_budget': [0.0, 2e4]}, {}, ValueError, 'log scale'), + # A quoted "false" is truthy, so it would silently leave the cache on. + ({'planet.mass_tot': [0.7, 3.0]}, {'spectral_cache': 'false'}, ValueError, 'true or'), ], - ids=['misspelt_parameter', 'log_scaled_range_reaching_zero'], + ids=['misspelt_parameter', 'log_scaled_range_reaching_zero', 'spectral_cache_not_bool'], ) def test_run_inference_validates_reference_config_before_emptying_output( - monkeypatch, tmp_path, parameters, error, match + monkeypatch, tmp_path, parameters, extra, error, match ): """``run_inference`` validates the reference config before it empties the study output folder and before it generates any initial design. Re-running @@ -334,6 +377,7 @@ def test_run_inference_validates_reference_config_before_emptying_output( 'seed': 1, 'observables': {'P_surf': 1.0}, 'parameters': parameters, + **extra, } # Stand in for a completed earlier study occupying the same output folder. output_root = tmp_path / 'output' @@ -368,6 +412,65 @@ def test_run_inference_validates_reference_config_before_emptying_output( assert not (output_root / 'ref_config.toml').exists() +class _StopAfterSetup(Exception): + """Raised in place of the initial design, once startup has finished.""" + + +@pytest.mark.unit +@pytest.mark.parametrize('switch', [None, True, False], ids=['default', 'on', 'off']) +def test_run_inference_reports_the_spectral_cache_the_study_uses( + monkeypatch, tmp_path, caplog, switch +): + """The study log names the spectral cache the workers will use, so a study + running without one is visible. The cache is on unless the inference config + turns it off, and the setting reaches the workers through the environment. + """ + config = { + 'output': 'unit_inference', + 'logging': 'INFO', + 'n_workers': 1, + 'ref_config': BASE_CONFIG, + 'n_steps': 1, + 'kernel': 'MAT3/2', + 'acqf': 'LogEI', + 'seed': 1, + 'observables': {'P_surf': 1.0}, + 'parameters': {'planet.mass_tot': [0.7, 3.0]}, + } + if switch is not None: + config['spectral_cache'] = switch + output_root = tmp_path / 'output' + monkeypatch.setattr( + inference_mod, + 'get_proteus_directories', + lambda _output: {'output': str(output_root), 'proteus': ''}, + ) + monkeypatch.setattr(inference_mod, 'setup_logger', lambda **_kwargs: None) + monkeypatch.setattr(inference_mod.os, 'cpu_count', lambda: 8) + + def _stop(*_a, **_kw): + raise _StopAfterSetup + + monkeypatch.setattr(inference_mod, 'create_init', _stop) + # Registered so the values run_inference writes are undone afterwards. + for env in (SPECTRAL_CACHE_ENV, ABORT_ON_FAILURE_ENV, CHILD_TIMEOUT_ENV): + monkeypatch.setenv(env, 'unset') + + with caplog.at_level('INFO'), pytest.raises(_StopAfterSetup): + inference_mod.run_inference(config) + + lines = [r.getMessage() for r in caplog.records if 'Spectral cache' in r.getMessage()] + assert len(lines) == 1 + if switch is False: + assert ( + lines[0] == 'Spectral cache: off (spectral_cache = false in the inference config)' + ) + assert os.environ[SPECTRAL_CACHE_ENV] == '0' + else: + assert lines[0].endswith(f'unit_inference/{SPECTRAL_CACHE_DIR}') + assert os.environ[SPECTRAL_CACHE_ENV] == '1' + + # ============================================================================ # Regression: no stray prints + docstring uses current schema # ============================================================================ diff --git a/tests/inference/test_objective.py b/tests/inference/test_objective.py index 4f22fa03b..b906e9f67 100644 --- a/tests/inference/test_objective.py +++ b/tests/inference/test_objective.py @@ -82,6 +82,69 @@ def test_update_toml_updates_nested_keys(tmp_path): assert loaded['new']['branch']['leaf'] == 3 +@pytest.mark.unit +@pytest.mark.parametrize('enabled', [True, False], ids=['cache-on', 'cache-off']) +@pytest.mark.parametrize( + 'ref_cache', [None, 'none', '/my/cache'], ids=['unset', 'none', 'path'] +) +def test_run_proteus_takes_the_spectral_cache_from_the_inference_switch( + monkeypatch, tmp_path, enabled, ref_cache +): + """The inference config's `spectral_cache` switch decides the cache every + worker runs with, whatever the reference config sets: a reference config + copied from `all_options.toml` carries "none" without the user choosing it. + On, workers share the study's cache folder; off, they run with "none". + Both config writes, before and after the run, carry the same value. + """ + monkeypatch.setenv('PROTEUS_OUTPUT_PATH', str(tmp_path)) + monkeypatch.setenv(objective_mod.SPECTRAL_CACHE_ENV, '1' if enabled else '0') + ref = {'planet': {'mass_tot': 1.0}} + if ref_cache is not None: + ref['atmos_clim'] = {'spectral_cache': ref_cache} + ref_config = tmp_path / 'reference.toml' + ref_config.write_text(toml.dumps(ref), encoding='utf-8') + run_dir = tmp_path / 'study' / 'workers' / 'w_0' / 'i_0' + + seen = [] + + def _fake_run(command, **_kwargs): + seen.append(toml.load(command[3])['atmos_clim']['spectral_cache']) + pd.DataFrame([{'P_surf': 1e5}]).to_csv( + run_dir / 'runtime_helpfile.csv', sep=' ', index=False + ) + + monkeypatch.setattr(objective_mod.subprocess, 'run', _fake_run) + objective_mod.run_proteus( + parameters={'planet.mass_tot': 2.0}, + worker=0, + iter=0, + observables=['P_surf'], + ref_config=str(ref_config), + output='study', + ) + written = toml.load(run_dir / 'input.toml') + study_cache = str(tmp_path / 'study' / objective_mod.SPECTRAL_CACHE_DIR) + assert seen == [study_cache if enabled else 'none'] + assert written['atmos_clim']['spectral_cache'] == seen[0] + # The swept value is still applied alongside. + assert written['planet']['mass_tot'] == pytest.approx(2.0) + + +@pytest.mark.unit +def test_spectral_cache_switch_defaults_to_on(monkeypatch): + """Workers started without the switch recorded share the cache, and only + an explicit "0" turns it off. + """ + monkeypatch.delenv(objective_mod.SPECTRAL_CACHE_ENV, raising=False) + assert objective_mod.spectral_cache_enabled() is True + monkeypatch.setenv(objective_mod.SPECTRAL_CACHE_ENV, '0') + assert objective_mod.spectral_cache_enabled() is False + monkeypatch.setenv(objective_mod.SPECTRAL_CACHE_ENV, '1') + assert objective_mod.spectral_cache_enabled() is True + # Off gives the literal the config converter reads as disabled. + assert objective_mod.worker_spectral_cache('study', False) == 'none' + + @pytest.mark.unit def test_apply_nested_updates_mutates_in_place_and_rejects_value_paths(): """``apply_nested_updates`` writes dotted keys into the dict it was given, @@ -153,6 +216,7 @@ def test_run_proteus_success_handles_escaped_atmosphere(monkeypatch, tmp_path): # value as a number and names only the swept parameters. assert updates[0][1]['params.out.path'] == 'dummy_output/workers/w_1/i_2' assert updates[0][1]['params.out.plot_mod'] == 'none' + assert updates[0][1]['atmos_clim.spectral_cache'].endswith('spectral_cache') assert list(parameters) == ['planet.mass_tot'] assert parameters['planet.mass_tot'] == pytest.approx(2.0) # No status file was written, which is reported as such rather than as a From a7322ee269482e5275a400544c9848f62c4411d4 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Mon, 28 Sep 2026 16:31:36 +0200 Subject: [PATCH 70/72] cache only the spectral file at first build of each run --- src/proteus/atmos_clim/agni.py | 6 ++- src/proteus/atmos_clim/wrapper.py | 3 +- tests/atmos_clim/test_agni.py | 30 ++++++++++++++ tests/atmos_clim/test_wrapper.py | 65 +++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index 6f7df16d3..2fa3c2b23 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -454,7 +454,7 @@ def _determine_aerosols(dirs: dict) -> list: return aerosols -def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): +def init_agni_atmos(dirs: dict, config: Config, hf_row: dict, use_cache: bool = True): """Initialise atmosphere struct for use by AGNI. Does not set the temperature profile. @@ -467,6 +467,8 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): Configuration options and other variables hf_row : dict Dictionary containing simulation variables for current iteration + use_cache : bool + Whether to read from and store into `atmos_clim.spectral_cache`. Returns ---------- @@ -550,7 +552,7 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict): # Reuse a cached file built earlier from this base file and this stellar # spectrum, and skip the insertion. - if config.atmos_clim.spectral_cache: + if config.atmos_clim.spectral_cache and use_cache: key = cache_key( input_sf, sflux_path, diff --git a/src/proteus/atmos_clim/wrapper.py b/src/proteus/atmos_clim/wrapper.py index 799815c3a..412f4517e 100644 --- a/src/proteus/atmos_clim/wrapper.py +++ b/src/proteus/atmos_clim/wrapper.py @@ -364,7 +364,8 @@ def run_atmosphere( deallocate_atmos(atmos_o._atm) # allocate new - atmos_o._atm = init_agni_atmos(dirs, config, hf_row) + # Only the first build can match another run's spectral file. + atmos_o._atm = init_agni_atmos(dirs, config, hf_row, use_cache=no_atm) # Check allocation was ok if not bool(atmos_o._atm.is_alloc): diff --git a/tests/atmos_clim/test_agni.py b/tests/atmos_clim/test_agni.py index 97adb38cb..91cbf8a07 100644 --- a/tests/atmos_clim/test_agni.py +++ b/tests/atmos_clim/test_agni.py @@ -580,6 +580,36 @@ def test_a_cached_spectral_file_is_reused_without_reinserting_the_spectrum( assert not (ctx.output_dir / 'runtime.sf').exists() +@pytest.mark.unit +def test_a_rebuild_after_a_spectrum_update_bypasses_the_spectral_cache(monkeypatch, tmp_path): + """A rebuild after a stellar-spectrum update neither reads nor fills the + cache: its spectrum depends on the run's own timestep, so an entry stored + for it would never be reused, and a study would accumulate one per update. + The first build of the same run, with the same entry present, does use it. + """ + cache = tmp_path / 'cache' + cache.mkdir() + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) + key = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=None) + (cache / f'{key}.sf').write_text('cached prepared file', encoding='utf-8') + (cache / f'{key}.sf_k').write_text('cached ktable', encoding='utf-8') + before = sorted(p.name for p in cache.iterdir()) + + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row, use_cache=False) is not None + # Built from the spectrum, not seeded, and nothing added to the cache. + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + assert (ctx.scratch / 'runtime.sf').read_text() != 'cached prepared file' + assert sorted(p.name for p in cache.iterdir()) == before + + # Discrimination: the first build reads the same entry, so the bypass above + # comes from the flag and not from a cache that could not be read. + for suffix in ('', '_k'): + (ctx.scratch / f'runtime.sf{suffix}').unlink() + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) is not None + assert ctx.fake_agni.last_allocate_input_star == '' + assert (ctx.scratch / 'runtime.sf').read_text() == 'cached prepared file' + + @pytest.mark.unit def test_a_cached_spectral_file_is_not_reused_across_rayleigh_settings(monkeypatch, tmp_path): """A cache folder shared by runs that differ only in `atmos_clim.rayleigh` diff --git a/tests/atmos_clim/test_wrapper.py b/tests/atmos_clim/test_wrapper.py index dc4f19100..4a495d7bc 100644 --- a/tests/atmos_clim/test_wrapper.py +++ b/tests/atmos_clim/test_wrapper.py @@ -452,6 +452,71 @@ def test_run_atmosphere_keeps_the_column_janus_solved(): assert atmos_o._atm_janus_last is solved +class _StopAfterInit(Exception): + """Raised by the stand-in for `init_agni_atmos`, once its arguments are known.""" + + +@pytest.mark.parametrize( + ('first_build', 'use_cache'), + [(True, True), (False, False)], + ids=['first-build-uses-cache', 'spectrum-update-bypasses-it'], +) +def test_run_atmosphere_uses_the_spectral_cache_only_for_the_first_agni_build( + tmp_path, first_build, use_cache +): + """The first AGNI build of a run may use the spectral-file cache, because + its stellar spectrum can match another run's. A rebuild after a spectrum + update may not: its spectrum depends on when this run's adaptive timestep + reached the update, so a cache entry for it would never be read. + """ + atmos_o = Atmos_t() + atmos_o._atm = None if first_build else object() + config = SimpleNamespace( + atmos_clim=SimpleNamespace( + module='agni', + albedo_pl=0.0, + rayleigh=False, + cloud_enabled=False, + surf_state='fixed', + agni=SimpleNamespace(verbosity=0), + ), + interior_energetics=SimpleNamespace(module='aragog'), + params=SimpleNamespace(resume=False), + ) + hf_row = {'T_magma': 1800.0, 'T_surf': 0.0} + # A rebuild removes the previous prepared file first. + (tmp_path / 'runtime.sf').write_text('from the previous spectrum', encoding='utf-8') + + seen = {} + + def _init(_dirs, _config, _hf_row, use_cache=True): + seen['use_cache'] = use_cache + raise _StopAfterInit + + with ( + patch('proteus.atmos_clim.agni.activate_julia'), + patch('proteus.atmos_clim.agni.deallocate_atmos'), + patch('proteus.atmos_clim.agni.init_agni_atmos', side_effect=_init), + pytest.raises(_StopAfterInit), + ): + atmos_wrapper.run_atmosphere( + atmos_o, + config, + {'output': str(tmp_path), 'fwl': str(tmp_path)}, + {'total': 0 if first_build else 7}, + [1.0], + [1.0], + not first_build, + None, + hf_row, + ) + + assert seen == {'use_cache': use_cache} + # Edge case: the first build keeps a prepared file already on disk, which a + # resume relies on; a rebuild removes it so the new spectrum is inserted. + assert (tmp_path / 'runtime.sf').exists() == first_build + + # --------------------------------------------------------------------------- # carry_converged_levels: levels of a rejected structure are not used # --------------------------------------------------------------------------- From c9d245e2c20dd9584ac685b690da90778ea49b5f Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Tue, 29 Sep 2026 14:51:15 +0200 Subject: [PATCH 71/72] update agni.py and test spectral cache order to match new main --- src/proteus/atmos_clim/agni.py | 47 ++++++++++++++++------------------ tests/atmos_clim/test_agni.py | 46 ++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index 6ac8e51b5..41223de7f 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -515,14 +515,7 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict, use_cache: bool = # Set when this run built a prepared spectral file that the cache does not # yet hold, so it can be stored once the build is known to have succeeded. cache_store_key = None - - # Aerosol species dictionary (set MMR to zero initially). Determined before - # the spectral file, whose cache key depends on it. - aerosol_species = {} - if config.atmos_clim.aerosols_enabled: - aerosol_species = {a: 0.0 for a in _determine_aerosols(dirs)} - if len(aerosol_species) == 0: - log.warning('No data found for aerosol species') + cache_candidate = False # Spectral file path provided? if config.atmos_clim.agni.spectral_file is not None: @@ -567,23 +560,9 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict, use_cache: bool = input_sf = get_spfile_path(dirs['fwl'], config) input_star = sflux_path - # Reuse a cached file built earlier from this base file and this stellar - # spectrum, and skip the insertion. - if config.atmos_clim.spectral_cache and use_cache: - key = cache_key( - input_sf, - sflux_path, - config.atmos_clim.spectral_group, - config.atmos_clim.spectral_bands, - rayleigh=config.atmos_clim.rayleigh, - aerosols=list(aerosol_species) if config.atmos_clim.aerosols_enabled else None, - ) - if seed_from_cache(config.atmos_clim.spectral_cache, key, io_dir): - log.debug('Reusing prepared spectral file from cache') - input_sf = os.path.join(io_dir, 'runtime.sf') - input_star = '' - else: - cache_store_key = key + # The cache is consulted once the aerosol species are known, because they + # are part of the cache key. + cache_candidate = bool(config.atmos_clim.spectral_cache and use_cache) # composition vol_dict = _construct_voldict(config, hf_row, dirs) @@ -651,6 +630,24 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict, use_cache: bool = if len(aerosol_species) == 0: log.warning(' No aerosols mapped or data unavailable') + # Reuse a cached file built earlier from this base file and this stellar + # spectrum, and skip the insertion. + if cache_candidate: + key = cache_key( + input_sf, + sflux_path, + config.atmos_clim.spectral_group, + config.atmos_clim.spectral_bands, + rayleigh=config.atmos_clim.rayleigh, + aerosols=list(aerosol_species) if config.atmos_clim.aerosols_enabled else None, + ) + if seed_from_cache(config.atmos_clim.spectral_cache, key, io_dir): + log.debug('Reusing prepared spectral file from cache') + input_sf = os.path.join(io_dir, 'runtime.sf') + input_star = '' + else: + cache_store_key = key + # Build the AGNI setup! kwargs. setup_kwargs = dict( IO_DIR=io_dir, diff --git a/tests/atmos_clim/test_agni.py b/tests/atmos_clim/test_agni.py index bbdb5f5b8..e1019b6ca 100644 --- a/tests/atmos_clim/test_agni.py +++ b/tests/atmos_clim/test_agni.py @@ -544,7 +544,9 @@ def _setup_cached_spectral_run(monkeypatch, tmp_path, cache_dir, verbosity=1, lo AGNI works in: the output folder only when verbose or debug-logged. """ fake_agni = _SpectralWritingAGNI() - fake_jl = SimpleNamespace(AGNI=fake_agni, Dict=dict, Char=str) + fake_jl = SimpleNamespace( + AGNI=fake_agni, Dict=dict, Char=str, Array=_FAKE_JL_ARRAY, String=str + ) output_dir = tmp_path / 'out' data_dir = output_dir / 'data' @@ -592,6 +594,7 @@ def _fake_tmp_folder(): 'axial_period': 86400.0, 'longitude': 0.0, 'latitude': 0.0, + 'hill_radius': 6.4e8, }, output_dir=output_dir, scratch=scratch, @@ -737,6 +740,47 @@ def test_a_cached_spectral_file_is_not_reused_across_rayleigh_settings(monkeypat assert (cache / f'{without}.sf').read_text() == 'built without rayleigh' +@pytest.mark.unit +def test_a_cached_spectral_file_is_keyed_on_the_aerosols_agni_receives(monkeypatch, tmp_path): + """The cache key names the aerosols tied to a condensate, not all available ones. + + AGNI only receives aerosols tied to a condensate, and writes their blocks + into the prepared file. A run without condensates therefore passes no + aerosols even when SiO2 data exist, and must not reuse an entry built by a + run whose rainout tied SiO2 in. Keying on every available species would + give both runs the same key and hand the first the wrong file. + """ + cache = tmp_path / 'cache' + cache.mkdir() + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) + ctx.config.atmos_clim.aerosols_enabled = True + monkeypatch.setattr(agni_mod, '_determine_aerosols', lambda _d: {'SiO2': 'mon'}) + monkeypatch.setattr(agni_mod, '_determine_condensates', lambda _v: ['SiO2']) + + tied = cache_key( + ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=['SiO2'] + ) + untied = cache_key(ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=[]) + # Discrimination guard: the two runs must map to different entries. + assert tied != untied + (cache / f'{tied}.sf').write_text('built with SiO2 block', encoding='utf-8') + (cache / f'{tied}.sf_k').write_text('ktable with SiO2 block', encoding='utf-8') + + # Edge case: SiO2 data available, but no condensates, so nothing is tied. + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) is not None + assert ctx.fake_agni.last_setup_kwargs['aerosol_species'] == {} + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + assert (cache / f'{untied}.sf').read_text() == 'prepared from 100.sflux' + assert (cache / f'{tied}.sf').read_text() == 'built with SiO2 block' + + # With rainout on, SiO2 is tied and the matching entry is reused as is. + ctx.config.atmos_clim.agni.rainout = True + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) is not None + assert list(ctx.fake_agni.last_setup_kwargs['aerosol_species']) == ['SiO2'] + assert ctx.fake_agni.last_allocate_input_star == '' + assert (ctx.scratch / 'runtime.sf').read_text() == 'built with SiO2 block' + + @pytest.mark.unit def test_a_cache_that_cannot_be_written_costs_time_and_not_correctness( monkeypatch, tmp_path, caplog From fb4e32c11b31f4857be92be080051d8c6159b4d5 Mon Sep 17 00:00:00 2001 From: Karen Stuitje Date: Tue, 29 Sep 2026 15:48:15 +0200 Subject: [PATCH 72/72] refuse cache when using mie aerosols --- src/proteus/atmos_clim/agni.py | 7 +++++++ tests/atmos_clim/test_agni.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/proteus/atmos_clim/agni.py b/src/proteus/atmos_clim/agni.py index 41223de7f..ad50b5bb1 100644 --- a/src/proteus/atmos_clim/agni.py +++ b/src/proteus/atmos_clim/agni.py @@ -630,6 +630,13 @@ def init_agni_atmos(dirs: dict, config: Config, hf_row: dict, use_cache: bool = if len(aerosol_species) == 0: log.warning(' No aerosols mapped or data unavailable') + # AGNI computes Mie aerosol properties from the stellar spectrum while it + # builds the file, so a run with a Mie aerosol always refuses the cache. + if cache_candidate and config.atmos_clim.aerosols_enabled: + if any(entry['method'] == 'mie' for entry in aerosol_species.values()): + log.debug('Mie aerosols present; not using the spectral-file cache') + cache_candidate = False + # Reuse a cached file built earlier from this base file and this stellar # spectrum, and skip the insertion. if cache_candidate: diff --git a/tests/atmos_clim/test_agni.py b/tests/atmos_clim/test_agni.py index e1019b6ca..1d1a31520 100644 --- a/tests/atmos_clim/test_agni.py +++ b/tests/atmos_clim/test_agni.py @@ -523,10 +523,17 @@ class _SpectralWritingAGNI(_FakeAGNI): AGNI builds `/runtime.sf` and its `_k` companion inside `allocate!`, and only when a stellar spectrum is supplied; an empty spectrum means the - spectral file it was handed is already prepared and is used untouched. + spectral file it was handed is already prepared and is used untouched. As + in AGNI, a prepared file is refused when an enabled aerosol uses Mie theory, + whose properties are computed from the stellar spectrum. """ def _allocate_b(self, atmos, input_star, **kwargs): + setup = self.last_setup_kwargs + aerosols = setup.get('aerosol_species', {}) + if not input_star and setup.get('flag_aerosol'): + if any(entry['method'] == 'mie' for entry in aerosols.values()): + return False if input_star: io_dir = Path(self.last_setup_kwargs['IO_DIR']) io_dir.mkdir(parents=True, exist_ok=True) @@ -781,6 +788,29 @@ def test_a_cached_spectral_file_is_keyed_on_the_aerosols_agni_receives(monkeypat assert (ctx.scratch / 'runtime.sf').read_text() == 'built with SiO2 block' +@pytest.mark.unit +def test_a_run_with_a_mie_aerosol_builds_its_own_spectral_file(monkeypatch, tmp_path): + """A Mie aerosol skips the cache: AGNI refuses a prepared file for it.""" + cache = tmp_path / 'cache' + cache.mkdir() + ctx = _setup_cached_spectral_run(monkeypatch, tmp_path, cache) + ctx.config.atmos_clim.aerosols_enabled = True + ctx.config.atmos_clim.agni.rainout = True + monkeypatch.setattr(agni_mod, '_determine_aerosols', lambda _d: {'SiO2': 'mie'}) + monkeypatch.setattr(agni_mod, '_determine_condensates', lambda _v: ['SiO2']) + key = cache_key( + ctx.base_sf, ctx.sflux, 'Honeyside', '16', rayleigh=False, aerosols=['SiO2'] + ) + (cache / f'{key}.sf').write_text('built with SiO2 block', encoding='utf-8') + (cache / f'{key}.sf_k').write_text('ktable with SiO2 block', encoding='utf-8') + + # A hit would reach the fake's refusal and raise; the run builds instead. + assert init_agni_atmos(ctx.dirs, ctx.config, ctx.hf_row) is not None + assert ctx.fake_agni.last_allocate_input_star == str(ctx.sflux) + assert sorted(p.name for p in cache.iterdir()) == [f'{key}.sf', f'{key}.sf_k'] + assert (cache / f'{key}.sf').read_text() == 'built with SiO2 block' + + @pytest.mark.unit def test_a_cache_that_cannot_be_written_costs_time_and_not_correctness( monkeypatch, tmp_path, caplog