From 239ab13c4353e9508fd4846321f03ff724f79253 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Fri, 25 Sep 2026 12:39:47 -0600 Subject: [PATCH 01/85] agents: limit use of pydantic to data validation --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index f1865059..7a185e4b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ - Use python type hints. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. - Do not quote class names in hints. Use `from __future__ import annotations` when necessary. - Since the arguments types and return types are already documented by the hints, there's no need to duplicate this information in the docstrings. -- Apart from particular cases, use pydantic BaseModel instead of dataclass +- Use pydantic BaseModel when validation is needed, and dataclass for internal data exchange. - Place imports at the top of the file. Only break this rule to prevent heavy imports in a rarely used function (for example, CLI options). ## General comments From 93b85b832e76fe6a47afdd9a7cfe0f2fc2c71549 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 13:32:28 -0600 Subject: [PATCH 02/85] first draft --- docs/cli.rst | 24 +- docs/conf_file.rst | 88 +- pyproject.toml | 5 +- src/itzi/cli_parser.py | 19 +- src/itzi/ensemble.py | 1072 +++++++++++++++++++++++++ src/itzi/grass_session.py | 69 +- src/itzi/itzi.py | 567 +++++++++---- src/itzi/providers/__init__.py | 3 + src/itzi/providers/csv_output.py | 37 + src/itzi/providers/grass_input.py | 24 +- src/itzi/providers/grass_interface.py | 131 +-- src/itzi/providers/grass_output.py | 18 +- src/itzi/resolution.py | 476 +++++++++++ src/itzi/run_plan.py | 149 ++++ src/itzi/simulation_runner.py | 179 +++++ tests/cli/test_cli.py | 9 + tests/cli/test_ensemble_config.py | 234 ++++++ tests/grass/test_ensemble.py | 58 ++ uv.lock | 2 + 19 files changed, 2942 insertions(+), 222 deletions(-) create mode 100644 src/itzi/ensemble.py create mode 100644 src/itzi/providers/csv_output.py create mode 100644 src/itzi/resolution.py create mode 100644 src/itzi/run_plan.py create mode 100644 src/itzi/simulation_runner.py create mode 100644 tests/cli/test_ensemble_config.py create mode 100644 tests/grass/test_ensemble.py diff --git a/docs/cli.rst b/docs/cli.rst index 5b891bb6..9d349606 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -9,7 +9,29 @@ Run a simulation :func: build_parser :prog: itzi :path: run - :nodefault: + :nodefault: + +YAML ensembles +~~~~~~~~~~~~~~ + +Each YAML document is one ensemble. Use ``--dry`` to perform parsing, +expansion, GRASS input resolution, member-ID calculation, and artifact +validation without writing manifests or model outputs: + +.. code-block:: bash + + itzi run --dry studies.yaml + +Select resolved members with a qualified ensemble and simulation ID. The +unqualified ID form is accepted only when the batch contains one ensemble. + +.. code-block:: bash + + itzi run studies.yaml --member central-city#sim-c1f52e9a + +Stage 1 provides basic YAML ensemble execution. YAML hotstart resume mapping +and checkpoint provenance are introduced in Stage 2; the retained INI resume +syntax below continues to apply to legacy INI-only invocations. Hotstart usage diff --git a/docs/conf_file.rst b/docs/conf_file.rst index 61e2210a..b05b2034 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -2,10 +2,90 @@ Configuration file ================== -The parameters of a simulation are given through a configuration file in -a format similar to Microsoft Windows INI files. -An example is given in the tutorial above. -The file is separated in sections described below. +The public configuration format is a versioned YAML stream. Each non-empty +YAML document defines one ensemble, and a document may contain sweep lists for +Cartesian member expansion. A command may receive several YAML streams and +legacy INI files in order. + +YAML files must use the ``.yaml`` extension. Other extensions are interpreted +as deprecated legacy INI configurations during the migration release. A YAML +syntax or schema error never falls back to INI parsing. + +The supported YAML subset rejects duplicate mapping keys, merge keys, custom +tags, empty documents, and unknown schema fields. Quote timestamps, durations, +map identifiers, paths, templates, and ambiguous strings such as ``yes`` or +``on``. Documents after an explicit ``---`` marker are independently loaded, +so an invalid document does not prevent a later explicitly delimited document +from being reported. + +Basic YAML example +------------------ + +.. code-block:: yaml + + schema_version: 1 + ensemble: + id: "central-city" + name: "Central city rainfall study" + domain: + grass: + database: "/srv/grassdata" + project: "central_city" + mapset: "itzi" + executable: "grass" + region: "central_city_5m" + mask: "central_city_mask" + time: + start: "2026-09-01T00:00:00" + duration: "PT2H" + record_step: "PT5M" + input: + ground_elevation: "elevation_5m@PERMANENT" + friction: "manning_n@PERMANENT" + rainfall_rate: ["rain_10yr@PERMANENT", "rain_100yr@PERMANENT"] + infiltration: + type: "none" + options: + cfl: [0.5, 0.7] + outputs: + rasters: + prefix: "central_city_{simulation}" + variables: [water_depth, flow_speed] + statistics: + file: "results/{simulation}.csv" + +The example expands to four simulations: two rainfall maps multiplied by two +``cfl`` values. Scalars are fixed across an ensemble, while non-empty lists in +``input``, ``options``, or ``drainage`` are sweep dimensions. ``domain`` and +``time`` are always scalar. Output variable lists are selections, not sweeps. +An ensemble may contain at most 100 candidate simulations; a full command may +contain at most 200. + +All simulations in an ensemble use one geographical domain, time form, and +``record_step``. Time accepts exactly one of a relative ``duration``, absolute +``start`` plus ``duration``, or absolute ``start`` plus ``end``. Durations use +the supported non-calendar ISO 8601 subset, such as ``"PT5M"`` and ``"P1DT2H"``. +Timezone offsets are retained as source metadata but execute as naive GRASS +wall-clock times without conversion. + +YAML infiltration alternatives are explicit under ``input.infiltration``: +``type: "none"``, ``type: "constant"`` with a ``rate`` map, or +``type: "green-ampt"`` with ``effective_porosity``, ``capillary_pressure``, and +``hydraulic_conductivity`` maps. A list of complete alternatives is one sweep +dimension. The optional ``label`` is documentation only. + +``{ensemble}`` and ``{simulation}`` are the only output-template substitutions. +Output maps are always bare names in the active execution mapset; qualified +output names are rejected. YAML filesystem paths are relative to the YAML +source file. Use ``itzi run --dry study.yaml`` to resolve and validate members +without creating output artifacts. + +Legacy INI reference +-------------------- + +The remaining sections document the deprecated single-simulation INI format. +It remains supported for one announced migration release. New configurations +should use YAML. [time] ------ diff --git a/pyproject.toml b/pyproject.toml index 66c7e8bf..aff8c4da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,8 @@ dependencies = [ "numpy>=2.5", "itzi-core==0.8.0", "bmipy>=2.0.1", - "pydantic>=2.10" + "pydantic>=2.10", + "pyyaml>=6.0" ] # min 3.12 for typing. requires-python = ">=3.12" @@ -57,7 +58,7 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -"itzi.data" = ["*.ini"] +"itzi.data" = ["*.ini", "*.yaml"] "itzi.data.colortable" = ["*.txt"] [tool.pytest.ini_options] diff --git a/src/itzi/cli_parser.py b/src/itzi/cli_parser.py index 1b42dd67..1c42a5a5 100644 --- a/src/itzi/cli_parser.py +++ b/src/itzi/cli_parser.py @@ -25,11 +25,11 @@ def build_parser() -> argparse.ArgumentParser: subparsers = arg_parser.add_subparsers(dest="command", required=True) # run a simulation - run_parser = subparsers.add_parser("run", help="Run a simulation.") + run_parser = subparsers.add_parser("run", help="Run a configuration file or YAML stream.") run_parser.add_argument( "config_file", nargs="+", - help=("An Itzï configuration file (if several given, run in batch mode.)"), + help=("An Itzï configuration file or YAML stream (several files form one batch)."), ) run_parser.add_argument("-o", action="store_true", help="Overwrite files if exist.") verbosity_parser = run_parser.add_mutually_exclusive_group() @@ -44,9 +44,22 @@ def build_parser() -> argparse.ArgumentParser: help=( "Resume a simulation from a hotstart file. " "If only the path to a hotstart file is given, batch is not allowed. " - "For batch processing, use the 'CONFIG_PATH=HOTSTART_PATH' construct." + "For batch processing, use CONFIG_PATH=HOTSTART_PATH or " + "ENSEMBLE#SIMULATION=HOTSTART_PATH." ), ) + run_parser.add_argument( + "--member", + action="append", + default=[], + metavar="[ENSEMBLE#]SIMULATION", + help="Run only a resolved ensemble member. Repeat to select multiple members.", + ) + run_parser.add_argument( + "--dry", + action="store_true", + help="Resolve and validate selected members without writing user artifacts.", + ) # display version subparsers.add_parser("version", help="Display software version number.") diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py new file mode 100644 index 00000000..a41bff79 --- /dev/null +++ b/src/itzi/ensemble.py @@ -0,0 +1,1072 @@ +"""YAML ensemble configuration loading and deterministic expansion. + +This module intentionally has no GRASS imports. A configuration stream is +validated and expanded in the parent process before a resolver opens a GRASS +session. +""" + +from __future__ import annotations + +import hashlib +import itertools +import json +import math +import re +import string +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Annotated, Any, Literal + +import yaml +from itzi_core import ( + ARRAY_DEFINITIONS, + ArrayCategory, + DomainData, + InfiltrationModelType, + SimulationConfig, + TemporalType, +) +from itzi_core.const import DefaultValues +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictFloat, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +import itzi.messenger as msgr +from itzi.grass_session import GrassParams + +INPUT_KEYS = tuple( + definition.key + for definition in ARRAY_DEFINITIONS + if ArrayCategory.INPUT in definition.category +) +OUTPUT_KEYS = tuple( + definition.key + for definition in ARRAY_DEFINITIONS + if ArrayCategory.OUTPUT in definition.category +) +GREEN_AMPT_KEYS = ( + "effective_porosity", + "capillary_pressure", + "hydraulic_conductivity", + "soil_water_content", +) +DIRECT_INPUT_KEYS = tuple( + key for key in INPUT_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} +) +SURFACE_FLOW_DEFAULTS = { + "hmin": DefaultValues.HFMIN, + "cfl": DefaultValues.CFL, + "theta": DefaultValues.THETA, + "g": DefaultValues.G, + "dtmax": DefaultValues.DTMAX, + "slope_threshold": DefaultValues.SLOPE_THRESHOLD, + "max_slope": DefaultValues.MAX_SLOPE, + "max_error": DefaultValues.MAX_ERROR, +} +DRAINAGE_DEFAULTS = { + "orifice_coeff": DefaultValues.ORIFICE_COEFF, + "free_weir_coeff": DefaultValues.FREE_WEIR_COEFF, + "submerged_weir_coeff": DefaultValues.SUBMERGED_WEIR_COEFF, +} +MAX_ENSEMBLE_MEMBERS = 100 +MAX_BATCH_MEMBERS = 200 +DURATION_RE = re.compile( + r"P(?:(?P\d+)D)?(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?" + r"(?:(?P\d+(?:\.\d+)?)S)?)?\Z" +) +ENSEMBLE_ID_START_CHARS = string.ascii_letters + string.digits +ENSEMBLE_ID_CHARS = ENSEMBLE_ID_START_CHARS + "._-" + +type JsonValue = ( + None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] +) +type SweepString = StrictStr | list[StrictStr] +type SweepFloat = StrictFloat | list[StrictFloat] + + +class EnsembleError(ValueError): + """Configuration error associated with a single YAML document.""" + + +@dataclass(frozen=True) +class SourceDocument: + """Stable source metadata for one document in a YAML stream.""" + + path: Path + document_index: int + start_line: int + file_digest: str + document_digest: str + + +@dataclass(frozen=True) +class DocumentFailure: + """A document-local error that does not prevent later documents loading.""" + + path: Path + document_index: int + line: int | None + column: int | None + phase: Literal["parse", "schema", "expansion"] + detail: str + + def format(self) -> str: + location = f"{self.path} document {self.document_index}" + if self.line is not None and self.column is not None: + location += f" line {self.line}, column {self.column}" + return f"{location}: {self.phase}: {self.detail}" + + +class StrictModel(BaseModel): + """Base model for the public YAML schema.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EnsembleMetadata(StrictModel): + id: StrictStr + name: StrictStr | None = None + + @field_validator("id") + @classmethod + def validate_id(cls, value: str) -> str: + if ( + not value + or value[0] not in ENSEMBLE_ID_START_CHARS + or any(character not in ENSEMBLE_ID_CHARS for character in value[1:]) + ): + raise ValueError("must match [A-Za-z0-9][A-Za-z0-9._-]*") + return value + + +class GrassContextConfig(StrictModel): + database: StrictStr | None = None + project: StrictStr | None = None + mapset: StrictStr | None = None + executable: StrictStr | None = None + + @model_validator(mode="after") + def validate_context(self) -> GrassContextConfig: + values = (self.database, self.project, self.mapset) + if any(value is not None for value in values) and not all( + value is not None for value in values + ): + raise ValueError("database, project, and mapset must be supplied together") + if self.executable is not None and self.database is None: + raise ValueError("executable requires database, project, and mapset") + return self + + +class DomainConfig(StrictModel): + grass: GrassContextConfig | None = None + region: StrictStr | None = None + mask: StrictStr | None = None + + +class RelativeTimeConfig(StrictModel): + duration: StrictStr + record_step: StrictStr + start: None = None + end: None = None + + +class AbsoluteDurationTimeConfig(StrictModel): + start: StrictStr + duration: StrictStr + record_step: StrictStr + end: None = None + + +class AbsoluteEndTimeConfig(StrictModel): + start: StrictStr + end: StrictStr + record_step: StrictStr + duration: None = None + + +type TimeConfig = RelativeTimeConfig | AbsoluteDurationTimeConfig | AbsoluteEndTimeConfig + + +class NoInfiltration(StrictModel): + type: Literal["none"] + label: StrictStr | None = None + + +class ConstantInfiltration(StrictModel): + type: Literal["constant"] + label: StrictStr | None = None + rate: StrictStr + + +class GreenAmptInfiltration(StrictModel): + type: Literal["green-ampt"] + label: StrictStr | None = None + effective_porosity: StrictStr + capillary_pressure: StrictStr + hydraulic_conductivity: StrictStr + soil_water_content: StrictStr | None = None + + +type InfiltrationAlternative = Annotated[ + NoInfiltration | ConstantInfiltration | GreenAmptInfiltration, + Field(discriminator="type"), +] + + +class InputSweepConfig(StrictModel): + ground_elevation: SweepString + friction: SweepString + water_depth: SweepString | None = None + water_surface_elevation: SweepString | None = None + losses: SweepString | None = None + rainfall_rate: SweepString | None = None + inflow: SweepString | None = None + boundary_value: SweepString | None = None + boundary_type: SweepString | None = None + infiltration: InfiltrationAlternative | list[InfiltrationAlternative] = NoInfiltration( + type="none" + ) + + @field_validator( + "ground_elevation", + "friction", + "water_depth", + "water_surface_elevation", + "losses", + "rainfall_rate", + "inflow", + "boundary_value", + "boundary_type", + ) + @classmethod + def validate_string_sweep(cls, value: SweepString | None) -> SweepString | None: + return _validate_sweep(value) + + @field_validator("infiltration") + @classmethod + def validate_infiltration_sweep( + cls, value: InfiltrationAlternative | list[InfiltrationAlternative] + ) -> InfiltrationAlternative | list[InfiltrationAlternative]: + return _validate_sweep(value) + + @model_validator(mode="after") + def validate_initial_conditions(self) -> InputSweepConfig: + if self.water_depth is not None and self.water_surface_elevation is not None: + raise ValueError("water_depth and water_surface_elevation are mutually exclusive") + return self + + +class OptionSweepConfig(StrictModel): + hmin: SweepFloat | None = None + cfl: SweepFloat | None = None + theta: SweepFloat | None = None + g: SweepFloat | None = None + dtmax: SweepFloat | None = None + slope_threshold: SweepFloat | None = None + max_slope: SweepFloat | None = None + max_error: SweepFloat | None = None + dtinf: SweepFloat | None = None + + @field_validator("*") + @classmethod + def validate_numeric_sweep(cls, value: SweepFloat | None) -> SweepFloat | None: + return _validate_sweep(value, numeric=True) + + +class DrainageSweepConfig(StrictModel): + swmm_input: SweepString + orifice_coeff: SweepFloat + free_weir_coeff: SweepFloat + submerged_weir_coeff: SweepFloat + + @field_validator("swmm_input") + @classmethod + def validate_input_sweep(cls, value: SweepString) -> SweepString: + return _validate_sweep(value) + + @field_validator("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") + @classmethod + def validate_coeff_sweep(cls, value: SweepFloat) -> SweepFloat: + return _validate_sweep(value, numeric=True) + + +class RasterOutputs(StrictModel): + prefix: StrictStr + variables: list[StrictStr] + + @field_validator("variables") + @classmethod + def validate_variables(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("must not be empty") + unknown = sorted(set(value) - set(OUTPUT_KEYS)) + if unknown: + raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") + if len(value) != len(set(value)): + raise ValueError("contains duplicate output variables") + return value + + +class StatisticsOutputs(StrictModel): + file: StrictStr + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class DrainageOutputs(StrictModel): + vector_dataset: StrictStr + + +class ManifestOutputs(StrictModel): + file: StrictStr + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class OutputConfig(StrictModel): + rasters: RasterOutputs | None = None + statistics: StatisticsOutputs | None = None + drainage: DrainageOutputs | None = None + manifest: ManifestOutputs | None = None + + +class HotstartOutputConfig(StrictModel): + wallclock_step: StrictStr + file: StrictStr + + @field_validator("wallclock_step") + @classmethod + def validate_step(cls, value: str) -> str: + if parse_iso_duration(value) <= timedelta(): + raise ValueError("must be strictly positive") + return value + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class YamlEnsembleDocumentV1(StrictModel): + schema_version: Literal[1] + ensemble: EnsembleMetadata + domain: DomainConfig + time: TimeConfig + input: InputSweepConfig + options: OptionSweepConfig + drainage: DrainageSweepConfig | None = None + outputs: OutputConfig + hotstart: HotstartOutputConfig | None = None + + @model_validator(mode="after") + def validate_drainage_output(self) -> YamlEnsembleDocumentV1: + if self.outputs.drainage is not None and self.drainage is None: + raise ValueError("outputs.drainage requires a complete drainage configuration") + return self + + +@dataclass(frozen=True) +class NormalizedTime: + """Time semantics retained until the final core configuration boundary.""" + + temporal_type: TemporalType + start: datetime | None + end: datetime | None + duration: timedelta + record_step: timedelta + had_timezone_offset: bool + + +@dataclass(frozen=True) +class NormalizedInfiltration: + model: InfiltrationModelType + input_maps: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class OutputTemplates: + raster_prefix: str | None + raster_variables: tuple[str, ...] + statistics_file: str | None + drainage_dataset: str | None + hotstart_file: str | None + hotstart_interval: timedelta | None + + +@dataclass(frozen=True) +class ExpandedSimulation: + """One fully scalar pre-GRASS application configuration.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + coordinates: tuple[tuple[str, JsonValue], ...] + domain: DomainConfig + time: NormalizedTime + input_maps: tuple[tuple[str, str], ...] + infiltration: NormalizedInfiltration + options: tuple[tuple[str, float], ...] + drainage: tuple[tuple[str, str | float], ...] | None + outputs: OutputTemplates + + +@dataclass(frozen=True) +class ExpandedEnsemble: + """One YAML document after Cartesian expansion.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + manifest_template: str | None + simulations: tuple[ExpandedSimulation, ...] + + +@dataclass(frozen=True) +class EffectiveMask: + """The immutable mask source selected during GRASS resolution.""" + + mode: Literal["explicit", "active", "none"] + source: str | None + + +@dataclass(frozen=True) +class ArtifactSummary: + """Rendered, application-owned output destinations for one member.""" + + output_map_names: tuple[tuple[str, str], ...] + drainage_output: str | None + statistics_file: Path | None + + +@dataclass(frozen=True) +class ResolvedSimulation: + """Spawn-serializable execution payload with one final core configuration.""" + + source: SourceDocument + ensemble_id: str + simulation_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + grass_params: GrassParams + domain_data: DomainData + effective_mask: EffectiveMask + input_kinds: tuple[tuple[str, Literal["raster", "strds"]], ...] + simulation_config: SimulationConfig + artifacts: ArtifactSummary + normalized_payload: str + + +@dataclass(frozen=True) +class ValidationFailure: + """A coordinate that could not become an executable resolved simulation.""" + + source: SourceDocument + ensemble_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + phase: str + detail: str + + +@dataclass(frozen=True) +class ResolvedEnsemble: + """Resolved member payloads and local validation failures for one ensemble.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + manifest_file: Path + simulations: tuple[ResolvedSimulation, ...] + failures: tuple[ValidationFailure, ...] + + +@dataclass(frozen=True) +class LoadedYamlStream: + """Valid ensembles and document-local failures from one YAML file.""" + + ensembles: tuple[ExpandedEnsemble, ...] + failures: tuple[DocumentFailure, ...] + + +class _StrictSafeLoader(yaml.SafeLoader): + """SafeLoader variant that rejects duplicate and merge mapping keys.""" + + +def _construct_mapping( + loader: _StrictSafeLoader, node: yaml.MappingNode, deep: bool = False +) -> dict: + mapping: dict[Any, Any] = {} + for key_node, value_node in node.value: + if key_node.tag == "tag:yaml.org,2002:merge": + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "YAML merge keys are not supported", + key_node.start_mark, + ) + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as error: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "mapping keys must be hashable", + key_node.start_mark, + ) from error + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"duplicate mapping key {key!r}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_StrictSafeLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + _construct_mapping, +) + + +def _validate_sweep( + value: Any, + *, + numeric: bool = False, +) -> Any: + """Validate one scalar-or-list sweep field after Pydantic's type validation.""" + values = value if isinstance(value, list) else [value] + if not values: + raise ValueError("sweep lists must not be empty") + for item in values: + if isinstance(item, list): + raise ValueError("nested sweep lists are not supported") # noqa: TRY004 + if numeric and (not isinstance(item, float) or not math.isfinite(item)): + raise ValueError("must contain finite numbers") + + semantic_values = [_canonical_json(_semantic_value(item)) for item in values] + if len(semantic_values) != len(set(semantic_values)): + raise ValueError("contains duplicate semantic values") + return value + + +def _semantic_value(value: Any) -> Any: + if isinstance(value, BaseModel): + dumped = value.model_dump(mode="python") + dumped.pop("label", None) + return dumped + return value + + +def parse_iso_duration(value: str) -> timedelta: + """Parse the supported, non-calendar ISO 8601 duration subset.""" + match = DURATION_RE.fullmatch(value) + if match is None or not any(match.groupdict().values()): + raise ValueError("must be an ISO 8601 duration without years or months") + groups = match.groupdict(default="0") + if "T" in value and not any(groups[key] != "0" for key in ("hours", "minutes", "seconds")): + raise ValueError("must include a time component after T") + duration = timedelta( + days=int(groups["days"]), + hours=int(groups["hours"]), + minutes=int(groups["minutes"]), + seconds=float(groups["seconds"]), + ) + if duration <= timedelta(): + raise ValueError("must be strictly positive") + return duration + + +def format_iso_duration(value: timedelta) -> str: + """Render a positive duration in a stable ISO 8601 representation.""" + total_seconds = value.total_seconds() + if total_seconds <= 0: + raise ValueError("duration must be strictly positive") + days, remainder = divmod(total_seconds, 86400) + hours, remainder = divmod(remainder, 3600) + minutes, seconds = divmod(remainder, 60) + parts = ["P"] + if days: + parts.append(f"{int(days)}D") + if hours or minutes or seconds or not days: + parts.append("T") + if hours: + parts.append(f"{int(hours)}H") + if minutes: + parts.append(f"{int(minutes)}M") + if seconds or (not hours and not minutes): + rendered_seconds = format(seconds, ".6f").rstrip("0").rstrip(".") + parts.append(f"{rendered_seconds}S") + return "".join(parts) + + +def normalize_time(value: TimeConfig) -> NormalizedTime: + """Validate a public time form without leaking core's relative sentinels.""" + duration = parse_iso_duration(value.duration) if value.duration is not None else None + record_step = parse_iso_duration(value.record_step) + start = _parse_datetime(value.start) if value.start is not None else None + end = _parse_datetime(value.end) if value.end is not None else None + aware = [timestamp for timestamp in (start, end) if timestamp is not None and timestamp.tzinfo] + naive = [ + timestamp for timestamp in (start, end) if timestamp is not None and not timestamp.tzinfo + ] + if aware and naive: + raise EnsembleError( + "absolute timestamps must either all have timezone offsets or all be naive" + ) + had_timezone_offset = bool(aware) + if aware: + start = start.replace(tzinfo=None) if start is not None else None + end = end.replace(tzinfo=None) if end is not None else None + + if start is None: + assert duration is not None + return NormalizedTime( + temporal_type=TemporalType.RELATIVE, + start=None, + end=None, + duration=duration, + record_step=record_step, + had_timezone_offset=False, + ) + if end is None: + assert duration is not None + end = start + duration + else: + duration = end - start + if duration <= timedelta(): + raise EnsembleError("simulation duration must be strictly positive") + return NormalizedTime( + temporal_type=TemporalType.ABSOLUTE, + start=start, + end=end, + duration=duration, + record_step=record_step, + had_timezone_offset=had_timezone_offset, + ) + + +def _parse_datetime(value: str) -> datetime: + if "T" not in value: + raise EnsembleError("timestamps must use ISO 8601 date and time notation") + try: + return datetime.fromisoformat(value) + except ValueError as error: + raise EnsembleError(f"invalid ISO 8601 timestamp {value!r}") from error + + +def _canonical_json(value: Any) -> str: + return json.dumps( + _canonical_value(value), + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_value(value: Any) -> Any: + if isinstance(value, BaseModel): + return _canonical_value(value.model_dump(mode="python")) + if isinstance(value, dict): + return {str(key): _canonical_value(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_canonical_value(item) for item in value] + if isinstance(value, datetime): + return {"$datetime": value.isoformat()} + if isinstance(value, date): + return {"$date": value.isoformat()} + if isinstance(value, timedelta): + return {"$duration": format_iso_duration(value)} + if isinstance(value, float) and not math.isfinite(value): + raise EnsembleError("NaN and infinity are not supported") + return value + + +def _document_digest(document: Any) -> str: + payload = {"schema": "itzi-yaml-document-v1", "document": _canonical_value(document)} + return hashlib.blake2b(_canonical_json(payload).encode(), digest_size=32).hexdigest() + + +def _is_document_marker(line: str, *, terminated: bool) -> bool: + """Return whether a physical line is a supported YAML document marker.""" + if terminated and line.endswith("\r"): + line = line[:-1] + if not line.startswith("---"): + return False + suffix = line[3:] + if suffix and suffix[0] not in " \t": + return False + suffix = suffix.lstrip(" \t") + return not suffix or suffix.startswith("#") + + +def _segments(text: str) -> tuple[tuple[int, int, str], ...]: + """Split at supported explicit document markers without parsing adjacent docs.""" + markers: list[tuple[int, int]] = [] + offset = 0 + lines = text.split("\n") + for index, line in enumerate(lines): + terminated = index < len(lines) - 1 + if _is_document_marker(line, terminated=terminated): + markers.append((offset, index + 1)) + offset += len(line) + int(terminated) + if not markers: + return ((1, 0, text),) + segments: list[tuple[int, int, str]] = [] + first = text[: markers[0][0]] + if first.strip(): + segments.append((1, 0, first)) + for index, (marker_offset, start_line) in enumerate(markers): + end = markers[index + 1][0] if index + 1 < len(markers) else len(text) + segments.append((start_line, marker_offset, text[marker_offset:end])) + return tuple(segments) + + +def _parse_segment(segment: str) -> dict[str, Any]: + if any(line.startswith("%") for line in segment.split("\n")): + raise EnsembleError("YAML directives are not supported") + value = yaml.load(segment, Loader=_StrictSafeLoader) + if value is None: + raise EnsembleError("empty YAML documents are not supported") + if not isinstance(value, dict): + raise EnsembleError("document root must be a mapping") + return value + + +def load_yaml_stream(path: str | Path) -> LoadedYamlStream: + """Load every explicitly delimited YAML document, retaining local failures.""" + source_path = Path(path).expanduser().resolve() + try: + content = source_path.read_bytes() + except OSError as error: + raise EnsembleError(f"cannot read {source_path}: {error}") from error + try: + text = content.decode("utf-8") + except UnicodeDecodeError as error: + raise EnsembleError(f"{source_path}: YAML must be UTF-8") from error + + file_digest = hashlib.blake2b(content, digest_size=32).hexdigest() + ensembles: list[ExpandedEnsemble] = [] + failures: list[DocumentFailure] = [] + for document_index, (start_line, _offset, segment) in enumerate(_segments(text)): + try: + raw_document = _parse_segment(segment) + except yaml.YAMLError as error: + marker = getattr(error, "problem_mark", None) + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line + marker.line if marker is not None else None, + column=marker.column + 1 if marker is not None else None, + phase="parse", + detail=str(getattr(error, "problem", error)), + ) + ) + continue + except EnsembleError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="parse", + detail=str(error), + ) + ) + continue + + source = SourceDocument( + path=source_path, + document_index=document_index, + start_line=start_line, + file_digest=file_digest, + document_digest=_document_digest(raw_document), + ) + try: + document = YamlEnsembleDocumentV1.model_validate(raw_document) + ensemble = expand_yaml_document(source, document) + if ensemble.simulations[0].time.had_timezone_offset: + msgr.warning( + f"{source.path} document {source.document_index}: timezone offsets are " + "informational and are ignored by GRASS" + ) + ensembles.append(ensemble) + except ValidationError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="schema", + detail=_format_validation_error(error), + ) + ) + except EnsembleError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="expansion", + detail=str(error), + ) + ) + return LoadedYamlStream(tuple(ensembles), tuple(failures)) + + +def _format_validation_error(error: ValidationError) -> str: + details = [] + for item in error.errors(include_url=False): + location = ".".join(str(part) for part in item["loc"]) + details.append(f"{location}: {item['msg']}" if location else item["msg"]) + return "; ".join(details) + + +def expand_yaml_document( + source: SourceDocument, document: YamlEnsembleDocumentV1 +) -> ExpandedEnsemble: + """Expand sweep dimensions in canonical path order into scalar simulations.""" + if document.hotstart is not None: + raise EnsembleError("YAML hotstart output requires Stage 2 checkpoint support") + normalized_time = normalize_time(document.time) + dimensions = _collect_dimensions(document) + count = _checked_product(tuple(len(values) for _, values in dimensions)) + if count > MAX_ENSEMBLE_MEMBERS: + sizes = ", ".join(f"{path}={len(values)}" for path, values in dimensions) + raise EnsembleError( + f"ensemble expands to {count} simulations, exceeding the {MAX_ENSEMBLE_MEMBERS} limit ({sizes})" + ) + + simulations = [] + for selected_values in itertools.product(*(values for _, values in dimensions)): + selected = { + path: value for (path, _), value in zip(dimensions, selected_values, strict=True) + } + coordinates = tuple( + (path, _freeze_json(_semantic_value(value))) + for path, value in sorted(selected.items()) + ) + input_maps = _selected_input_maps(document.input, selected) + infiltration = _normalize_infiltration( + selected.get("input.infiltration", document.input.infiltration) + ) + options = _selected_options(document.options, selected) + drainage = _selected_drainage(document.drainage, selected) + outputs = _output_templates(document) + simulations.append( + ExpandedSimulation( + source=source, + ensemble_id=document.ensemble.id, + ensemble_name=document.ensemble.name, + coordinates=coordinates, + domain=document.domain, + time=normalized_time, + input_maps=tuple(sorted(input_maps.items())), + infiltration=infiltration, + options=tuple(sorted(options.items())), + drainage=tuple(sorted(drainage.items())) if drainage is not None else None, + outputs=outputs, + ) + ) + manifest_template = document.outputs.manifest.file if document.outputs.manifest else None + if manifest_template is not None: + render_template(manifest_template, ensemble=document.ensemble.id, simulation=None) + return ExpandedEnsemble( + source=source, + ensemble_id=document.ensemble.id, + ensemble_name=document.ensemble.name, + manifest_template=manifest_template, + simulations=tuple(simulations), + ) + + +def _collect_dimensions( + document: YamlEnsembleDocumentV1, +) -> tuple[tuple[str, tuple[Any, ...]], ...]: + dimensions: list[tuple[str, tuple[Any, ...]]] = [] + for key in DIRECT_INPUT_KEYS: + value = getattr(document.input, key) + if isinstance(value, list): + dimensions.append((f"input.{key}", tuple(value))) + if isinstance(document.input.infiltration, list): + dimensions.append(("input.infiltration", tuple(document.input.infiltration))) + for key in type(document.options).model_fields: + value = getattr(document.options, key) + if isinstance(value, list): + dimensions.append((f"options.{key}", tuple(value))) + if document.drainage is not None: + for key in type(document.drainage).model_fields: + value = getattr(document.drainage, key) + if isinstance(value, list): + dimensions.append((f"drainage.{key}", tuple(value))) + return tuple(sorted(dimensions, key=lambda item: item[0])) + + +def _checked_product(sizes: tuple[int, ...]) -> int: + product = 1 + for size in sizes: + if size <= 0: + raise EnsembleError("sweep lists must not be empty") + product *= size + if product > MAX_ENSEMBLE_MEMBERS: + return product + return product + + +def _selected_input_maps( + input_config: InputSweepConfig, selected: dict[str, Any] +) -> dict[str, str]: + values: dict[str, str] = {} + for key in DIRECT_INPUT_KEYS: + value = selected.get(f"input.{key}", getattr(input_config, key)) + if value is not None: + assert isinstance(value, str) + values[key] = value + return values + + +def _normalize_infiltration( + value: InfiltrationAlternative | list[InfiltrationAlternative], +) -> NormalizedInfiltration: + assert not isinstance(value, list), "infiltration must be scalar after expansion" + if isinstance(value, NoInfiltration): + return NormalizedInfiltration(InfiltrationModelType.NULL, ()) + if isinstance(value, ConstantInfiltration): + return NormalizedInfiltration( + InfiltrationModelType.CONSTANT, (("infiltration", value.rate),) + ) + assert isinstance(value, GreenAmptInfiltration) + maps = { + "effective_porosity": value.effective_porosity, + "capillary_pressure": value.capillary_pressure, + "hydraulic_conductivity": value.hydraulic_conductivity, + } + if value.soil_water_content is not None: + maps["soil_water_content"] = value.soil_water_content + return NormalizedInfiltration(InfiltrationModelType.GREEN_AMPT, tuple(sorted(maps.items()))) + + +def _selected_options(options: OptionSweepConfig, selected: dict[str, Any]) -> dict[str, float]: + values = dict(SURFACE_FLOW_DEFAULTS) + values["dtinf"] = DefaultValues.DTINF + for key in type(options).model_fields: + value = selected.get(f"options.{key}", getattr(options, key)) + if value is not None: + assert isinstance(value, float) + values[key] = value + return values + + +def _selected_drainage( + drainage: DrainageSweepConfig | None, selected: dict[str, Any] +) -> dict[str, str | float] | None: + if drainage is None: + return None + values: dict[str, str | float] = {} + for key in type(drainage).model_fields: + value = selected.get(f"drainage.{key}", getattr(drainage, key)) + assert not isinstance(value, list), "drainage must be scalar after expansion" + values[key] = value + return values + + +def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: + raster_prefix = document.outputs.rasters.prefix if document.outputs.rasters else None + variables = tuple(document.outputs.rasters.variables) if document.outputs.rasters else () + statistics_file = document.outputs.statistics.file if document.outputs.statistics else None + drainage_dataset = ( + document.outputs.drainage.vector_dataset if document.outputs.drainage else None + ) + hotstart_file = document.hotstart.file if document.hotstart else None + interval = parse_iso_duration(document.hotstart.wallclock_step) if document.hotstart else None + for template in (raster_prefix, statistics_file, drainage_dataset, hotstart_file): + if template is not None: + render_template(template, ensemble=document.ensemble.id, simulation="simulation") + return OutputTemplates( + raster_prefix=raster_prefix, + raster_variables=variables, + statistics_file=statistics_file, + drainage_dataset=drainage_dataset, + hotstart_file=hotstart_file, + hotstart_interval=interval, + ) + + +def render_template(template: str, *, ensemble: str, simulation: str | None) -> str: + """Render the intentionally small output-template language.""" + formatter = string.Formatter() + values = {"ensemble": ensemble, "simulation": simulation} + try: + fragments = formatter.parse(template) + rendered: list[str] = [] + for literal, field_name, format_spec, conversion in fragments: + rendered.append(literal) + if field_name is None: + continue + if field_name not in values: + raise EnsembleError(f"unsupported template placeholder {{{field_name}}}") + if values[field_name] is None: + raise EnsembleError(f"{{{field_name}}} is not permitted in this template") + if format_spec or conversion: + raise EnsembleError( + "template format specifications and conversions are not supported" + ) + rendered.append(str(values[field_name])) + except ValueError as error: + raise EnsembleError(f"invalid output template: {error}") from error + return "".join(rendered) + + +def _freeze_json(value: Any) -> JsonValue: + if isinstance(value, BaseModel): + return _freeze_json(value.model_dump(mode="python")) + if isinstance(value, dict): + return tuple((str(key), _freeze_json(item)) for key, item in sorted(value.items())) + if isinstance(value, list | tuple): + return tuple(_freeze_json(item) for item in value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + raise TypeError(f"unsupported coordinate value {value!r}") + + +def check_batch_limits(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a valid batch that is too large before any resolver is started.""" + count = sum(len(ensemble.simulations) for ensemble in ensembles) + if count > MAX_BATCH_MEMBERS: + raise EnsembleError( + f"batch expands to {count} simulations, exceeding the {MAX_BATCH_MEMBERS} limit" + ) + + +def check_unique_ensemble_ids(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a batch-global duplicate ensemble identifier.""" + seen: set[str] = set() + for ensemble in ensembles: + if ensemble.ensemble_id in seen: + raise EnsembleError(f"duplicate ensemble ID {ensemble.ensemble_id!r}") + seen.add(ensemble.ensemble_id) diff --git a/src/itzi/grass_session.py b/src/itzi/grass_session.py index 492a0a05..e5ba6b7d 100644 --- a/src/itzi/grass_session.py +++ b/src/itzi/grass_session.py @@ -18,6 +18,7 @@ import os import subprocess import sys +from pathlib import Path from pydantic import BaseModel, ConfigDict @@ -43,15 +44,59 @@ class GrassSessionManager: def __init__(self, grass_params: GrassParams): self.grass_params = grass_params self.grass_session = None - if importlib.util.find_spec("grass"): - self._is_active = True - else: - self._is_active = False + self._owns_session = False + + @staticmethod + def current_context() -> tuple[str, str, str] | None: + """Return the active GRASS context, or ``None`` outside a session.""" + # Importing PyGRASS without a GISRC can terminate through GRASS' C API. + # The environment guard keeps configuration parsing and external launches + # independent from GRASS imports. + if not os.environ.get("GISRC"): + return None + if importlib.util.find_spec("grass") is None and "grass" not in sys.modules: + return None + try: + from grass.pygrass.utils import getenv + + values = (getenv("GISDBASE"), getenv("LOCATION_NAME"), getenv("MAPSET")) + except (ImportError, RuntimeError, SystemExit): + return None + if not all(values): + return None + return values + + def _validate_requested_context(self) -> None: + """Ensure an explicit request matches the session selected for this process.""" + active = self.current_context() + if active is None: + msgr.fatal("Unable to determine the active GRASS context") + requested = ( + self.grass_params.grassdata, + self.grass_params.location, + self.grass_params.mapset, + ) + if not any(requested): + return + if not all(requested): + msgr.fatal("GRASS database, location, and mapset must be supplied together") + requested_database = str(Path(requested[0]).expanduser().resolve()) + active_database = str(Path(active[0]).expanduser().resolve()) + if (requested_database, requested[1], requested[2]) != ( + active_database, + active[1], + active[2], + ): + msgr.fatal( + "Requested GRASS context does not match the active session " + f"({active_database}/{active[1]}/{active[2]})" + ) def open(self): """Open a GRASS session if needed.""" - if self._is_active: - return # Already started + if self.current_context() is not None: + self._validate_requested_context() + return # Check if mandatory GRASS parameters are present if not all( @@ -87,17 +132,21 @@ def open(self): self.grass_session = gscript.setup.init( path=gisdb, location=location, mapset=mapset, grass_path=grassbin ) - - self._is_active = True + self._owns_session = True + # GRASS 8.4 setup establishes GISRC. The fallback retains support for + # embedders whose setup object deliberately hides its environment. + if self.current_context() is not None: + self._validate_requested_context() def close(self): """Stop GRASS session.""" - if self.grass_session is not None and self._is_active: + if self.grass_session is not None and self._owns_session: try: self.grass_session.finish() except Exception as e: print(f"Warning: Error cleaning up GRASS session: {e}") - self._is_active = False + self.grass_session = None + self._owns_session = False def __enter__(self): self.open() diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 084f2aa9..adc9c168 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -30,26 +30,33 @@ import sys import time from collections.abc import Callable -from datetime import datetime, timedelta +from datetime import timedelta from importlib.metadata import version -from multiprocessing import Process -from typing import TYPE_CHECKING - -import numpy as np -from itzi_core import SimulationBuilder -from itzi_core.providers import CSVMassBalanceOutputProvider +from multiprocessing import Process, get_context +from pathlib import Path +from queue import Empty import itzi.messenger as msgr from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader +from itzi.ensemble import ( + DocumentFailure, + EnsembleError, + ExpandedEnsemble, + ExpandedSimulation, + ResolvedSimulation, + ValidationFailure, + render_template, +) from itzi.grass_session import GrassSessionManager from itzi.messenger import VerbosityLevel - -if TYPE_CHECKING: - from itzi_core import Simulation, SimulationConfig - - from itzi.grass_session import GrassParams - from itzi.providers.grass_interface import GrassInterface +from itzi.resolution import ( + resolve_simulation, + validate_resolved_ensemble, + verify_resolved_simulation, +) +from itzi.run_plan import load_batch +from itzi.simulation_runner import SimulationRunner def main(argv: list[str] | None = None) -> int: @@ -69,140 +76,6 @@ def main(argv: list[str] | None = None) -> int: return 0 -class SimulationRunner: - """Provide the necessary tools to run one simulation.""" - - def __init__( - self, - sim_config: SimulationConfig, - grass_params: GrassParams, - hotstart_path: str | None = None, - stats_file: str | None = None, - ) -> None: - self.grass_required_version = "8.4.0" - self.g_interface: GrassInterface - self.sim: Simulation - - # display parameters (if verbose) - msgr.display_sim_param(sim_config) - - # Check GRASS version - import grass.script as gscript - - grass_version = gscript.parse_command("g.version", flags="g")["version"] - if grass_version < self.grass_required_version: - msgr.fatal( - f"Itzi requires at least GRASS {self.grass_required_version}, " - f"version {grass_version} detected." - ) - msgr.debug("GRASS session set") - - # return error if output files exist - from itzi.providers import grass_interface - - output_names = list(sim_config.output_map_names.values()) - if sim_config.drainage_output is not None: - output_names.append(sim_config.drainage_output) - grass_interface.check_output_files(output_names) - msgr.debug("Output files OK") - - data_type = np.float32 - # Create the grass_interface object - self.g_interface = grass_interface.GrassInterface( - start_time=sim_config.start_time, - end_time=sim_config.end_time, - dtype=data_type, - region_id=grass_params.region, - raster_mask_id=grass_params.mask, - non_blocking_write=False, - ) - # Create Simulation with GRASS backend - msgr.verbose("Setting up GRASS simulation...") - from itzi.providers.grass_input import GrassRasterInputProvider - from itzi.providers.grass_output import ( - GrassRasterOutputProvider, - GrassVectorOutputProvider, - ) - - raster_input_provider = GrassRasterInputProvider( - { - "grass_interface": self.g_interface, - "input_map_names": sim_config.input_map_names, - "default_start_time": sim_config.start_time, - "default_end_time": sim_config.end_time, - } - ) - raster_output_provider = GrassRasterOutputProvider( - { - "grass_interface": self.g_interface, - "out_map_names": sim_config.output_map_names, - "hmin": sim_config.surface_flow_parameters.hmin, - "temporal_type": sim_config.temporal_type, - } - ) - sim_builder = ( - SimulationBuilder(sim_config, self.g_interface.get_npmask(), data_type) - .with_input_provider(raster_input_provider) - .with_raster_output_provider(raster_output_provider) - ) - if sim_config.drainage_output is not None: - vector_output_provider = GrassVectorOutputProvider( - { - "grass_interface": self.g_interface, - "temporal_type": sim_config.temporal_type, - "drainage_map_name": sim_config.drainage_output, - } - ) - sim_builder.with_vector_output_provider(vector_output_provider) - if stats_file: - sim_builder.with_mass_balance_output_provider(CSVMassBalanceOutputProvider(stats_file)) - if hotstart_path: - sim_builder.with_hotstart(hotstart_path) - self.sim: Simulation = sim_builder.build() - # Initialize the simulation - self.sim.initialize() - - def run(self): - """Run a full simulation""" - sim_start_time = datetime.now() - msgr.verbose("Starting time-stepping...") - while self.sim.sim_time < self.sim.end_time: - # display advance of simulation - msgr.percent( - self.sim.start_time, - self.sim.end_time, - self.sim.sim_time, - sim_start_time, - ) - # step models - self.step() - return self - - def finalize(self): - """Tear down the simulation and return to previous state.""" - self.sim.finalize() - # Cleanup the grass interface object - if hasattr(self, "g_interface"): - self.g_interface.finalize() - self.g_interface.cleanup() - return self - - def step(self): - """Do one simulation step.""" - self.sim.update() - return self - - @property - def origin(self): - return (self.sim.domain_data.north, self.sim.domain_data.west) - - def __del__(self): - # Cleanup the grass interface object - if hasattr(self, "g_interface"): - self.g_interface.finalize() - self.g_interface.cleanup() - - def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: """Run one simulation""" msgr.raise_on_error = True @@ -232,6 +105,45 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: raise SystemExit(1) from None +def resolved_sim_runner_worker(simulation: ResolvedSimulation, result_queue) -> None: + """Run one resolved member and return a small structured worker result.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + runner: SimulationRunner | None = None + try: + with GrassSessionManager(simulation.grass_params): + verify_resolved_simulation(simulation) + runner = SimulationRunner( + simulation.simulation_config, + simulation.grass_params, + stats_file=str(simulation.artifacts.statistics_file) + if simulation.artifacts.statistics_file is not None + else None, + effective_mask=simulation.effective_mask, + input_kinds=dict(simulation.input_kinds), + exclusive_stats=True, + ) + runner.run().finalize() + result_queue.put(("completed", None)) + except Exception as error: + if runner is not None: + try: + runner.finalize() + except Exception: + pass + result_queue.put(("execution_failed", f"{type(error).__name__}: {error}")) + + +def resolver_worker(expanded: ExpandedSimulation, result_queue) -> None: + """Resolve one scalar member in a short-lived spawned GRASS process.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + try: + result_queue.put(("resolved", resolve_simulation(expanded))) + except Exception as error: + result_queue.put(("validation_failed", f"{type(error).__name__}: {error}")) + + def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: """Run a simulation in a subprocess""" worker_args = (conf_file, hotstart_file) @@ -341,6 +253,25 @@ def itzi_run(cli_args): # only warnings os.environ["GRASS_VERBOSE"] = "0" + if _uses_legacy_execution_path(cli_args): + _run_legacy_batch(cli_args) + return + _run_ensemble_batch(cli_args) + + +def _uses_legacy_execution_path(cli_args) -> bool: + """Keep the one-simulation INI execution API during its deprecation window.""" + return ( + all(Path(path).suffix != ".yaml" for path in cli_args.config_file) + and not getattr(cli_args, "member", []) + and not getattr(cli_args, "dry", False) + ) + + +def _run_legacy_batch(cli_args) -> None: + """Execute the retained pre-ensemble INI path.""" + for config_file in cli_args.config_file: + msgr.warning(f"INI configuration <{config_file}> is deprecated; use YAML instead.") # start total time counter total_sim_start = time.time() # dictionary to store computation times @@ -376,6 +307,348 @@ def itzi_run(cli_args): msgr.fatal(f"{len(failed_files)} simulation(s) failed") +def _run_ensemble_batch(cli_args) -> None: + """Resolve and execute an ordered YAML/INI batch through spawn boundaries.""" + if getattr(cli_args, "resume_from", []): + msgr.fatal("Resume is not available for YAML ensembles until Stage 2") + ensembles, document_failures = load_batch(cli_args.config_file) + for failure in document_failures: + msgr.warning(failure.format()) + + resolved: list[ + tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] + ] = [] + for ensemble in ensembles: + successful: list[ResolvedSimulation] = [] + failures: list[ValidationFailure] = [] + for expanded in ensemble.simulations: + result = _resolve_in_spawn(expanded) + if isinstance(result, ResolvedSimulation): + successful.append(result) + else: + failures.append(result) + try: + validate_resolved_ensemble(tuple(successful)) + except EnsembleError as error: + failures.extend( + ValidationFailure( + source=simulation.source, + ensemble_id=ensemble.ensemble_id, + coordinates=simulation.coordinates, + phase="artifact_validation", + detail=str(error), + ) + for simulation in successful + ) + successful = [] + resolved.append((ensemble, tuple(successful), tuple(failures))) + + selected = _select_members(resolved, getattr(cli_args, "member", [])) + if getattr(cli_args, "dry", False): + _display_dry_plan(resolved, selected, document_failures) + if document_failures or any(failures for _, _, failures in resolved): + msgr.fatal("YAML batch validation failed") + return + + failure_count = len(document_failures) + for ensemble, simulations, failures in resolved: + selected_ids = selected.get(ensemble.ensemble_id, set()) + if not selected_ids and not (not getattr(cli_args, "member", []) and failures): + continue + manifest_path = _manifest_path(ensemble) + states = _initial_member_states( + simulations, failures, selected_ids, bool(getattr(cli_args, "member", [])) + ) + try: + _validate_manifest_destination(manifest_path, ensemble, simulations) + except EnsembleError as error: + msgr.warning(str(error)) + failure_count += max(1, len(selected_ids)) + continue + try: + _create_manifest(manifest_path, ensemble, states, overwrite=bool(cli_args.o)) + except OSError as error: + msgr.warning(f"Cannot create manifest <{manifest_path}>: {error}") + failure_count += max(1, len(selected_ids)) + continue + + for simulation in simulations: + if simulation.simulation_id not in selected_ids: + continue + if states[simulation.simulation_id]["status"] != "planned": + continue + if ( + simulation.artifacts.statistics_file is not None + and simulation.artifacts.statistics_file.exists() + and not cli_args.o + ): + states[simulation.simulation_id]["status"] = "validation_failed" + states[simulation.simulation_id]["failure"] = { + "phase": "artifact_validation", + "detail": f"statistics file <{simulation.artifacts.statistics_file}> already exists", + } + _update_manifest(manifest_path, ensemble, states) + continue + states[simulation.simulation_id]["status"] = "running" + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + started = time.monotonic() + status, detail = _run_resolved_in_spawn(simulation) + states[simulation.simulation_id]["status"] = status + states[simulation.simulation_id]["elapsed_seconds"] = time.monotonic() - started + if detail is not None: + states[simulation.simulation_id]["failure"] = detail + failure_count += 1 + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + failure_count += sum( + 1 + for state in states.values() + if state["status"] == "validation_failed" and state.get("selected", False) + ) + + if failure_count: + msgr.fatal(f"{failure_count} ensemble simulation(s) failed validation or execution") + msgr.message("Simulation(s) complete.") + + +def _resolve_in_spawn(expanded: ExpandedSimulation) -> ResolvedSimulation | ValidationFailure: + context = get_context("spawn") + result_queue = context.Queue() + process = context.Process(target=resolver_worker, args=(expanded, result_queue)) + process.start() + process.join() + try: + status, payload = result_queue.get(timeout=1) + except Empty: + status = "validation_failed" + payload = f"resolver exited with status {process.exitcode} without returning a result" + finally: + result_queue.close() + process.close() + if status == "resolved" and isinstance(payload, ResolvedSimulation): + return payload + return ValidationFailure( + source=expanded.source, + ensemble_id=expanded.ensemble_id, + coordinates=expanded.coordinates, + phase="input_resolution", + detail=str(payload), + ) + + +def _run_resolved_in_spawn(simulation: ResolvedSimulation) -> tuple[str, str | None]: + context = get_context("spawn") + result_queue = context.Queue() + process = context.Process(target=resolved_sim_runner_worker, args=(simulation, result_queue)) + process.start() + process.join() + try: + status, detail = result_queue.get(timeout=1) + except Empty: + reason = ( + f"signal {-process.exitcode}" + if process.exitcode and process.exitcode < 0 + else str(process.exitcode) + ) + status, detail = ( + "execution_failed", + f"worker exited with {reason} without returning a result", + ) + finally: + result_queue.close() + process.close() + return status, detail + + +def _select_members( + resolved: list[ + tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] + ], + selectors: list[str], +) -> dict[str, set[str]]: + """Resolve member selectors only against successfully resolved members.""" + selected = {ensemble.ensemble_id: set() for ensemble, _, _ in resolved} + all_members = { + f"{ensemble.ensemble_id}#{simulation.simulation_id}": simulation.simulation_id + for ensemble, simulations, _ in resolved + for simulation in simulations + } + if not selectors: + for ensemble, simulations, _ in resolved: + selected[ensemble.ensemble_id] = { + simulation.simulation_id for simulation in simulations + } + return selected + if any("#" not in selector for selector in selectors) and len(resolved) != 1: + msgr.fatal("Unqualified --member IDs require a batch with exactly one ensemble") + for selector in selectors: + qualified = selector if "#" in selector else f"{resolved[0][0].ensemble_id}#{selector}" + simulation_id = all_members.get(qualified) + if simulation_id is None: + msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") + ensemble_id, _separator, _member = qualified.partition("#") + selected[ensemble_id].add(simulation_id) + return selected + + +def _manifest_path(ensemble: ExpandedEnsemble) -> Path: + if ensemble.manifest_template is not None: + rendered = render_template( + ensemble.manifest_template, ensemble=ensemble.ensemble_id, simulation=None + ) + path = Path(rendered).expanduser() + return (path if path.is_absolute() else ensemble.source.path.parent / path).resolve() + return ( + ensemble.source.path.parent + / f"{ensemble.source.path.stem}.{ensemble.ensemble_id}.manifest.json" + ) + + +def _validate_manifest_destination( + manifest_path: Path, + ensemble: ExpandedEnsemble, + simulations: tuple[ResolvedSimulation, ...], +) -> None: + """Keep the parent-owned manifest away from files required by the run.""" + protected = {ensemble.source.path.resolve()} + for simulation in simulations: + if simulation.simulation_config.swmm_inp is not None: + protected.add(simulation.simulation_config.swmm_inp.resolve()) + if simulation.artifacts.statistics_file is not None: + protected.add(simulation.artifacts.statistics_file.resolve()) + if manifest_path in protected: + raise EnsembleError( + f"manifest <{manifest_path}> aliases a protected input or member artifact" + ) + for path in protected: + if manifest_path.exists() and path.exists() and manifest_path.samefile(path): + raise EnsembleError( + f"manifest <{manifest_path}> aliases a protected input or member artifact" + ) + + +def _initial_member_states( + simulations: tuple[ResolvedSimulation, ...], + failures: tuple[ValidationFailure, ...], + selected_ids: set[str], + has_selectors: bool, +) -> dict[str, dict]: + states: dict[str, dict] = {} + for simulation in simulations: + selected = simulation.simulation_id in selected_ids + states[simulation.simulation_id] = { + "simulation_id": simulation.simulation_id, + "coordinates": simulation.coordinates, + "status": "planned" if selected else "not_selected", + "selected": selected, + "artifacts": { + "rasters": dict(simulation.artifacts.output_map_names), + "drainage": simulation.artifacts.drainage_output, + "statistics": str(simulation.artifacts.statistics_file) + if simulation.artifacts.statistics_file is not None + else None, + }, + } + for index, failure in enumerate(failures): + key = f"validation-{index}" + states[key] = { + "simulation_id": None, + "coordinates": failure.coordinates, + "status": "not_selected" if has_selectors else "validation_failed", + "selected": not has_selectors, + "failure": {"phase": failure.phase, "detail": failure.detail}, + } + return states + + +def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: + return { + "manifest_version": 1, + "ensemble": {"id": ensemble.ensemble_id, "name": ensemble.ensemble_name}, + "source": { + "path": str(ensemble.source.path), + "document_index": ensemble.source.document_index, + "file_digest": ensemble.source.file_digest, + "document_digest": ensemble.source.document_digest, + }, + "members": list(states.values()), + } + + +def _create_manifest( + path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict], *, overwrite: bool +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + mode = "w" if overwrite else "x" + with path.open(mode, encoding="utf-8") as file_obj: + import json + + json.dump( + _manifest_document(ensemble, states), + file_obj, + allow_nan=False, + indent=2, + sort_keys=True, + ) + file_obj.write("\n") + + +def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: + import json + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as file_obj: + temporary_path = Path(file_obj.name) + json.dump( + _manifest_document(ensemble, states), + file_obj, + allow_nan=False, + indent=2, + sort_keys=True, + ) + file_obj.write("\n") + file_obj.flush() + os.fsync(file_obj.fileno()) + try: + temporary_path.replace(path) + except Exception: + temporary_path.unlink(missing_ok=True) + raise + + +def _display_dry_plan( + resolved: list[ + tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] + ], + selected: dict[str, set[str]], + document_failures: tuple[DocumentFailure, ...], +) -> None: + for ensemble, simulations, failures in resolved: + msgr.message(f"Ensemble {ensemble.ensemble_id}:") + for simulation in simulations: + selection = ( + "selected" + if simulation.simulation_id in selected[ensemble.ensemble_id] + else "not selected" + ) + msgr.message(f" {simulation.simulation_id} ({selection})") + for failure in failures: + msgr.warning(f" validation failed: {failure.detail}") + for failure in document_failures: + msgr.warning(failure.format()) + + def itzi_version(cli_args): """Display the software version number from the installed version""" print(version("itzi")) diff --git a/src/itzi/providers/__init__.py b/src/itzi/providers/__init__.py index e69de29b..86b486b1 100644 --- a/src/itzi/providers/__init__.py +++ b/src/itzi/providers/__init__.py @@ -0,0 +1,3 @@ +from itzi.providers.csv_output import ( + ExclusiveCSVMassBalanceOutputProvider as ExclusiveCSVMassBalanceOutputProvider, +) diff --git a/src/itzi/providers/csv_output.py b/src/itzi/providers/csv_output.py new file mode 100644 index 00000000..b2b68de5 --- /dev/null +++ b/src/itzi/providers/csv_output.py @@ -0,0 +1,37 @@ +"""Application-owned CSV output with explicit first-creation ownership.""" + +from __future__ import annotations + +import csv +import numbers +from pathlib import Path + +from itzi_core.data_containers import MassBalanceData +from itzi_core.providers.base import MassBalanceOutputProvider + + +class ExclusiveCSVMassBalanceOutputProvider(MassBalanceOutputProvider): + """Write statistics without silently truncating a pre-existing destination.""" + + def __init__( + self, file_name: str | Path, *, overwrite: bool, float_format: str = ".3f" + ) -> None: + self.fields = list(MassBalanceData.model_fields.keys()) + self.file_name = Path(file_name) + self.file_name.parent.mkdir(parents=True, exist_ok=True) + self.float_format = float_format + mode = "w" if overwrite else "x" + with self.file_name.open(mode, newline="") as file_obj: + csv.DictWriter(file_obj, fieldnames=self.fields).writeheader() + + def log(self, report_data: MassBalanceData) -> None: + line_to_write = {} + for key, value in report_data.model_dump().items(): + if value != value: # noqa: PLR0124 + line_to_write[key] = "-" + elif isinstance(value, numbers.Real) and not isinstance(value, int): + line_to_write[key] = format(value, self.float_format) + else: + line_to_write[key] = value + with self.file_name.open("a", newline="") as file_obj: + csv.DictWriter(file_obj, fieldnames=self.fields).writerow(line_to_write) diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 4a6f7211..9322f571 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,7 +15,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, Literal, TypedDict import numpy as np from itzi_core import ARRAY_DEFINITIONS, ArrayCategory, DomainData @@ -36,6 +36,7 @@ class GrassRasterInputConfig(TypedDict): input_map_names: Mapping[str, str | None] default_start_time: datetime default_end_time: datetime + input_kinds: Mapping[str, Literal["raster", "strds"]] | None class GrassRasterInputProvider(RasterInputProvider): @@ -43,6 +44,7 @@ def __init__(self, config: GrassRasterInputConfig) -> None: self.grass_interface = config["grass_interface"] self.start_time = config["default_start_time"] self.end_time = config["default_end_time"] + self.input_kinds = config.get("input_kinds") self.map_lists = self._get_map_lists(config["input_map_names"]) def get_domain_data(self) -> DomainData: @@ -81,7 +83,25 @@ def _get_map_lists( map_list = None continue map_id = self.grass_interface.format_id(map_name) - if self.grass_interface.name_is_stds(map_id): + expected_kind = self.input_kinds.get(k) if self.input_kinds is not None else None + if expected_kind == "strds": + if not self.grass_interface.name_is_stds(map_id): + msgr.fatal(f"Resolved STRDS input <{map_id}> is no longer available") + strds_id = map_id + if not self.grass_interface.stds_temporal_sanity(strds_id): + msgr.fatal(f"{map_name}: inadequate temporal format") + map_list = self.grass_interface.raster_list_from_strds(strds_id) + elif expected_kind == "raster": + if not self.grass_interface.name_is_map(map_id): + msgr.fatal(f"Resolved raster input <{map_id}> is no longer available") + map_list = [ + MapData( + id=map_id, + start_time=self.start_time, + end_time=self.end_time, + ) + ] + elif self.grass_interface.name_is_stds(map_id): strds_id = map_id if not self.grass_interface.stds_temporal_sanity(strds_id): msgr.fatal(f"{map_name}: inadequate temporal format") diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index f3610e86..f9a4ab03 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -182,6 +182,7 @@ def __init__( dtype, region_id: str | None, raster_mask_id: str | None, + effective_mask: tuple[str, str | None] | None = None, non_blocking_write=True, ) -> None: assert isinstance(start_time, datetime), "start_time not a datetime object!" @@ -195,8 +196,6 @@ def __init__( self.dtype = dtype self.non_blocking_write = non_blocking_write - self.old_mask_name = None - # LatLon is not supported if gscript.locn_is_latlong(): msgr.fatal("latlong location is not supported. Please use a projected location") @@ -218,9 +217,7 @@ def __init__( "n": self.region.north, "s": self.region.south, } - # Set temporary mask - if self.raster_mask_id: - self.set_temp_mask() + self.mask_mode, self.mask_source = self._select_effective_mask(effective_mask) self.overwrite = gscript.overwrite() # init temporal module _init_temporal() @@ -252,10 +249,7 @@ def finalize(self) -> None: self.raster_writer_queue.join() def cleanup(self) -> None: - """Remove temporary region and mask.""" - if self.raster_mask_id: - msgr.debug("Remove temp MASK...") - self.del_temp_mask() + """Remove the temporary region and stop the raster writer.""" if self.region_id: msgr.debug("Remove temp region...") gscript.del_temp_region() @@ -296,45 +290,44 @@ def to_datetime(self, unit: str, time: int) -> datetime: @staticmethod def has_mask() -> bool: """Return True if the mapset has a mask, False otherwise.""" - return bool(gscript.read_command("g.list", type="raster", pattern="MASK")) + return bool(gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET"))) + + def _select_effective_mask( + self, effective_mask: tuple[str, str | None] | None + ) -> tuple[str, str | None]: + """Select a mask descriptor without modifying the mapset-wide MASK.""" + if effective_mask is not None: + mode, source = effective_mask + if mode == "none": + return mode, None + if source is None: + msgr.fatal(f"Effective {mode} mask has no source") + return mode, source + if self.raster_mask_id: + return "explicit", self.format_id(self.raster_mask_id) + if self.has_mask(): + return "active", f"MASK@{gutils.getenv('MAPSET')}" + return "none", None def get_npmask(self) -> np.ndarray: """Return a boolean numpy ndarray where True is outside the domain.""" - if self.has_mask(): - grass_mask = self.read_raster_map("MASK") - return ~np.isclose(grass_mask, 1.0) - else: + if self.mask_mode == "none": return np.full(shape=(self.yr, self.xr), fill_value=False, dtype=np.bool_) + assert self.mask_source is not None + grass_mask = self.read_raster_map(self.mask_source) + if self.mask_mode == "explicit": + # r.mask accepts every non-NULL cell, including a zero value. + return np.isnan(grass_mask) + # A mapset MASK uses CELL semantics: zero and NULL are outside. + return np.isnan(grass_mask) | (grass_mask == 0) def set_temp_mask(self) -> Self: - """If a mask is already set, keep it for later. - Set a new mask. - """ - has_old_mask: bool = self.has_mask() - if has_old_mask: - # Save the current MASK under a temp name - self.old_mask_name = f"itzi_old_MASK_{os.getpid()}" - gscript.run_command( - "g.rename", - quiet=True, - overwrite=True, - raster=f"MASK,{self.old_mask_name}", - ) - gscript.run_command("r.mask", quiet=True, raster=self.raster_mask_id) - assert self.has_mask() + """Retained API: masks are now applied in NumPy without GRASS mutation.""" + msgr.fatal("Itzi never installs a temporary mapset MASK") return self def del_temp_mask(self) -> Self: - """Reset the old mask, remove if there was not.""" - if self.old_mask_name is not None: - gscript.run_command( - "g.rename", - quiet=True, - overwrite=True, - raster=f"{self.old_mask_name},MASK", - ) - else: - gscript.run_command("r.mask", quiet=True, flags="r") + """Retained API: Itzi never installs a temporary mapset MASK.""" return self def coor2pixel(self, coor: tuple[float, float]) -> tuple[int, int]: @@ -508,16 +501,56 @@ def validate_output_stds_temporal_type( ) def read_raster_map(self, rast_name: str) -> np.ndarray: - """Read a GRASS raster and return a numpy array""" + """Read a raster through GRASS 8.4's no-mask row APIs. + + The active mapset MASK must not alter model input values. The one + effective mask selected for the simulation is applied separately by + :meth:`get_npmask` and passed to itzi-core. + """ if self.non_blocking_write: - with self.raster_lock, raster.RasterRow(rast_name, mode="r") as rast: - array = np.array(rast, dtype=self.dtype) - array = self._replace_cell_null_sentinel(rast.mtype, array) - else: - with raster.RasterRow(rast_name, mode="r") as rast: - array = np.array(rast, dtype=self.dtype) - array = self._replace_cell_null_sentinel(rast.mtype, array) - return array + with self.raster_lock: + return self._read_raster_map_nomask(rast_name) + return self._read_raster_map_nomask(rast_name) + + def _read_raster_map_nomask(self, rast_name: str) -> np.ndarray: + """Read the current computational region while bypassing GRASS MASK.""" + from grass.lib import raster as libraster + + name, separator, mapset = rast_name.partition("@") + if separator and (not name or not mapset or "@" in mapset): + msgr.fatal(f"Invalid raster identifier <{rast_name}>") + if not separator: + mapset = gutils.get_mapset_raster(name) + if not mapset: + msgr.fatal(f"Raster <{rast_name}> not found") + raster_type = libraster.Rast_map_type(name, mapset) + reader_data = { + libraster.CELL_TYPE: (libraster.Rast_allocate_c_buf, libraster.Rast_get_c_row_nomask), + libraster.FCELL_TYPE: (libraster.Rast_allocate_f_buf, libraster.Rast_get_f_row_nomask), + libraster.DCELL_TYPE: (libraster.Rast_allocate_d_buf, libraster.Rast_get_d_row_nomask), + }.get(raster_type) + if reader_data is None: + msgr.fatal(f"Raster <{rast_name}> has an unsupported data type") + allocate, read_row = reader_data + file_descriptor = libraster.Rast_open_old(name, mapset) + if file_descriptor < 0: + msgr.fatal(f"Raster <{rast_name}> cannot be opened") + buffer = allocate() + try: + array = np.empty((self.yr, self.xr), dtype=self.dtype) + for row_index in range(self.yr): + read_row(file_descriptor, buffer, row_index) + array[row_index] = np.ctypeslib.as_array(buffer, shape=(self.xr,)) + finally: + libraster.Rast_close(file_descriptor) + return self._replace_cell_null_sentinel( + { + libraster.CELL_TYPE: "CELL", + libraster.FCELL_TYPE: "FCELL", + libraster.DCELL_TYPE: "DCELL", + }[raster_type], + array, + ) @staticmethod def _replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarray: @@ -597,7 +630,7 @@ def create_db_links( vect_map.dblinks.add(dblink) # create table dbtable = dblink.table() - dbtable.create(layer_dscr.columns, overwrite=True) + dbtable.create(layer_dscr.columns, overwrite=self.overwrite) dblinks[layer_name] = DBLinkDescription(dblink.layer, dbtable) return dblinks diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 0c239198..8fde9d6f 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -42,6 +42,18 @@ class GrassVectorOutputConfig(TypedDict): temporal_type: TemporalType +def derived_record_name(dataset_name: str, record_index: int) -> str: + """Return the shared raster/vector child name for one output record.""" + if record_index < 0: + raise ValueError("record_index must be non-negative") + return f"{dataset_name}_{record_index:04d}" + + +def derived_drainage_table_names(vector_name: str) -> tuple[str, str]: + """Return the table names created for a drainage vector child.""" + return (f"{vector_name}_node", f"{vector_name}_link") + + class GrassRasterOutputProvider(RasterOutputProvider): """Write simulation outputs to GRASS.""" @@ -65,8 +77,7 @@ def _write_array( self, array: np.ndarray, map_key: str, sim_time: datetime | timedelta ) -> None: """Write simulation data for current time step.""" - suffix = str(self.record_counter[map_key]).zfill(4) - map_name = f"{self.out_map_names[map_key]}_{suffix}" + map_name = derived_record_name(self.out_map_names[map_key], self.record_counter[map_key]) # write the raster self.grass_interface.write_raster_map(array, map_name, map_key, self.hmin) # Set depth values to null under the given threshold. Temporarily in gis.py @@ -128,8 +139,7 @@ def write_attributes( """Write drainage simulation data for current time step.""" if self.drainage_topology is None: raise RuntimeError("Drainage attributes cannot be written before topology.") - suffix = str(self.record_counter).zfill(4) - map_name = f"{self.drainage_map_name}_{suffix}" + map_name = derived_record_name(self.drainage_map_name, self.record_counter) self.grass_interface.write_vector_map(self.drainage_topology, attributes, map_name) self.vector_drainage_maplist.append((map_name, sim_time)) self.record_counter += 1 diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py new file mode 100644 index 00000000..bf5e180d --- /dev/null +++ b/src/itzi/resolution.py @@ -0,0 +1,476 @@ +"""GRASS-backed resolution of scalar ensemble members. + +Only functions in this module import GRASS, and only after a resolver or +execution worker has opened a session. The parent process exchanges the +immutable payloads defined in :mod:`itzi.ensemble`. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from pathlib import Path +from typing import Literal + +from itzi_core import DomainData, SimulationConfig, SurfaceFlowParameters +from pydantic import ValidationError + +from itzi.ensemble import ( + ArtifactSummary, + EffectiveMask, + EnsembleError, + ExpandedSimulation, + ResolvedSimulation, + format_iso_duration, + render_template, +) +from itzi.grass_session import GrassParams, GrassSessionManager + + +def resolve_simulation(expanded: ExpandedSimulation) -> ResolvedSimulation: + """Resolve one scalar member inside an already selected GRASS context.""" + requested_params = _requested_grass_params(expanded) + with GrassSessionManager(requested_params): + return _resolve_in_active_context(expanded, requested_params) + + +def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams: + context = expanded.domain.grass + if context is None or context.database is None: + return GrassParams(region=expanded.domain.region, mask=expanded.domain.mask) + return GrassParams( + grassdata=str(Path(context.database).expanduser().resolve()), + location=context.project, + mapset=context.mapset, + region=expanded.domain.region, + mask=expanded.domain.mask, + grass_bin=context.executable or "grass", + ) + + +def _resolve_in_active_context( + expanded: ExpandedSimulation, requested_params: GrassParams +) -> ResolvedSimulation: + """Resolve GRASS sources and create the final core configuration.""" + import grass.script as gscript + from grass.pygrass.gis import Mapset + from grass.pygrass.gis.region import Region + from grass.pygrass.utils import getenv + + actual_params = GrassParams( + grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), + location=getenv("LOCATION_NAME"), + mapset=getenv("MAPSET"), + region=requested_params.region, + mask=requested_params.mask, + grass_bin=requested_params.grass_bin, + ) + domain = _read_domain(expanded.domain.region, Region, gscript) + effective_mask = _resolve_effective_mask(expanded.domain.mask, getenv) + input_names, input_kinds = _resolve_inputs( + dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), + Mapset, + ) + swmm_path, swmm_digest = _resolve_swmm_input(expanded) + simulation_id, normalized_payload = _simulation_identity( + expanded, + actual_params, + domain, + effective_mask, + input_names, + swmm_digest, + ) + artifacts = _render_artifacts(expanded, simulation_id, actual_params.mapset) + simulation_config = _build_simulation_config( + expanded, + input_names, + artifacts, + swmm_path, + ) + return ResolvedSimulation( + source=expanded.source, + ensemble_id=expanded.ensemble_id, + simulation_id=simulation_id, + coordinates=expanded.coordinates, + grass_params=actual_params, + domain_data=domain, + effective_mask=effective_mask, + input_kinds=tuple(sorted(input_kinds.items())), + simulation_config=simulation_config, + artifacts=artifacts, + normalized_payload=normalized_payload, + ) + + +def _read_domain(region_id: str | None, region_type: type, gscript) -> DomainData: + """Read the selected computational region without leaving it changed.""" + if gscript.locn_is_latlong(): + raise EnsembleError("latlong locations are not supported") + using_temp_region = region_id is not None + if using_temp_region: + gscript.use_temp_region() + gscript.run_command("g.region", region=region_id) + try: + region = region_type() + if region.cols < 3 or region.rows < 3: + raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") + return DomainData( + north=region.north, + south=region.south, + east=region.east, + west=region.west, + rows=region.rows, + cols=region.cols, + crs_wkt=gscript.read_command("g.proj", flags="fw"), + ) + finally: + if using_temp_region: + gscript.del_temp_region() + + +def _split_identifier(identifier: str) -> tuple[str, str | None]: + name, separator, mapset = identifier.partition("@") + if separator and (not name or not mapset or "@" in mapset): + raise EnsembleError(f"invalid GRASS identifier {identifier!r}") + return name, mapset if separator else None + + +def _resolve_inputs( + inputs: dict[str, str], + mapset_type: type, +) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: + resolved: dict[str, str] = {} + kinds: dict[str, Literal["raster", "strds"]] = {} + for key, identifier in inputs.items(): + source, kind = _resolve_input_identifier(identifier, mapset_type) + resolved[key] = source + kinds[key] = kind + return resolved, kinds + + +def _resolve_input_identifier( + identifier: str, mapset_type: type +) -> tuple[str, Literal["raster", "strds"]]: + """Resolve both input namespaces and reject a visible ambiguity.""" + import grass.temporal as tgis + from grass.pygrass import utils as gutils + + tgis.init(raise_fatal_error=True) + tgis.set_raise_on_error(True) + + name, requested_mapset = _split_identifier(identifier) + raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") + raster_id = f"{name}@{raster_mapset}" if raster_mapset else None + visible_mapsets = [requested_mapset] if requested_mapset else mapset_type().visible.read() + strds_id = None + for mapset in visible_mapsets: + candidate = f"{name}@{mapset}" + try: + is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() + except BaseException as error: + # A visible mapset may have no temporal database or may be + # read-inaccessible. It cannot contribute a STRDS candidate. + if isinstance(error, (KeyboardInterrupt, GeneratorExit)): + raise + is_strds = False + if is_strds: + strds_id = candidate + break + if raster_id is not None and strds_id is not None: + raise EnsembleError( + f"input {identifier!r} is ambiguous: raster {raster_id} and STRDS {strds_id} both exist" + ) + if raster_id is not None: + return raster_id, "raster" + if strds_id is not None: + return strds_id, "strds" + raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") + + +def _resolve_effective_mask(mask: str | None, getenv) -> EffectiveMask: + from grass.pygrass import utils as gutils + + current_mapset = getenv("MAPSET") + if mask is not None: + name, requested_mapset = _split_identifier(mask) + mapset = gutils.get_mapset_raster(name, requested_mapset or "") + if not mapset: + raise EnsembleError(f"explicit mask {mask!r} was not found") + return EffectiveMask("explicit", f"{name}@{mapset}") + if gutils.get_mapset_raster("MASK", current_mapset): + return EffectiveMask("active", f"MASK@{current_mapset}") + return EffectiveMask("none", None) + + +def _resolve_swmm_input(expanded: ExpandedSimulation) -> tuple[Path | None, str | None]: + if expanded.drainage is None: + return None, None + drainage = dict(expanded.drainage) + configured = Path(str(drainage["swmm_input"])).expanduser() + path = configured if configured.is_absolute() else expanded.source.path.parent / configured + path = path.resolve() + if not path.is_file(): + raise EnsembleError(f"SWMM input file <{path}> not found") + return path, f"blake2b:{hashlib.blake2b(path.read_bytes(), digest_size=32).hexdigest()}" + + +def _simulation_identity( + expanded: ExpandedSimulation, + grass_params: GrassParams, + domain: DomainData, + effective_mask: EffectiveMask, + input_names: dict[str, str], + swmm_digest: str | None, +) -> tuple[str, str]: + """Build the versioned identity payload after all source normalization.""" + drainage = dict(expanded.drainage) if expanded.drainage is not None else None + if drainage is not None: + drainage["swmm_digest"] = swmm_digest + drainage["swmm_input"] = str( + (expanded.source.path.parent / str(drainage["swmm_input"])).resolve() + ) + time_payload: dict[str, str | None] = { + "temporal_type": str(expanded.time.temporal_type), + "start": expanded.time.start.isoformat() if expanded.time.start is not None else None, + "end": expanded.time.end.isoformat() if expanded.time.end is not None else None, + "duration": format_iso_duration(expanded.time.duration), + "record_step": format_iso_duration(expanded.time.record_step), + } + payload = { + "identity_version": 1, + "grass": { + "database": grass_params.grassdata, + "project": grass_params.location, + "mapset": grass_params.mapset, + }, + "domain": domain.model_dump(), + "mask": {"mode": effective_mask.mode, "source": effective_mask.source}, + "time": time_payload, + "inputs": input_names, + "infiltration": {"model": str(expanded.infiltration.model)}, + "options": dict(expanded.options), + "drainage": drainage, + } + canonical = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) + digest = hashlib.blake2b(canonical.encode(), digest_size=16).hexdigest() + return f"sim-{digest}", canonical + + +def _render_artifacts( + expanded: ExpandedSimulation, + simulation_id: str, + mapset: str | None, +) -> ArtifactSummary: + output_map_names: dict[str, str] = {} + if expanded.outputs.raster_prefix is not None: + prefix = render_template( + expanded.outputs.raster_prefix, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ) + output_map_names = { + variable: f"{prefix}_{variable}" for variable in expanded.outputs.raster_variables + } + drainage_output = ( + render_template( + expanded.outputs.drainage_dataset, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ) + if expanded.outputs.drainage_dataset is not None + else None + ) + stats_file = ( + _source_relative_path(expanded.outputs.statistics_file, expanded.source.path.parent) + if expanded.outputs.statistics_file is not None + else None + ) + _validate_output_names(output_map_names, drainage_output) + _validate_protected_file_output(stats_file, expanded.source.path, _drainage_path(expanded)) + return ArtifactSummary(tuple(sorted(output_map_names.items())), drainage_output, stats_file) + + +def _source_relative_path(value: str, source_dir: Path) -> Path: + path = Path(value).expanduser() + return (path if path.is_absolute() else source_dir / path).resolve() + + +def _drainage_path(expanded: ExpandedSimulation) -> Path | None: + if expanded.drainage is None: + return None + return _source_relative_path( + str(dict(expanded.drainage)["swmm_input"]), expanded.source.path.parent + ) + + +def _validate_protected_file_output( + output: Path | None, config_path: Path, swmm_path: Path | None +) -> None: + if output is None: + return + protected = (config_path.resolve(), *(path for path in (swmm_path,) if path is not None)) + for source in protected: + if output == source or (output.exists() and source.exists() and output.samefile(source)): + raise EnsembleError(f"output file <{output}> aliases protected input <{source}>") + + +def _validate_output_names(output_map_names: dict[str, str], drainage_output: str | None) -> None: + from grass.pygrass import utils as gutils + + from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name + + names = list(output_map_names.values()) + if drainage_output is not None: + names.append(drainage_output) + for name in names: + if "@" in name: + raise EnsembleError(f"GRASS output names must be unqualified: {name!r}") + if name == "MASK": + raise EnsembleError("MASK cannot be used as an output name") + if not gutils.is_clean_name(name): + raise EnsembleError(f"invalid GRASS output name {name!r}") + child = derived_record_name(name, 0) + if child == "MASK" or not gutils.is_clean_name(child): + raise EnsembleError(f"invalid derived GRASS output name {child!r}") + if drainage_output is not None: + child = derived_record_name(drainage_output, 0) + for table_name in derived_drainage_table_names(child): + if not gutils.is_clean_name(table_name): + raise EnsembleError(f"invalid derived drainage table name {table_name!r}") + + +def _build_simulation_config( + expanded: ExpandedSimulation, + input_names: dict[str, str], + artifacts: ArtifactSummary, + swmm_path: Path | None, +) -> SimulationConfig: + start = expanded.time.start + end = expanded.time.end + if start is None: + start = datetime.min # noqa: DTZ901 # Core's established relative-time adapter. + end = start + expanded.time.duration + assert end is not None + options = dict(expanded.options) + surface_options = {key: value for key, value in options.items() if key != "dtinf"} + drainage = dict(expanded.drainage) if expanded.drainage is not None else {} + try: + return SimulationConfig( + start_time=start, + end_time=end, + record_step=expanded.time.record_step, + temporal_type=expanded.time.temporal_type, + hotstart_config=None, + input_map_names=input_names, + output_map_names=dict(artifacts.output_map_names), + surface_flow_parameters=SurfaceFlowParameters(**surface_options), + dtinf=options["dtinf"], + infiltration_model=expanded.infiltration.model, + swmm_inp=swmm_path, + drainage_output=artifacts.drainage_output, + **{ + key: drainage[key] + for key in ("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") + if key in drainage + }, + ) + except ValidationError as error: + details = "; ".join(item["msg"] for item in error.errors(include_url=False)) + raise EnsembleError(details) from error + + +def validate_resolved_ensemble(simulations: tuple[ResolvedSimulation, ...]) -> None: + """Reject identity and artifact collisions among resolved ensemble members.""" + identities: dict[str, str] = {} + artifacts: dict[str, str] = {} + ensemble_sources: set[str] = set() + for simulation in simulations: + ensemble_sources.update(simulation.simulation_config.input_map_names.values()) + if simulation.effective_mask.source is not None: + ensemble_sources.add(simulation.effective_mask.source) + for simulation in simulations: + existing_payload = identities.get(simulation.simulation_id) + if existing_payload is not None: + if existing_payload == simulation.normalized_payload: + raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}") + raise EnsembleError(f"simulation ID digest collision {simulation.simulation_id}") + identities[simulation.simulation_id] = simulation.normalized_payload + mapset = simulation.grass_params.mapset + from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name + + for _, name in simulation.artifacts.output_map_names: + _claim_artifact(artifacts, f"{name}@{mapset}", simulation.simulation_id) + _reject_output_source_alias(name, mapset, ensemble_sources) + child = derived_record_name(name, 0) + _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) + _reject_output_source_alias(child, mapset, ensemble_sources) + if simulation.artifacts.drainage_output is not None: + drainage_name = simulation.artifacts.drainage_output + _claim_artifact( + artifacts, + f"{drainage_name}@{mapset}", + simulation.simulation_id, + ) + _reject_output_source_alias(drainage_name, mapset, ensemble_sources) + child = derived_record_name(drainage_name, 0) + _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) + _reject_output_source_alias(child, mapset, ensemble_sources) + for table_name in derived_drainage_table_names(child): + _claim_artifact(artifacts, f"table:{table_name}", simulation.simulation_id) + if simulation.artifacts.statistics_file is not None: + _claim_artifact( + artifacts, + str(simulation.artifacts.statistics_file), + simulation.simulation_id, + ) + + +def _claim_artifact(artifacts: dict[str, str], artifact: str, simulation_id: str) -> None: + owner = artifacts.setdefault(artifact, simulation_id) + if owner != simulation_id: + raise EnsembleError( + f"output artifact {artifact!r} is shared by simulations {owner} and {simulation_id}" + ) + + +def _reject_output_source_alias(name: str, mapset: str | None, sources: set[str]) -> None: + qualified = f"{name}@{mapset}" + if qualified in sources: + raise EnsembleError(f"output {qualified!r} aliases an ensemble input or mask") + + +def verify_resolved_simulation(simulation: ResolvedSimulation) -> None: + """Re-resolve worker facts and reject a changed GRASS environment.""" + import grass.script as gscript + from grass.pygrass.gis import Mapset + from grass.pygrass.gis.region import Region + from grass.pygrass.utils import getenv + + actual = GrassParams( + grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), + location=getenv("LOCATION_NAME"), + mapset=getenv("MAPSET"), + region=simulation.grass_params.region, + mask=simulation.grass_params.mask, + grass_bin=simulation.grass_params.grass_bin, + ) + if actual != simulation.grass_params: + raise EnsembleError("GRASS context changed after resolution") + domain = _read_domain(simulation.grass_params.region, Region, gscript) + if domain != simulation.domain_data: + raise EnsembleError("GRASS domain changed after resolution") + mask = _resolve_effective_mask(simulation.grass_params.mask, getenv) + if mask != simulation.effective_mask: + raise EnsembleError("effective GRASS mask changed after resolution") + _verify_input_kinds( + simulation.simulation_config.input_map_names, dict(simulation.input_kinds), Mapset + ) + + +def _verify_input_kinds( + inputs: dict[str, str], expected: dict[str, Literal["raster", "strds"]], mapset_type: type +) -> None: + resolved, kinds = _resolve_inputs(inputs, mapset_type) + if resolved != inputs or kinds != expected: + raise EnsembleError("resolved GRASS input sources changed after resolution") diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py new file mode 100644 index 00000000..ccae0fe8 --- /dev/null +++ b/src/itzi/run_plan.py @@ -0,0 +1,149 @@ +"""Pure ordered batch loading for YAML streams and deprecated INI files.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from itzi_core import TemporalType + +import itzi.messenger as msgr +from itzi.configreader import ConfigReader +from itzi.ensemble import ( + DIRECT_INPUT_KEYS, + GREEN_AMPT_KEYS, + DocumentFailure, + DomainConfig, + EnsembleError, + ExpandedEnsemble, + ExpandedSimulation, + GrassContextConfig, + NormalizedInfiltration, + NormalizedTime, + OutputTemplates, + SourceDocument, + check_batch_limits, + check_unique_ensemble_ids, + load_yaml_stream, +) + + +def load_batch( + config_paths: list[str], +) -> tuple[tuple[ExpandedEnsemble, ...], tuple[DocumentFailure, ...]]: + """Load an ordered mixed configuration batch without initializing GRASS.""" + ensembles: list[ExpandedEnsemble] = [] + failures: list[DocumentFailure] = [] + for raw_path in config_paths: + path = Path(raw_path) + if path.suffix == ".yaml": + stream = load_yaml_stream(path) + ensembles.extend(stream.ensembles) + failures.extend(stream.failures) + else: + msgr.warning(f"INI configuration <{path}> is deprecated; use YAML instead.") + try: + ensembles.append(_legacy_ensemble(path)) + except Exception as error: # noqa: BLE001 - preserve document-local continuation. + failures.append( + DocumentFailure( + path=path.expanduser().resolve(), + document_index=0, + line=None, + column=None, + phase="schema", + detail=str(error), + ) + ) + check_unique_ensemble_ids(tuple(ensembles)) + check_batch_limits(tuple(ensembles)) + return tuple(ensembles), tuple(failures) + + +def _legacy_ensemble(path: Path) -> ExpandedEnsemble: + """Adapt the public one-member INI reader to the scalar batch model.""" + reader = ConfigReader(str(path)) + config = reader.get_sim_params() + if config.hotstart_config is not None: + raise EnsembleError("hotstart output requires Stage 2 checkpoint support in mixed batches") + grass = reader.get_grass_params() + source_path = path.expanduser().resolve() + source_bytes = source_path.read_bytes() + source = SourceDocument( + path=source_path, + document_index=0, + start_line=1, + file_digest=hashlib.blake2b(source_bytes, digest_size=32).hexdigest(), + document_digest=hashlib.blake2b(b"legacy-v1\0" + source_bytes, digest_size=32).hexdigest(), + ) + ensemble_id = f"legacy-{hashlib.blake2b(str(source_path).encode(), digest_size=8).hexdigest()}" + context = None + if grass.grassdata is not None: + context = GrassContextConfig( + database=grass.grassdata, + project=grass.location, + mapset=grass.mapset, + executable=grass.grass_bin, + ) + input_names = config.input_map_names + infiltration_maps = { + key: input_names[key] for key in ("infiltration", *GREEN_AMPT_KEYS) if key in input_names + } + direct_inputs = {key: input_names[key] for key in DIRECT_INPUT_KEYS if key in input_names} + options = { + **config.surface_flow_parameters.model_dump(), + "dtinf": config.dtinf, + } + drainage = None + if config.swmm_inp is not None: + drainage = { + "swmm_input": str(config.swmm_inp), + "orifice_coeff": config.orifice_coeff, + "free_weir_coeff": config.free_weir_coeff, + "submerged_weir_coeff": config.submerged_weir_coeff, + } + if config.temporal_type == TemporalType.RELATIVE: + normalized_time = NormalizedTime( + TemporalType.RELATIVE, + None, + None, + config.end_time - config.start_time, + config.record_step, + False, + ) + else: + normalized_time = NormalizedTime( + TemporalType.ABSOLUTE, + config.start_time, + config.end_time, + config.end_time - config.start_time, + config.record_step, + False, + ) + stats_file = reader.get_stats_file() + if stats_file is not None: + # The INI deprecation release preserves cwd-relative destinations. + stats_file = str((Path.cwd() / stats_file).resolve()) + expanded = ExpandedSimulation( + source=source, + ensemble_id=ensemble_id, + ensemble_name=source_path.stem, + coordinates=(), + domain=DomainConfig(grass=context, region=grass.region, mask=grass.mask), + time=normalized_time, + input_maps=tuple(sorted(direct_inputs.items())), + infiltration=NormalizedInfiltration( + config.infiltration_model, tuple(sorted(infiltration_maps.items())) + ), + options=tuple(sorted(options.items())), + drainage=tuple(sorted(drainage.items())) if drainage is not None else None, + outputs=OutputTemplates( + raster_prefix=reader.out_prefix, + raster_variables=tuple(reader.out_values), + statistics_file=stats_file, + drainage_dataset=config.drainage_output, + hotstart_file=None, + hotstart_interval=None, + ), + ) + return ExpandedEnsemble(source, ensemble_id, source_path.stem, None, (expanded,)) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py new file mode 100644 index 00000000..dc59f4b4 --- /dev/null +++ b/src/itzi/simulation_runner.py @@ -0,0 +1,179 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from collections.abc import Mapping +from datetime import datetime +from typing import TYPE_CHECKING + +import numpy as np +from itzi_core import SimulationBuilder, SimulationConfig +from itzi_core.providers import CSVMassBalanceOutputProvider + +import itzi.messenger as msgr +from itzi.ensemble import EffectiveMask +from itzi.grass_session import GrassParams + +if TYPE_CHECKING: + from itzi_core import Simulation + + from itzi.providers.grass_interface import GrassInterface + + +class SimulationRunner: + """Provide the necessary tools to run one simulation.""" + + def __init__( + self, + sim_config: SimulationConfig, + grass_params: GrassParams, + hotstart_path: str | None = None, + stats_file: str | None = None, + effective_mask: EffectiveMask | None = None, + input_kinds: Mapping[str, str] | None = None, + exclusive_stats: bool = False, + ) -> None: + self.grass_required_version = "8.4.0" + self.g_interface: GrassInterface + self.sim: Simulation + + # display parameters (if verbose) + msgr.display_sim_param(sim_config) + + # Check GRASS version + import grass.script as gscript + + grass_version = gscript.parse_command("g.version", flags="g")["version"] + if grass_version < self.grass_required_version: + msgr.fatal( + f"Itzi requires at least GRASS {self.grass_required_version}, " + f"version {grass_version} detected." + ) + msgr.debug("GRASS session set") + + # return error if output files exist + from itzi.providers import grass_interface + + output_names = list(sim_config.output_map_names.values()) + if sim_config.drainage_output is not None: + output_names.append(sim_config.drainage_output) + grass_interface.check_output_files(output_names) + msgr.debug("Output files OK") + + data_type = np.float32 + # Create the grass_interface object + self.g_interface = grass_interface.GrassInterface( + start_time=sim_config.start_time, + end_time=sim_config.end_time, + dtype=data_type, + region_id=grass_params.region, + raster_mask_id=grass_params.mask, + effective_mask=(effective_mask.mode, effective_mask.source) + if effective_mask is not None + else None, + non_blocking_write=False, + ) + # Create Simulation with GRASS backend + msgr.verbose("Setting up GRASS simulation...") + from itzi.providers.grass_input import GrassRasterInputProvider + from itzi.providers.grass_output import ( + GrassRasterOutputProvider, + GrassVectorOutputProvider, + ) + + raster_input_provider = GrassRasterInputProvider( + { + "grass_interface": self.g_interface, + "input_map_names": sim_config.input_map_names, + "default_start_time": sim_config.start_time, + "default_end_time": sim_config.end_time, + "input_kinds": input_kinds, + } + ) + raster_output_provider = GrassRasterOutputProvider( + { + "grass_interface": self.g_interface, + "out_map_names": sim_config.output_map_names, + "hmin": sim_config.surface_flow_parameters.hmin, + "temporal_type": sim_config.temporal_type, + } + ) + sim_builder = ( + SimulationBuilder(sim_config, self.g_interface.get_npmask(), data_type) + .with_input_provider(raster_input_provider) + .with_raster_output_provider(raster_output_provider) + ) + if sim_config.drainage_output is not None: + vector_output_provider = GrassVectorOutputProvider( + { + "grass_interface": self.g_interface, + "temporal_type": sim_config.temporal_type, + "drainage_map_name": sim_config.drainage_output, + } + ) + sim_builder.with_vector_output_provider(vector_output_provider) + if stats_file: + if exclusive_stats: + from itzi.providers import ExclusiveCSVMassBalanceOutputProvider + + stats_provider = ExclusiveCSVMassBalanceOutputProvider( + stats_file, overwrite=self.g_interface.overwrite + ) + else: + stats_provider = CSVMassBalanceOutputProvider(stats_file) + sim_builder.with_mass_balance_output_provider(stats_provider) + if hotstart_path: + sim_builder.with_hotstart(hotstart_path) + self.sim: Simulation = sim_builder.build() + # Initialize the simulation + self.sim.initialize() + + def run(self): + """Run a full simulation""" + sim_start_time = datetime.now() + msgr.verbose("Starting time-stepping...") + while self.sim.sim_time < self.sim.end_time: + # display advance of simulation + msgr.percent( + self.sim.start_time, + self.sim.end_time, + self.sim.sim_time, + sim_start_time, + ) + # step models + self.step() + return self + + def finalize(self): + """Tear down the simulation and return to previous state.""" + self.sim.finalize() + # Cleanup the grass interface object + if hasattr(self, "g_interface"): + self.g_interface.finalize() + self.g_interface.cleanup() + return self + + def step(self): + """Do one simulation step.""" + self.sim.update() + return self + + @property + def origin(self): + return (self.sim.domain_data.north, self.sim.domain_data.west) + + def __del__(self): + # Cleanup the grass interface object + if hasattr(self, "g_interface"): + self.g_interface.finalize() + self.g_interface.cleanup() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 02f24cc9..3a9d0dd3 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -40,6 +40,15 @@ def test_run_parser_accepts_resume_from_args(): assert args.resume_from == [("a.ini", "restart_a.zip"), ("b.ini", "restart_b.zip")] +def test_run_parser_accepts_yaml_dry_run_and_member_selection(): + args = build_parser().parse_args( + ["run", "study.yaml", "--dry", "--member", "study#sim-a", "--member", "study#sim-b"] + ) + + assert args.dry is True + assert args.member == ["study#sim-a", "study#sim-b"] + + def test_run_parser_rejects_v_and_q_together(): with pytest.raises(SystemExit): build_parser().parse_args(["run", "a.ini", "-v", "-q"]) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py new file mode 100644 index 00000000..3cc7b970 --- /dev/null +++ b/tests/cli/test_ensemble_config.py @@ -0,0 +1,234 @@ +"""Pure YAML ensemble loader and expansion tests.""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +from itzi_core import InfiltrationModelType, TemporalType + +from itzi.ensemble import ( + MAX_ENSEMBLE_MEMBERS, + format_iso_duration, + load_yaml_stream, + parse_iso_duration, + render_template, +) + + +def write_yaml(tmp_path, content: str): + path = tmp_path / "study.yaml" + path.write_text(content, encoding="utf-8") + return path + + +def document(*, ensemble_id: str = "study", extra: str = "") -> str: + body = ( + extra + or """\ +input: + ground_elevation: "elevation" + friction: "manning" +options: {} +outputs: + rasters: + prefix: "results_{ensemble}_{simulation}" + variables: [water_depth] +""" + ) + return f"""\ +schema_version: 1 +ensemble: + id: {ensemble_id} +domain: {{}} +time: + duration: "PT2H" + record_step: "PT5M" +{body}""" + + +def test_scalar_document_expands_to_one_member(tmp_path): + stream = load_yaml_stream(write_yaml(tmp_path, document())) + + assert stream.failures == () + assert len(stream.ensembles) == 1 + simulation = stream.ensembles[0].simulations[0] + assert simulation.coordinates == () + assert simulation.time.temporal_type == TemporalType.RELATIVE + assert simulation.time.start is None + assert simulation.time.end is None + assert simulation.time.duration == timedelta(hours=2) + + +def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): + content = document( + extra="""\ +input: + ground_elevation: ["elevation_a", "elevation_b"] + friction: "manning" + rainfall_rate: ["rain_a", "rain_b"] +options: + cfl: [0.5, 0.7] +outputs: + rasters: + prefix: "out_{simulation}" + variables: [water_depth] +""" + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + simulations = stream.ensembles[0].simulations + assert len(simulations) == 8 + assert tuple(path for path, _ in simulations[0].coordinates) == ( + "input.ground_elevation", + "input.rainfall_rate", + "options.cfl", + ) + assert dict(simulations[0].coordinates)["options.cfl"] == 0.5 + assert dict(simulations[-1].coordinates)["options.cfl"] == 0.7 + + +def test_loader_continues_after_failed_explicit_document(tmp_path): + content = """\ +schema_version: 1 +ensemble: + id: first +domain: {} +time: + duration: "PT1H" + record_step: "PT5M" +input: + ground_elevation: elevation + friction: manning +options: + cfl: "0.7" +outputs: {} +--- +""" + document(ensemble_id="second") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert len(stream.failures) == 1 + assert stream.failures[0].document_index == 0 + assert stream.failures[0].phase == "schema" + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["second"] + + +def test_loader_rejects_duplicate_keys_and_keeps_later_document(tmp_path): + content = """\ +schema_version: 1 +schema_version: 1 +--- +""" + document(ensemble_id="second") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures[0].phase == "parse" + assert stream.failures[0].line == 2 + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["second"] + + +def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): + content = """\ +--- +--- +defaults: &defaults + cfl: 0.7 +schema_version: 1 +ensemble: + id: invalid-merge +domain: {} +time: + duration: "PT1H" + record_step: "PT5M" +input: + ground_elevation: elevation + friction: manning +options: + <<: *defaults +outputs: {} +--- +""" + document(ensemble_id="valid") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert [failure.phase for failure in stream.failures] == ["parse", "parse"] + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] + + +def test_infiltration_alternatives_are_explicit_and_labels_are_not_semantic(tmp_path): + content = document( + extra="""\ +input: + ground_elevation: elevation + friction: manning + infiltration: + - type: none + label: "first" + - type: constant + rate: infiltration_rate +options: {} +outputs: {} +""" + ) + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + simulations = stream.ensembles[0].simulations + assert [simulation.infiltration.model for simulation in simulations] == [ + InfiltrationModelType.NULL, + InfiltrationModelType.CONSTANT, + ] + + duplicate = content.replace( + " - type: constant\n rate: infiltration_rate", + ' - type: none\n label: "second"', + ) + duplicate_stream = load_yaml_stream(write_yaml(tmp_path, duplicate)) + assert duplicate_stream.failures[0].phase == "schema" + assert "duplicate semantic values" in duplicate_stream.failures[0].detail + + +def test_absolute_offsets_keep_wall_clock_values(tmp_path): + content = document().replace( + ' duration: "PT2H"\n record_step: "PT5M"', + ' start: "2026-09-01T00:00:00+02:00"\n duration: "PT2H"\n record_step: "PT5M"', + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + simulation = stream.ensembles[0].simulations[0] + assert simulation.time.temporal_type == TemporalType.ABSOLUTE + assert simulation.time.start == datetime(2026, 9, 1, 0, 0) # noqa: DTZ001 + assert simulation.time.end == datetime(2026, 9, 1, 2, 0) # noqa: DTZ001 + assert simulation.time.had_timezone_offset is True + + +def test_expansion_limit_is_checked_before_materialization(tmp_path): + values = ", ".join(f"{0.01 + index / 1000:.3f}" for index in range(MAX_ENSEMBLE_MEMBERS + 1)) + content = document( + extra=f"""\ +input: + ground_elevation: elevation + friction: manning +options: + cfl: [{values}] +outputs: {{}} +""" + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.ensembles == () + assert stream.failures[0].phase == "expansion" + assert "exceeding the 100 limit" in stream.failures[0].detail + + +def test_duration_and_template_helpers_are_strict(): + assert parse_iso_duration("P1DT2H3M4.5S") == timedelta(days=1, hours=2, minutes=3, seconds=4.5) + assert format_iso_duration(timedelta(minutes=5)) == "PT5M" + assert ( + render_template("{{{ensemble}}}-{simulation}", ensemble="study", simulation="sim-a") + == "{study}-sim-a" + ) diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py new file mode 100644 index 00000000..670217a3 --- /dev/null +++ b/tests/grass/test_ensemble.py @@ -0,0 +1,58 @@ +"""GRASS integration coverage for basic spawned YAML ensemble execution.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pytest + +from itzi.itzi import itzi_run + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): + config_path = Path(test_data_temp_path) / "basic.yaml" + config_path.write_text( + """\ +schema_version: 1 +ensemble: + id: basic +domain: {} +time: + duration: "PT1M" + record_step: "PT30S" +input: + ground_elevation: "z" + friction: "n" +options: + cfl: 0.2 + dtmax: 0.3 +outputs: + rasters: + prefix: "basic_{simulation}" + variables: [water_depth] + manifest: + file: "basic.manifest.json" +""", + encoding="utf-8", + ) + args = argparse.Namespace( + config_file=[str(config_path)], + o=True, + v=None, + q=None, + resume_from=[], + member=[], + dry=False, + ) + + itzi_run(args) + + manifest_path = Path(test_data_temp_path) / "basic.manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + member = manifest["members"][0] + assert member["status"] == "completed" + assert member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") diff --git a/uv.lock b/uv.lock index 71aa33b8..c33f3d53 100644 --- a/uv.lock +++ b/uv.lock @@ -346,6 +346,7 @@ dependencies = [ { name = "itzi-core" }, { name = "numpy" }, { name = "pydantic" }, + { name = "pyyaml" }, ] [package.dev-dependencies] @@ -368,6 +369,7 @@ requires-dist = [ { name = "itzi-core", specifier = "==0.8.0" }, { name = "numpy", specifier = ">=2.5" }, { name = "pydantic", specifier = ">=2.10" }, + { name = "pyyaml", specifier = ">=6.0" }, ] [package.metadata.requires-dev] From d9ed06c6c05879daa79e85db78213f10acd231c1 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 16:45:03 -0600 Subject: [PATCH 03/85] split ensemble in three python files --- src/itzi/ensemble.py | 687 +++--------------------------------- src/itzi/ensemble_models.py | 389 ++++++++++++++++++++ src/itzi/ensemble_schema.py | 292 +++++++++++++++ 3 files changed, 731 insertions(+), 637 deletions(-) create mode 100644 src/itzi/ensemble_models.py create mode 100644 src/itzi/ensemble_schema.py diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py index a41bff79..12d569f4 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble.py @@ -1,509 +1,64 @@ -"""YAML ensemble configuration loading and deterministic expansion. +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. -This module intentionally has no GRASS imports. A configuration stream is -validated and expanded in the parent process before a resolver opens a GRASS -session. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. """ from __future__ import annotations import hashlib import itertools -import json -import math -import re -import string -from dataclasses import dataclass -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta from pathlib import Path -from typing import Annotated, Any, Literal +from typing import Any import yaml -from itzi_core import ( - ARRAY_DEFINITIONS, - ArrayCategory, - DomainData, - InfiltrationModelType, - SimulationConfig, - TemporalType, -) +from itzi_core import InfiltrationModelType, TemporalType from itzi_core.const import DefaultValues -from pydantic import ( - BaseModel, - ConfigDict, - Field, - StrictFloat, - StrictStr, - ValidationError, - field_validator, - model_validator, -) +from pydantic import BaseModel, ValidationError +from yaml.constructor import ConstructorError +from yaml.resolver import BaseResolver import itzi.messenger as msgr -from itzi.grass_session import GrassParams - -INPUT_KEYS = tuple( - definition.key - for definition in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in definition.category +from itzi.ensemble_models import ( + DIRECT_INPUT_KEYS, + MAX_ENSEMBLE_MEMBERS, + SURFACE_FLOW_DEFAULTS, + DocumentFailure, + EnsembleError, + ExpandedEnsemble, + ExpandedSimulation, + JsonValue, + LoadedYamlStream, + NormalizedInfiltration, + NormalizedTime, + OutputTemplates, + SourceDocument, + _canonical_json, + _canonical_value, + _semantic_value, + parse_iso_duration, + render_template, ) -OUTPUT_KEYS = tuple( - definition.key - for definition in ARRAY_DEFINITIONS - if ArrayCategory.OUTPUT in definition.category -) -GREEN_AMPT_KEYS = ( - "effective_porosity", - "capillary_pressure", - "hydraulic_conductivity", - "soil_water_content", -) -DIRECT_INPUT_KEYS = tuple( - key for key in INPUT_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} -) -SURFACE_FLOW_DEFAULTS = { - "hmin": DefaultValues.HFMIN, - "cfl": DefaultValues.CFL, - "theta": DefaultValues.THETA, - "g": DefaultValues.G, - "dtmax": DefaultValues.DTMAX, - "slope_threshold": DefaultValues.SLOPE_THRESHOLD, - "max_slope": DefaultValues.MAX_SLOPE, - "max_error": DefaultValues.MAX_ERROR, -} -DRAINAGE_DEFAULTS = { - "orifice_coeff": DefaultValues.ORIFICE_COEFF, - "free_weir_coeff": DefaultValues.FREE_WEIR_COEFF, - "submerged_weir_coeff": DefaultValues.SUBMERGED_WEIR_COEFF, -} -MAX_ENSEMBLE_MEMBERS = 100 -MAX_BATCH_MEMBERS = 200 -DURATION_RE = re.compile( - r"P(?:(?P\d+)D)?(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?" - r"(?:(?P\d+(?:\.\d+)?)S)?)?\Z" -) -ENSEMBLE_ID_START_CHARS = string.ascii_letters + string.digits -ENSEMBLE_ID_CHARS = ENSEMBLE_ID_START_CHARS + "._-" - -type JsonValue = ( - None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] +from itzi.ensemble_schema import ( + ConstantInfiltration, + DrainageSweepConfig, + GreenAmptInfiltration, + InfiltrationAlternative, + InputSweepConfig, + NoInfiltration, + OptionSweepConfig, + TimeConfig, + YamlEnsembleDocumentV1, ) -type SweepString = StrictStr | list[StrictStr] -type SweepFloat = StrictFloat | list[StrictFloat] - - -class EnsembleError(ValueError): - """Configuration error associated with a single YAML document.""" - - -@dataclass(frozen=True) -class SourceDocument: - """Stable source metadata for one document in a YAML stream.""" - - path: Path - document_index: int - start_line: int - file_digest: str - document_digest: str - - -@dataclass(frozen=True) -class DocumentFailure: - """A document-local error that does not prevent later documents loading.""" - - path: Path - document_index: int - line: int | None - column: int | None - phase: Literal["parse", "schema", "expansion"] - detail: str - - def format(self) -> str: - location = f"{self.path} document {self.document_index}" - if self.line is not None and self.column is not None: - location += f" line {self.line}, column {self.column}" - return f"{location}: {self.phase}: {self.detail}" - - -class StrictModel(BaseModel): - """Base model for the public YAML schema.""" - - model_config = ConfigDict(extra="forbid", frozen=True, strict=True) - - -class EnsembleMetadata(StrictModel): - id: StrictStr - name: StrictStr | None = None - - @field_validator("id") - @classmethod - def validate_id(cls, value: str) -> str: - if ( - not value - or value[0] not in ENSEMBLE_ID_START_CHARS - or any(character not in ENSEMBLE_ID_CHARS for character in value[1:]) - ): - raise ValueError("must match [A-Za-z0-9][A-Za-z0-9._-]*") - return value - - -class GrassContextConfig(StrictModel): - database: StrictStr | None = None - project: StrictStr | None = None - mapset: StrictStr | None = None - executable: StrictStr | None = None - - @model_validator(mode="after") - def validate_context(self) -> GrassContextConfig: - values = (self.database, self.project, self.mapset) - if any(value is not None for value in values) and not all( - value is not None for value in values - ): - raise ValueError("database, project, and mapset must be supplied together") - if self.executable is not None and self.database is None: - raise ValueError("executable requires database, project, and mapset") - return self - - -class DomainConfig(StrictModel): - grass: GrassContextConfig | None = None - region: StrictStr | None = None - mask: StrictStr | None = None - - -class RelativeTimeConfig(StrictModel): - duration: StrictStr - record_step: StrictStr - start: None = None - end: None = None - - -class AbsoluteDurationTimeConfig(StrictModel): - start: StrictStr - duration: StrictStr - record_step: StrictStr - end: None = None - - -class AbsoluteEndTimeConfig(StrictModel): - start: StrictStr - end: StrictStr - record_step: StrictStr - duration: None = None - - -type TimeConfig = RelativeTimeConfig | AbsoluteDurationTimeConfig | AbsoluteEndTimeConfig - - -class NoInfiltration(StrictModel): - type: Literal["none"] - label: StrictStr | None = None - - -class ConstantInfiltration(StrictModel): - type: Literal["constant"] - label: StrictStr | None = None - rate: StrictStr - - -class GreenAmptInfiltration(StrictModel): - type: Literal["green-ampt"] - label: StrictStr | None = None - effective_porosity: StrictStr - capillary_pressure: StrictStr - hydraulic_conductivity: StrictStr - soil_water_content: StrictStr | None = None - - -type InfiltrationAlternative = Annotated[ - NoInfiltration | ConstantInfiltration | GreenAmptInfiltration, - Field(discriminator="type"), -] - - -class InputSweepConfig(StrictModel): - ground_elevation: SweepString - friction: SweepString - water_depth: SweepString | None = None - water_surface_elevation: SweepString | None = None - losses: SweepString | None = None - rainfall_rate: SweepString | None = None - inflow: SweepString | None = None - boundary_value: SweepString | None = None - boundary_type: SweepString | None = None - infiltration: InfiltrationAlternative | list[InfiltrationAlternative] = NoInfiltration( - type="none" - ) - - @field_validator( - "ground_elevation", - "friction", - "water_depth", - "water_surface_elevation", - "losses", - "rainfall_rate", - "inflow", - "boundary_value", - "boundary_type", - ) - @classmethod - def validate_string_sweep(cls, value: SweepString | None) -> SweepString | None: - return _validate_sweep(value) - - @field_validator("infiltration") - @classmethod - def validate_infiltration_sweep( - cls, value: InfiltrationAlternative | list[InfiltrationAlternative] - ) -> InfiltrationAlternative | list[InfiltrationAlternative]: - return _validate_sweep(value) - - @model_validator(mode="after") - def validate_initial_conditions(self) -> InputSweepConfig: - if self.water_depth is not None and self.water_surface_elevation is not None: - raise ValueError("water_depth and water_surface_elevation are mutually exclusive") - return self - - -class OptionSweepConfig(StrictModel): - hmin: SweepFloat | None = None - cfl: SweepFloat | None = None - theta: SweepFloat | None = None - g: SweepFloat | None = None - dtmax: SweepFloat | None = None - slope_threshold: SweepFloat | None = None - max_slope: SweepFloat | None = None - max_error: SweepFloat | None = None - dtinf: SweepFloat | None = None - - @field_validator("*") - @classmethod - def validate_numeric_sweep(cls, value: SweepFloat | None) -> SweepFloat | None: - return _validate_sweep(value, numeric=True) - - -class DrainageSweepConfig(StrictModel): - swmm_input: SweepString - orifice_coeff: SweepFloat - free_weir_coeff: SweepFloat - submerged_weir_coeff: SweepFloat - - @field_validator("swmm_input") - @classmethod - def validate_input_sweep(cls, value: SweepString) -> SweepString: - return _validate_sweep(value) - - @field_validator("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") - @classmethod - def validate_coeff_sweep(cls, value: SweepFloat) -> SweepFloat: - return _validate_sweep(value, numeric=True) - - -class RasterOutputs(StrictModel): - prefix: StrictStr - variables: list[StrictStr] - - @field_validator("variables") - @classmethod - def validate_variables(cls, value: list[str]) -> list[str]: - if not value: - raise ValueError("must not be empty") - unknown = sorted(set(value) - set(OUTPUT_KEYS)) - if unknown: - raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") - if len(value) != len(set(value)): - raise ValueError("contains duplicate output variables") - return value - - -class StatisticsOutputs(StrictModel): - file: StrictStr - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value - - -class DrainageOutputs(StrictModel): - vector_dataset: StrictStr - - -class ManifestOutputs(StrictModel): - file: StrictStr - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value - - -class OutputConfig(StrictModel): - rasters: RasterOutputs | None = None - statistics: StatisticsOutputs | None = None - drainage: DrainageOutputs | None = None - manifest: ManifestOutputs | None = None - - -class HotstartOutputConfig(StrictModel): - wallclock_step: StrictStr - file: StrictStr - - @field_validator("wallclock_step") - @classmethod - def validate_step(cls, value: str) -> str: - if parse_iso_duration(value) <= timedelta(): - raise ValueError("must be strictly positive") - return value - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value - - -class YamlEnsembleDocumentV1(StrictModel): - schema_version: Literal[1] - ensemble: EnsembleMetadata - domain: DomainConfig - time: TimeConfig - input: InputSweepConfig - options: OptionSweepConfig - drainage: DrainageSweepConfig | None = None - outputs: OutputConfig - hotstart: HotstartOutputConfig | None = None - - @model_validator(mode="after") - def validate_drainage_output(self) -> YamlEnsembleDocumentV1: - if self.outputs.drainage is not None and self.drainage is None: - raise ValueError("outputs.drainage requires a complete drainage configuration") - return self - - -@dataclass(frozen=True) -class NormalizedTime: - """Time semantics retained until the final core configuration boundary.""" - - temporal_type: TemporalType - start: datetime | None - end: datetime | None - duration: timedelta - record_step: timedelta - had_timezone_offset: bool - - -@dataclass(frozen=True) -class NormalizedInfiltration: - model: InfiltrationModelType - input_maps: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class OutputTemplates: - raster_prefix: str | None - raster_variables: tuple[str, ...] - statistics_file: str | None - drainage_dataset: str | None - hotstart_file: str | None - hotstart_interval: timedelta | None - - -@dataclass(frozen=True) -class ExpandedSimulation: - """One fully scalar pre-GRASS application configuration.""" - - source: SourceDocument - ensemble_id: str - ensemble_name: str | None - coordinates: tuple[tuple[str, JsonValue], ...] - domain: DomainConfig - time: NormalizedTime - input_maps: tuple[tuple[str, str], ...] - infiltration: NormalizedInfiltration - options: tuple[tuple[str, float], ...] - drainage: tuple[tuple[str, str | float], ...] | None - outputs: OutputTemplates - - -@dataclass(frozen=True) -class ExpandedEnsemble: - """One YAML document after Cartesian expansion.""" - - source: SourceDocument - ensemble_id: str - ensemble_name: str | None - manifest_template: str | None - simulations: tuple[ExpandedSimulation, ...] - - -@dataclass(frozen=True) -class EffectiveMask: - """The immutable mask source selected during GRASS resolution.""" - - mode: Literal["explicit", "active", "none"] - source: str | None - - -@dataclass(frozen=True) -class ArtifactSummary: - """Rendered, application-owned output destinations for one member.""" - - output_map_names: tuple[tuple[str, str], ...] - drainage_output: str | None - statistics_file: Path | None - - -@dataclass(frozen=True) -class ResolvedSimulation: - """Spawn-serializable execution payload with one final core configuration.""" - - source: SourceDocument - ensemble_id: str - simulation_id: str - coordinates: tuple[tuple[str, JsonValue], ...] - grass_params: GrassParams - domain_data: DomainData - effective_mask: EffectiveMask - input_kinds: tuple[tuple[str, Literal["raster", "strds"]], ...] - simulation_config: SimulationConfig - artifacts: ArtifactSummary - normalized_payload: str - - -@dataclass(frozen=True) -class ValidationFailure: - """A coordinate that could not become an executable resolved simulation.""" - - source: SourceDocument - ensemble_id: str - coordinates: tuple[tuple[str, JsonValue], ...] - phase: str - detail: str - - -@dataclass(frozen=True) -class ResolvedEnsemble: - """Resolved member payloads and local validation failures for one ensemble.""" - - source: SourceDocument - ensemble_id: str - ensemble_name: str | None - manifest_file: Path - simulations: tuple[ResolvedSimulation, ...] - failures: tuple[ValidationFailure, ...] - - -@dataclass(frozen=True) -class LoadedYamlStream: - """Valid ensembles and document-local failures from one YAML file.""" - - ensembles: tuple[ExpandedEnsemble, ...] - failures: tuple[DocumentFailure, ...] class _StrictSafeLoader(yaml.SafeLoader): @@ -516,7 +71,7 @@ def _construct_mapping( mapping: dict[Any, Any] = {} for key_node, value_node in node.value: if key_node.tag == "tag:yaml.org,2002:merge": - raise yaml.constructor.ConstructorError( + raise ConstructorError( "while constructing a mapping", node.start_mark, "YAML merge keys are not supported", @@ -526,14 +81,14 @@ def _construct_mapping( try: duplicate = key in mapping except TypeError as error: - raise yaml.constructor.ConstructorError( + raise ConstructorError( "while constructing a mapping", node.start_mark, "mapping keys must be hashable", key_node.start_mark, ) from error if duplicate: - raise yaml.constructor.ConstructorError( + raise ConstructorError( "while constructing a mapping", node.start_mark, f"duplicate mapping key {key!r}", @@ -544,82 +99,11 @@ def _construct_mapping( _StrictSafeLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping, ) -def _validate_sweep( - value: Any, - *, - numeric: bool = False, -) -> Any: - """Validate one scalar-or-list sweep field after Pydantic's type validation.""" - values = value if isinstance(value, list) else [value] - if not values: - raise ValueError("sweep lists must not be empty") - for item in values: - if isinstance(item, list): - raise ValueError("nested sweep lists are not supported") # noqa: TRY004 - if numeric and (not isinstance(item, float) or not math.isfinite(item)): - raise ValueError("must contain finite numbers") - - semantic_values = [_canonical_json(_semantic_value(item)) for item in values] - if len(semantic_values) != len(set(semantic_values)): - raise ValueError("contains duplicate semantic values") - return value - - -def _semantic_value(value: Any) -> Any: - if isinstance(value, BaseModel): - dumped = value.model_dump(mode="python") - dumped.pop("label", None) - return dumped - return value - - -def parse_iso_duration(value: str) -> timedelta: - """Parse the supported, non-calendar ISO 8601 duration subset.""" - match = DURATION_RE.fullmatch(value) - if match is None or not any(match.groupdict().values()): - raise ValueError("must be an ISO 8601 duration without years or months") - groups = match.groupdict(default="0") - if "T" in value and not any(groups[key] != "0" for key in ("hours", "minutes", "seconds")): - raise ValueError("must include a time component after T") - duration = timedelta( - days=int(groups["days"]), - hours=int(groups["hours"]), - minutes=int(groups["minutes"]), - seconds=float(groups["seconds"]), - ) - if duration <= timedelta(): - raise ValueError("must be strictly positive") - return duration - - -def format_iso_duration(value: timedelta) -> str: - """Render a positive duration in a stable ISO 8601 representation.""" - total_seconds = value.total_seconds() - if total_seconds <= 0: - raise ValueError("duration must be strictly positive") - days, remainder = divmod(total_seconds, 86400) - hours, remainder = divmod(remainder, 3600) - minutes, seconds = divmod(remainder, 60) - parts = ["P"] - if days: - parts.append(f"{int(days)}D") - if hours or minutes or seconds or not days: - parts.append("T") - if hours: - parts.append(f"{int(hours)}H") - if minutes: - parts.append(f"{int(minutes)}M") - if seconds or (not hours and not minutes): - rendered_seconds = format(seconds, ".6f").rstrip("0").rstrip(".") - parts.append(f"{rendered_seconds}S") - return "".join(parts) - - def normalize_time(value: TimeConfig) -> NormalizedTime: """Validate a public time form without leaking core's relative sentinels.""" duration = parse_iso_duration(value.duration) if value.duration is not None else None @@ -675,34 +159,6 @@ def _parse_datetime(value: str) -> datetime: raise EnsembleError(f"invalid ISO 8601 timestamp {value!r}") from error -def _canonical_json(value: Any) -> str: - return json.dumps( - _canonical_value(value), - allow_nan=False, - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ) - - -def _canonical_value(value: Any) -> Any: - if isinstance(value, BaseModel): - return _canonical_value(value.model_dump(mode="python")) - if isinstance(value, dict): - return {str(key): _canonical_value(item) for key, item in value.items()} - if isinstance(value, list | tuple): - return [_canonical_value(item) for item in value] - if isinstance(value, datetime): - return {"$datetime": value.isoformat()} - if isinstance(value, date): - return {"$date": value.isoformat()} - if isinstance(value, timedelta): - return {"$duration": format_iso_duration(value)} - if isinstance(value, float) and not math.isfinite(value): - raise EnsembleError("NaN and infinity are not supported") - return value - - def _document_digest(document: Any) -> str: payload = {"schema": "itzi-yaml-document-v1", "document": _canonical_value(document)} return hashlib.blake2b(_canonical_json(payload).encode(), digest_size=32).hexdigest() @@ -1017,31 +473,6 @@ def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: ) -def render_template(template: str, *, ensemble: str, simulation: str | None) -> str: - """Render the intentionally small output-template language.""" - formatter = string.Formatter() - values = {"ensemble": ensemble, "simulation": simulation} - try: - fragments = formatter.parse(template) - rendered: list[str] = [] - for literal, field_name, format_spec, conversion in fragments: - rendered.append(literal) - if field_name is None: - continue - if field_name not in values: - raise EnsembleError(f"unsupported template placeholder {{{field_name}}}") - if values[field_name] is None: - raise EnsembleError(f"{{{field_name}}} is not permitted in this template") - if format_spec or conversion: - raise EnsembleError( - "template format specifications and conversions are not supported" - ) - rendered.append(str(values[field_name])) - except ValueError as error: - raise EnsembleError(f"invalid output template: {error}") from error - return "".join(rendered) - - def _freeze_json(value: Any) -> JsonValue: if isinstance(value, BaseModel): return _freeze_json(value.model_dump(mode="python")) @@ -1052,21 +483,3 @@ def _freeze_json(value: Any) -> JsonValue: if isinstance(value, (str, int, float, bool)) or value is None: return value raise TypeError(f"unsupported coordinate value {value!r}") - - -def check_batch_limits(ensembles: tuple[ExpandedEnsemble, ...]) -> None: - """Reject a valid batch that is too large before any resolver is started.""" - count = sum(len(ensemble.simulations) for ensemble in ensembles) - if count > MAX_BATCH_MEMBERS: - raise EnsembleError( - f"batch expands to {count} simulations, exceeding the {MAX_BATCH_MEMBERS} limit" - ) - - -def check_unique_ensemble_ids(ensembles: tuple[ExpandedEnsemble, ...]) -> None: - """Reject a batch-global duplicate ensemble identifier.""" - seen: set[str] = set() - for ensemble in ensembles: - if ensemble.ensemble_id in seen: - raise EnsembleError(f"duplicate ensemble ID {ensemble.ensemble_id!r}") - seen.add(ensemble.ensemble_id) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py new file mode 100644 index 00000000..95996c80 --- /dev/null +++ b/src/itzi/ensemble_models.py @@ -0,0 +1,389 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import json +import math +import re +import string +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any, Literal + +from itzi_core import ( + ARRAY_DEFINITIONS, + ArrayCategory, + DomainData, + InfiltrationModelType, + SimulationConfig, + TemporalType, +) +from itzi_core.const import DefaultValues +from pydantic import BaseModel, ConfigDict, StrictStr, model_validator + +from itzi.grass_session import GrassParams + +INPUT_KEYS = tuple( + definition.key + for definition in ARRAY_DEFINITIONS + if ArrayCategory.INPUT in definition.category +) +OUTPUT_KEYS = tuple( + definition.key + for definition in ARRAY_DEFINITIONS + if ArrayCategory.OUTPUT in definition.category +) +GREEN_AMPT_KEYS = ( + "effective_porosity", + "capillary_pressure", + "hydraulic_conductivity", + "soil_water_content", +) +DIRECT_INPUT_KEYS = tuple( + key for key in INPUT_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} +) +SURFACE_FLOW_DEFAULTS = { + "hmin": DefaultValues.HFMIN, + "cfl": DefaultValues.CFL, + "theta": DefaultValues.THETA, + "g": DefaultValues.G, + "dtmax": DefaultValues.DTMAX, + "slope_threshold": DefaultValues.SLOPE_THRESHOLD, + "max_slope": DefaultValues.MAX_SLOPE, + "max_error": DefaultValues.MAX_ERROR, +} +DRAINAGE_DEFAULTS = { + "orifice_coeff": DefaultValues.ORIFICE_COEFF, + "free_weir_coeff": DefaultValues.FREE_WEIR_COEFF, + "submerged_weir_coeff": DefaultValues.SUBMERGED_WEIR_COEFF, +} +MAX_ENSEMBLE_MEMBERS = 100 +MAX_BATCH_MEMBERS = 200 +DURATION_RE = re.compile( + r"P(?:(?P\d+)D)?(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?" + r"(?:(?P\d+(?:\.\d+)?)S)?)?\Z" +) +ENSEMBLE_ID_START_CHARS = string.ascii_letters + string.digits +ENSEMBLE_ID_CHARS = ENSEMBLE_ID_START_CHARS + "._-" + +type JsonValue = ( + None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] +) + + +class EnsembleError(ValueError): + """Configuration error associated with a single YAML document.""" + + +@dataclass(frozen=True) +class SourceDocument: + """Stable source metadata for one document in a YAML stream.""" + + path: Path + document_index: int + start_line: int + file_digest: str + document_digest: str + + +@dataclass(frozen=True) +class DocumentFailure: + """A document-local error that does not prevent later documents loading.""" + + path: Path + document_index: int + line: int | None + column: int | None + phase: Literal["parse", "schema", "expansion"] + detail: str + + def format(self) -> str: + location = f"{self.path} document {self.document_index}" + if self.line is not None and self.column is not None: + location += f" line {self.line}, column {self.column}" + return f"{location}: {self.phase}: {self.detail}" + + +class StrictModel(BaseModel): + """Base model for the public YAML schema.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class GrassContextConfig(StrictModel): + database: StrictStr | None = None + project: StrictStr | None = None + mapset: StrictStr | None = None + executable: StrictStr | None = None + + @model_validator(mode="after") + def validate_context(self) -> GrassContextConfig: + values = (self.database, self.project, self.mapset) + if any(value is not None for value in values) and not all( + value is not None for value in values + ): + raise ValueError("database, project, and mapset must be supplied together") + if self.executable is not None and self.database is None: + raise ValueError("executable requires database, project, and mapset") + return self + + +class DomainConfig(StrictModel): + grass: GrassContextConfig | None = None + region: StrictStr | None = None + mask: StrictStr | None = None + + +@dataclass(frozen=True) +class NormalizedTime: + """Time semantics retained until the final core configuration boundary.""" + + temporal_type: TemporalType + start: datetime | None + end: datetime | None + duration: timedelta + record_step: timedelta + had_timezone_offset: bool + + +@dataclass(frozen=True) +class NormalizedInfiltration: + model: InfiltrationModelType + input_maps: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class OutputTemplates: + raster_prefix: str | None + raster_variables: tuple[str, ...] + statistics_file: str | None + drainage_dataset: str | None + hotstart_file: str | None + hotstart_interval: timedelta | None + + +@dataclass(frozen=True) +class ExpandedSimulation: + """One fully scalar pre-GRASS application configuration.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + coordinates: tuple[tuple[str, JsonValue], ...] + domain: DomainConfig + time: NormalizedTime + input_maps: tuple[tuple[str, str], ...] + infiltration: NormalizedInfiltration + options: tuple[tuple[str, float], ...] + drainage: tuple[tuple[str, str | float], ...] | None + outputs: OutputTemplates + + +@dataclass(frozen=True) +class ExpandedEnsemble: + """One YAML document after Cartesian expansion.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + manifest_template: str | None + simulations: tuple[ExpandedSimulation, ...] + + +@dataclass(frozen=True) +class EffectiveMask: + """The immutable mask source selected during GRASS resolution.""" + + mode: Literal["explicit", "active", "none"] + source: str | None + + +@dataclass(frozen=True) +class ArtifactSummary: + """Rendered, application-owned output destinations for one member.""" + + output_map_names: tuple[tuple[str, str], ...] + drainage_output: str | None + statistics_file: Path | None + + +@dataclass(frozen=True) +class ResolvedSimulation: + """Spawn-serializable execution payload with one final core configuration.""" + + source: SourceDocument + ensemble_id: str + simulation_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + grass_params: GrassParams + domain_data: DomainData + effective_mask: EffectiveMask + input_kinds: tuple[tuple[str, Literal["raster", "strds"]], ...] + simulation_config: SimulationConfig + artifacts: ArtifactSummary + normalized_payload: str + + +@dataclass(frozen=True) +class ValidationFailure: + """A coordinate that could not become an executable resolved simulation.""" + + source: SourceDocument + ensemble_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + phase: str + detail: str + + +@dataclass(frozen=True) +class ResolvedEnsemble: + """Resolved member payloads and local validation failures for one ensemble.""" + + source: SourceDocument + ensemble_id: str + ensemble_name: str | None + manifest_file: Path + simulations: tuple[ResolvedSimulation, ...] + failures: tuple[ValidationFailure, ...] + + +@dataclass(frozen=True) +class LoadedYamlStream: + """Valid ensembles and document-local failures from one YAML file.""" + + ensembles: tuple[ExpandedEnsemble, ...] + failures: tuple[DocumentFailure, ...] + + +def parse_iso_duration(value: str) -> timedelta: + """Parse the supported, non-calendar ISO 8601 duration subset.""" + match = DURATION_RE.fullmatch(value) + if match is None or not any(match.groupdict().values()): + raise ValueError("must be an ISO 8601 duration without years or months") + groups = match.groupdict(default="0") + if "T" in value and not any(groups[key] != "0" for key in ("hours", "minutes", "seconds")): + raise ValueError("must include a time component after T") + duration = timedelta( + days=int(groups["days"]), + hours=int(groups["hours"]), + minutes=int(groups["minutes"]), + seconds=float(groups["seconds"]), + ) + if duration <= timedelta(): + raise ValueError("must be strictly positive") + return duration + + +def format_iso_duration(value: timedelta) -> str: + """Render a positive duration in a stable ISO 8601 representation.""" + total_seconds = value.total_seconds() + if total_seconds <= 0: + raise ValueError("duration must be strictly positive") + days, remainder = divmod(total_seconds, 86400) + hours, remainder = divmod(remainder, 3600) + minutes, seconds = divmod(remainder, 60) + parts = ["P"] + if days: + parts.append(f"{int(days)}D") + if hours or minutes or seconds or not days: + parts.append("T") + if hours: + parts.append(f"{int(hours)}H") + if minutes: + parts.append(f"{int(minutes)}M") + if seconds or (not hours and not minutes): + rendered_seconds = format(seconds, ".6f").rstrip("0").rstrip(".") + parts.append(f"{rendered_seconds}S") + return "".join(parts) + + +def _semantic_value(value: Any) -> Any: + if isinstance(value, BaseModel): + dumped = value.model_dump(mode="python") + dumped.pop("label", None) + return dumped + return value + + +def _canonical_json(value: Any) -> str: + return json.dumps( + _canonical_value(value), + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_value(value: Any) -> Any: + if isinstance(value, BaseModel): + return _canonical_value(value.model_dump(mode="python")) + if isinstance(value, dict): + return {str(key): _canonical_value(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_canonical_value(item) for item in value] + if isinstance(value, datetime): + return {"$datetime": value.isoformat()} + if isinstance(value, date): + return {"$date": value.isoformat()} + if isinstance(value, timedelta): + return {"$duration": format_iso_duration(value)} + if isinstance(value, float) and not math.isfinite(value): + raise EnsembleError("NaN and infinity are not supported") + return value + + +def render_template(template: str, *, ensemble: str, simulation: str | None) -> str: + """Render the intentionally small output-template language.""" + formatter = string.Formatter() + values = {"ensemble": ensemble, "simulation": simulation} + try: + fragments = formatter.parse(template) + rendered: list[str] = [] + for literal, field_name, format_spec, conversion in fragments: + rendered.append(literal) + if field_name is None: + continue + if field_name not in values: + raise EnsembleError(f"unsupported template placeholder {{{field_name}}}") + if values[field_name] is None: + raise EnsembleError(f"{{{field_name}}} is not permitted in this template") + if format_spec or conversion: + raise EnsembleError( + "template format specifications and conversions are not supported" + ) + rendered.append(str(values[field_name])) + except ValueError as error: + raise EnsembleError(f"invalid output template: {error}") from error + return "".join(rendered) + + +def check_batch_limits(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a valid batch that is too large before any resolver is started.""" + count = sum(len(ensemble.simulations) for ensemble in ensembles) + if count > MAX_BATCH_MEMBERS: + raise EnsembleError( + f"batch expands to {count} simulations, exceeding the {MAX_BATCH_MEMBERS} limit" + ) + + +def check_unique_ensemble_ids(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a batch-global duplicate ensemble identifier.""" + seen: set[str] = set() + for ensemble in ensembles: + if ensemble.ensemble_id in seen: + raise EnsembleError(f"duplicate ensemble ID {ensemble.ensemble_id!r}") + seen.add(ensemble.ensemble_id) diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble_schema.py new file mode 100644 index 00000000..0f1654d3 --- /dev/null +++ b/src/itzi/ensemble_schema.py @@ -0,0 +1,292 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import math +from datetime import timedelta +from typing import Annotated, Any, Literal + +from pydantic import ( + Field, + StrictFloat, + StrictStr, + field_validator, + model_validator, +) + +from itzi.ensemble_models import ( + ENSEMBLE_ID_CHARS, + ENSEMBLE_ID_START_CHARS, + OUTPUT_KEYS, + DomainConfig, + StrictModel, + _canonical_json, + _semantic_value, + parse_iso_duration, +) + +type SweepString = StrictStr | list[StrictStr] +type SweepFloat = StrictFloat | list[StrictFloat] + + +class EnsembleMetadata(StrictModel): + id: StrictStr + name: StrictStr | None = None + + @field_validator("id") + @classmethod + def validate_id(cls, value: str) -> str: + if ( + not value + or value[0] not in ENSEMBLE_ID_START_CHARS + or any(character not in ENSEMBLE_ID_CHARS for character in value[1:]) + ): + raise ValueError("must match [A-Za-z0-9][A-Za-z0-9._-]*") + return value + + +class RelativeTimeConfig(StrictModel): + duration: StrictStr + record_step: StrictStr + start: None = None + end: None = None + + +class AbsoluteDurationTimeConfig(StrictModel): + start: StrictStr + duration: StrictStr + record_step: StrictStr + end: None = None + + +class AbsoluteEndTimeConfig(StrictModel): + start: StrictStr + end: StrictStr + record_step: StrictStr + duration: None = None + + +type TimeConfig = RelativeTimeConfig | AbsoluteDurationTimeConfig | AbsoluteEndTimeConfig + + +class NoInfiltration(StrictModel): + type: Literal["none"] + label: StrictStr | None = None + + +class ConstantInfiltration(StrictModel): + type: Literal["constant"] + label: StrictStr | None = None + rate: StrictStr + + +class GreenAmptInfiltration(StrictModel): + type: Literal["green-ampt"] + label: StrictStr | None = None + effective_porosity: StrictStr + capillary_pressure: StrictStr + hydraulic_conductivity: StrictStr + soil_water_content: StrictStr | None = None + + +type InfiltrationAlternative = Annotated[ + NoInfiltration | ConstantInfiltration | GreenAmptInfiltration, + Field(discriminator="type"), +] + + +class InputSweepConfig(StrictModel): + ground_elevation: SweepString + friction: SweepString + water_depth: SweepString | None = None + water_surface_elevation: SweepString | None = None + losses: SweepString | None = None + rainfall_rate: SweepString | None = None + inflow: SweepString | None = None + boundary_value: SweepString | None = None + boundary_type: SweepString | None = None + infiltration: InfiltrationAlternative | list[InfiltrationAlternative] = NoInfiltration( + type="none" + ) + + @field_validator( + "ground_elevation", + "friction", + "water_depth", + "water_surface_elevation", + "losses", + "rainfall_rate", + "inflow", + "boundary_value", + "boundary_type", + ) + @classmethod + def validate_string_sweep(cls, value: SweepString | None) -> SweepString | None: + return _validate_sweep(value) + + @field_validator("infiltration") + @classmethod + def validate_infiltration_sweep( + cls, value: InfiltrationAlternative | list[InfiltrationAlternative] + ) -> InfiltrationAlternative | list[InfiltrationAlternative]: + return _validate_sweep(value) + + @model_validator(mode="after") + def validate_initial_conditions(self) -> InputSweepConfig: + if self.water_depth is not None and self.water_surface_elevation is not None: + raise ValueError("water_depth and water_surface_elevation are mutually exclusive") + return self + + +class OptionSweepConfig(StrictModel): + hmin: SweepFloat | None = None + cfl: SweepFloat | None = None + theta: SweepFloat | None = None + g: SweepFloat | None = None + dtmax: SweepFloat | None = None + slope_threshold: SweepFloat | None = None + max_slope: SweepFloat | None = None + max_error: SweepFloat | None = None + dtinf: SweepFloat | None = None + + @field_validator("*") + @classmethod + def validate_numeric_sweep(cls, value: SweepFloat | None) -> SweepFloat | None: + return _validate_sweep(value, numeric=True) + + +class DrainageSweepConfig(StrictModel): + swmm_input: SweepString + orifice_coeff: SweepFloat + free_weir_coeff: SweepFloat + submerged_weir_coeff: SweepFloat + + @field_validator("swmm_input") + @classmethod + def validate_input_sweep(cls, value: SweepString) -> SweepString: + return _validate_sweep(value) + + @field_validator("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") + @classmethod + def validate_coeff_sweep(cls, value: SweepFloat) -> SweepFloat: + return _validate_sweep(value, numeric=True) + + +class RasterOutputs(StrictModel): + prefix: StrictStr + variables: list[StrictStr] + + @field_validator("variables") + @classmethod + def validate_variables(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("must not be empty") + unknown = sorted(set(value) - set(OUTPUT_KEYS)) + if unknown: + raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") + if len(value) != len(set(value)): + raise ValueError("contains duplicate output variables") + return value + + +class StatisticsOutputs(StrictModel): + file: StrictStr + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class DrainageOutputs(StrictModel): + vector_dataset: StrictStr + + +class ManifestOutputs(StrictModel): + file: StrictStr + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class OutputConfig(StrictModel): + rasters: RasterOutputs | None = None + statistics: StatisticsOutputs | None = None + drainage: DrainageOutputs | None = None + manifest: ManifestOutputs | None = None + + +class HotstartOutputConfig(StrictModel): + wallclock_step: StrictStr + file: StrictStr + + @field_validator("wallclock_step") + @classmethod + def validate_step(cls, value: str) -> str: + if parse_iso_duration(value) <= timedelta(): + raise ValueError("must be strictly positive") + return value + + @field_validator("file") + @classmethod + def validate_file(cls, value: str) -> str: + if not value: + raise ValueError("must not be empty") + return value + + +class YamlEnsembleDocumentV1(StrictModel): + schema_version: Literal[1] + ensemble: EnsembleMetadata + domain: DomainConfig + time: TimeConfig + input: InputSweepConfig + options: OptionSweepConfig + drainage: DrainageSweepConfig | None = None + outputs: OutputConfig + hotstart: HotstartOutputConfig | None = None + + @model_validator(mode="after") + def validate_drainage_output(self) -> YamlEnsembleDocumentV1: + if self.outputs.drainage is not None and self.drainage is None: + raise ValueError("outputs.drainage requires a complete drainage configuration") + return self + + +def _validate_sweep( + value: Any, + *, + numeric: bool = False, +) -> Any: + """Validate one scalar-or-list sweep field after Pydantic's type validation.""" + values = value if isinstance(value, list) else [value] + if not values: + raise ValueError("sweep lists must not be empty") + for item in values: + if isinstance(item, list): + raise ValueError("nested sweep lists are not supported") # noqa: TRY004 + if numeric and (not isinstance(item, float) or not math.isfinite(item)): + raise ValueError("must contain finite numbers") + + semantic_values = [_canonical_json(_semantic_value(item)) for item in values] + if len(semantic_values) != len(set(semantic_values)): + raise ValueError("contains duplicate semantic values") + return value From 874e6797efcc8f56f6629bad60e7241cc99d5e30 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 16:45:29 -0600 Subject: [PATCH 04/85] simplify CSV import --- src/itzi/providers/__init__.py | 3 --- src/itzi/simulation_runner.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/itzi/providers/__init__.py b/src/itzi/providers/__init__.py index 86b486b1..e69de29b 100644 --- a/src/itzi/providers/__init__.py +++ b/src/itzi/providers/__init__.py @@ -1,3 +0,0 @@ -from itzi.providers.csv_output import ( - ExclusiveCSVMassBalanceOutputProvider as ExclusiveCSVMassBalanceOutputProvider, -) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index dc59f4b4..829867a4 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -18,7 +18,7 @@ import numpy as np from itzi_core import SimulationBuilder, SimulationConfig -from itzi_core.providers import CSVMassBalanceOutputProvider +from itzi_core.providers.csv_mass_balance_output import CSVMassBalanceOutputProvider import itzi.messenger as msgr from itzi.ensemble import EffectiveMask From 22f90e5d963604b7b4853cfd3d846f65953ecf96 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 17:06:37 -0600 Subject: [PATCH 05/85] fix imports, and remove TypedDict in providers --- src/itzi/itzi.py | 3 +- src/itzi/providers/grass_input.py | 30 +++++++++---------- src/itzi/providers/grass_output.py | 47 +++++++++++++----------------- src/itzi/resolution.py | 2 +- src/itzi/run_plan.py | 4 +-- src/itzi/simulation_runner.py | 39 +++++++++++-------------- tests/cli/test_ensemble_config.py | 4 +-- 7 files changed, 58 insertions(+), 71 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index adc9c168..5a94d360 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ NAME: Itzï @@ -39,7 +38,7 @@ import itzi.messenger as msgr from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader -from itzi.ensemble import ( +from itzi.ensemble_models import ( DocumentFailure, EnsembleError, ExpandedEnsemble, diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 9322f571..d6569bf8 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,7 +15,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Literal, TypedDict +from typing import TYPE_CHECKING, Literal import numpy as np from itzi_core import ARRAY_DEFINITIONS, ArrayCategory, DomainData @@ -30,22 +30,20 @@ from itzi.providers.grass_interface import GrassInterface -class GrassRasterInputConfig(TypedDict): - grass_interface: GrassInterface - # A dict of key: input names - input_map_names: Mapping[str, str | None] - default_start_time: datetime - default_end_time: datetime - input_kinds: Mapping[str, Literal["raster", "strds"]] | None - - class GrassRasterInputProvider(RasterInputProvider): - def __init__(self, config: GrassRasterInputConfig) -> None: - self.grass_interface = config["grass_interface"] - self.start_time = config["default_start_time"] - self.end_time = config["default_end_time"] - self.input_kinds = config.get("input_kinds") - self.map_lists = self._get_map_lists(config["input_map_names"]) + def __init__( + self, + grass_interface: GrassInterface, + input_map_names: Mapping[str, str | None], + default_start_time: datetime, + default_end_time: datetime, + input_kinds: Mapping[str, Literal["raster", "strds"]] | None, + ) -> None: + self.grass_interface = grass_interface + self.start_time = default_start_time + self.end_time = default_end_time + self.input_kinds = input_kinds + self.map_lists = self._get_map_lists(input_map_names) def get_domain_data(self) -> DomainData: """Return a DomainData object""" diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 8fde9d6f..5ce3b40e 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -15,7 +15,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING import numpy as np from itzi_core.providers import RasterOutputProvider, VectorOutputProvider @@ -29,19 +29,6 @@ from itzi.providers.grass_interface import GrassInterface -class GrassRasterOutputConfig(TypedDict): - grass_interface: GrassInterface - out_map_names: Mapping[str, str] - hmin: float - temporal_type: TemporalType - - -class GrassVectorOutputConfig(TypedDict): - grass_interface: GrassInterface - drainage_map_name: str - temporal_type: TemporalType - - def derived_record_name(dataset_name: str, record_index: int) -> str: """Return the shared raster/vector child name for one output record.""" if record_index < 0: @@ -57,13 +44,18 @@ def derived_drainage_table_names(vector_name: str) -> tuple[str, str]: class GrassRasterOutputProvider(RasterOutputProvider): """Write simulation outputs to GRASS.""" - def __init__(self, config: GrassRasterOutputConfig) -> None: - """Initialize output provider with configuration.""" - self.grass_interface = config["grass_interface"] + def __init__( + self, + grass_interface: GrassInterface, + out_map_names: Mapping[str, str], + hmin: float, + temporal_type: TemporalType, + ) -> None: + self.grass_interface = grass_interface # user-selected map names. Keys are user-facing names - self.out_map_names = config["out_map_names"] - self.hmin = config["hmin"] - self.temporal_type = config["temporal_type"] + self.out_map_names = out_map_names + self.hmin = hmin + self.temporal_type = temporal_type for stds_name in self.out_map_names.values(): self.grass_interface.validate_output_stds_temporal_type( stds_name=stds_name, @@ -76,7 +68,6 @@ def __init__(self, config: GrassRasterOutputConfig) -> None: def _write_array( self, array: np.ndarray, map_key: str, sim_time: datetime | timedelta ) -> None: - """Write simulation data for current time step.""" map_name = derived_record_name(self.out_map_names[map_key], self.record_counter[map_key]) # write the raster self.grass_interface.write_raster_map(array, map_name, map_key, self.hmin) @@ -112,11 +103,15 @@ def finalize(self) -> None: class GrassVectorOutputProvider(VectorOutputProvider): """Write drainage simulation outputs to GRASS.""" - def __init__(self, config: GrassVectorOutputConfig) -> None: - """Initialize output provider with simulation configuration.""" - self.grass_interface = config["grass_interface"] - self.drainage_map_name = config["drainage_map_name"] - self.temporal_type = config["temporal_type"] + def __init__( + self, + grass_interface: GrassInterface, + drainage_map_name: str, + temporal_type: TemporalType, + ) -> None: + self.grass_interface = grass_interface + self.drainage_map_name = drainage_map_name + self.temporal_type = temporal_type self.grass_interface.validate_output_stds_temporal_type( stds_name=self.drainage_map_name, stds_type="stvds", diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index bf5e180d..88b16b58 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -16,7 +16,7 @@ from itzi_core import DomainData, SimulationConfig, SurfaceFlowParameters from pydantic import ValidationError -from itzi.ensemble import ( +from itzi.ensemble_models import ( ArtifactSummary, EffectiveMask, EnsembleError, diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index ccae0fe8..c0ee5c1f 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -9,7 +9,8 @@ import itzi.messenger as msgr from itzi.configreader import ConfigReader -from itzi.ensemble import ( +from itzi.ensemble import load_yaml_stream +from itzi.ensemble_models import ( DIRECT_INPUT_KEYS, GREEN_AMPT_KEYS, DocumentFailure, @@ -24,7 +25,6 @@ SourceDocument, check_batch_limits, check_unique_ensemble_ids, - load_yaml_stream, ) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 829867a4..d29b9525 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -14,14 +14,14 @@ from collections.abc import Mapping from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import numpy as np from itzi_core import SimulationBuilder, SimulationConfig from itzi_core.providers.csv_mass_balance_output import CSVMassBalanceOutputProvider import itzi.messenger as msgr -from itzi.ensemble import EffectiveMask +from itzi.ensemble_models import EffectiveMask from itzi.grass_session import GrassParams if TYPE_CHECKING: @@ -31,7 +31,8 @@ class SimulationRunner: - """Provide the necessary tools to run one simulation.""" + """Provide the necessary tools to run one simulation. + Must be instantiated within an active GRASS session.""" def __init__( self, @@ -40,7 +41,7 @@ def __init__( hotstart_path: str | None = None, stats_file: str | None = None, effective_mask: EffectiveMask | None = None, - input_kinds: Mapping[str, str] | None = None, + input_kinds: Mapping[str, Literal["raster", "strds"]] | None = None, exclusive_stats: bool = False, ) -> None: self.grass_required_version = "8.4.0" @@ -92,21 +93,17 @@ def __init__( ) raster_input_provider = GrassRasterInputProvider( - { - "grass_interface": self.g_interface, - "input_map_names": sim_config.input_map_names, - "default_start_time": sim_config.start_time, - "default_end_time": sim_config.end_time, - "input_kinds": input_kinds, - } + grass_interface=self.g_interface, + input_map_names=sim_config.input_map_names, + default_start_time=sim_config.start_time, + default_end_time=sim_config.end_time, + input_kinds=input_kinds, ) raster_output_provider = GrassRasterOutputProvider( - { - "grass_interface": self.g_interface, - "out_map_names": sim_config.output_map_names, - "hmin": sim_config.surface_flow_parameters.hmin, - "temporal_type": sim_config.temporal_type, - } + grass_interface=self.g_interface, + out_map_names=sim_config.output_map_names, + hmin=sim_config.surface_flow_parameters.hmin, + temporal_type=sim_config.temporal_type, ) sim_builder = ( SimulationBuilder(sim_config, self.g_interface.get_npmask(), data_type) @@ -115,11 +112,9 @@ def __init__( ) if sim_config.drainage_output is not None: vector_output_provider = GrassVectorOutputProvider( - { - "grass_interface": self.g_interface, - "temporal_type": sim_config.temporal_type, - "drainage_map_name": sim_config.drainage_output, - } + grass_interface=self.g_interface, + temporal_type=sim_config.temporal_type, + drainage_map_name=sim_config.drainage_output, ) sim_builder.with_vector_output_provider(vector_output_provider) if stats_file: diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 3cc7b970..8d253467 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -6,10 +6,10 @@ from itzi_core import InfiltrationModelType, TemporalType -from itzi.ensemble import ( +from itzi.ensemble import load_yaml_stream +from itzi.ensemble_models import ( MAX_ENSEMBLE_MEMBERS, format_iso_duration, - load_yaml_stream, parse_iso_duration, render_template, ) From 5a0bb05e0cfa2f5166c39c310769d0f55bcb3356 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 17:22:00 -0600 Subject: [PATCH 06/85] use get() in configreader --- src/itzi/configreader.py | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index b4e367b9..ce94054c 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -14,7 +14,6 @@ from __future__ import annotations -from collections.abc import Callable from configparser import ConfigParser from datetime import datetime, timedelta from pathlib import Path @@ -102,25 +101,13 @@ def _read_parser(filename: str) -> ConfigParser: return params -def _read_optional_value( - params: ConfigParser, - section: str, - option: str, - reader: Callable[[str, str], Any], -) -> Any | None: - """Read one config value when the option exists.""" - if not params.has_option(section, option): - return None - return reader(section, option) - - def _read_string_options( params: ConfigParser, section: str, option_names: tuple[str, ...] ) -> dict[str, str]: """Collect the string-valued options present in a section.""" values: dict[str, str] = {} for option_name in option_names: - value = _read_optional_value(params, section, option_name, params.get) + value = params.get(section, option_name, fallback=None) if value is not None: values[option_name] = value return values @@ -132,7 +119,7 @@ def _read_float_options( """Collect the float-valued options present in a section.""" values: dict[str, float] = {} for option_name in option_names: - value = _read_optional_value(params, section, option_name, params.getfloat) + value = params.getfloat(section, option_name, fallback=None) if value is not None: values[option_name] = value return values @@ -169,8 +156,8 @@ def _read_hotstart_values(params: ConfigParser) -> HotstartRunConfig | None: return None hotstart_values = { - "wallclock_step": _read_optional_value(params, "hotstart", "wallclock_step", params.get), - "save_file_name": _read_optional_value(params, "hotstart", "save_file", params.get), + "wallclock_step": params.get("hotstart", "wallclock_step", fallback=None), + "save_file_name": params.get("hotstart", "save_file", fallback=None), } if all(value is None for value in hotstart_values.values()): return None @@ -227,12 +214,10 @@ def _generate_output_map_names(prefix: str, output_values: list[str]) -> dict[st def _read_output_config(params: ConfigParser) -> tuple[str, list[str], dict[str, str | None]]: """Read output settings and derive output map names.""" - prefix = _read_optional_value(params, "output", "prefix", params.get) + prefix = params.get("output", "prefix", fallback=None) if prefix is None: prefix = f"itzi_results_{datetime.now().strftime('%Y%m%dT%H%M%S')}" - output_values = _normalize_output_values( - _read_optional_value(params, "output", "values", params.get) - ) + output_values = _normalize_output_values(params.get("output", "values", fallback=None)) output_map_names = _generate_output_map_names(prefix, output_values) return prefix, output_values, output_map_names @@ -256,7 +241,7 @@ def _read_simulation_drainage_values(params: ConfigParser) -> dict[str, str | fl drainage_values: dict[str, str | float] = {} for option_name in DRAINAGE_STRING_KEYS: - value = _read_optional_value(params, "drainage", option_name, params.get) + value = params.get("drainage", option_name, fallback=None) if value is None: continue if option_name == "output": @@ -412,9 +397,7 @@ def __init__(self, filename: str | None) -> None: self.raw_input_times = _read_time_values(params) self.input_map_names = _read_input_map_names(params) self.out_prefix, self.out_values, self.output_map_names = _read_output_config(params) - self.stats_file = ( - _read_optional_value(params, "statistics", "stats_file", params.get) or None - ) + self.stats_file = params.get("statistics", "stats_file", fallback=None) or None self.sim_times = SimulationTimes.from_raw_values(self.raw_input_times) self._check_general_input(self.input_map_names) infiltration_model = self._resolve_infiltration_model(self.input_map_names) From c86464fcee1a6fac439bbbbc4b022131607243c3 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 17:24:56 -0600 Subject: [PATCH 07/85] remove unused hotstart, infiltration label, and ensemble transport --- docs/conf_file.rst | 6 +++--- src/itzi/cli_parser.py | 4 +--- src/itzi/ensemble.py | 14 ++------------ src/itzi/ensemble_models.py | 28 ---------------------------- src/itzi/ensemble_schema.py | 28 +--------------------------- src/itzi/itzi.py | 4 ---- src/itzi/resolution.py | 2 -- src/itzi/run_plan.py | 4 ---- tests/cli/test_ensemble_config.py | 27 +++++++++++++++++++-------- 9 files changed, 26 insertions(+), 91 deletions(-) diff --git a/docs/conf_file.rst b/docs/conf_file.rst index b05b2034..9ef76026 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -70,9 +70,9 @@ wall-clock times without conversion. YAML infiltration alternatives are explicit under ``input.infiltration``: ``type: "none"``, ``type: "constant"`` with a ``rate`` map, or -``type: "green-ampt"`` with ``effective_porosity``, ``capillary_pressure``, and -``hydraulic_conductivity`` maps. A list of complete alternatives is one sweep -dimension. The optional ``label`` is documentation only. +``type: "green-ampt"`` with ``effective_porosity``, ``capillary_pressure``, +and ``hydraulic_conductivity`` maps. +A list of complete alternatives is one sweep dimension. ``{ensemble}`` and ``{simulation}`` are the only output-template substitutions. Output maps are always bare names in the active execution mapset; qualified diff --git a/src/itzi/cli_parser.py b/src/itzi/cli_parser.py index 1c42a5a5..91ee3625 100644 --- a/src/itzi/cli_parser.py +++ b/src/itzi/cli_parser.py @@ -4,7 +4,6 @@ import argparse - DESCR = "A dynamic, fully distributed hydraulic and hydrologic model." @@ -44,8 +43,7 @@ def build_parser() -> argparse.ArgumentParser: help=( "Resume a simulation from a hotstart file. " "If only the path to a hotstart file is given, batch is not allowed. " - "For batch processing, use CONFIG_PATH=HOTSTART_PATH or " - "ENSEMBLE#SIMULATION=HOTSTART_PATH." + "For batch processing, use CONFIG_PATH=HOTSTART_PATH." ), ) run_parser.add_argument( diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py index 12d569f4..edf24530 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble.py @@ -44,7 +44,6 @@ SourceDocument, _canonical_json, _canonical_value, - _semantic_value, parse_iso_duration, render_template, ) @@ -257,7 +256,6 @@ def load_yaml_stream(path: str | Path) -> LoadedYamlStream: source = SourceDocument( path=source_path, document_index=document_index, - start_line=start_line, file_digest=file_digest, document_digest=_document_digest(raw_document), ) @@ -307,8 +305,6 @@ def expand_yaml_document( source: SourceDocument, document: YamlEnsembleDocumentV1 ) -> ExpandedEnsemble: """Expand sweep dimensions in canonical path order into scalar simulations.""" - if document.hotstart is not None: - raise EnsembleError("YAML hotstart output requires Stage 2 checkpoint support") normalized_time = normalize_time(document.time) dimensions = _collect_dimensions(document) count = _checked_product(tuple(len(values) for _, values in dimensions)) @@ -324,8 +320,7 @@ def expand_yaml_document( path: value for (path, _), value in zip(dimensions, selected_values, strict=True) } coordinates = tuple( - (path, _freeze_json(_semantic_value(value))) - for path, value in sorted(selected.items()) + (path, _freeze_json(value)) for path, value in sorted(selected.items()) ) input_maps = _selected_input_maps(document.input, selected) infiltration = _normalize_infiltration( @@ -338,7 +333,6 @@ def expand_yaml_document( ExpandedSimulation( source=source, ensemble_id=document.ensemble.id, - ensemble_name=document.ensemble.name, coordinates=coordinates, domain=document.domain, time=normalized_time, @@ -458,9 +452,7 @@ def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: drainage_dataset = ( document.outputs.drainage.vector_dataset if document.outputs.drainage else None ) - hotstart_file = document.hotstart.file if document.hotstart else None - interval = parse_iso_duration(document.hotstart.wallclock_step) if document.hotstart else None - for template in (raster_prefix, statistics_file, drainage_dataset, hotstart_file): + for template in (raster_prefix, statistics_file, drainage_dataset): if template is not None: render_template(template, ensemble=document.ensemble.id, simulation="simulation") return OutputTemplates( @@ -468,8 +460,6 @@ def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: raster_variables=variables, statistics_file=statistics_file, drainage_dataset=drainage_dataset, - hotstart_file=hotstart_file, - hotstart_interval=interval, ) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 95996c80..eaab2dc9 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -94,7 +94,6 @@ class SourceDocument: path: Path document_index: int - start_line: int file_digest: str document_digest: str @@ -171,8 +170,6 @@ class OutputTemplates: raster_variables: tuple[str, ...] statistics_file: str | None drainage_dataset: str | None - hotstart_file: str | None - hotstart_interval: timedelta | None @dataclass(frozen=True) @@ -181,7 +178,6 @@ class ExpandedSimulation: source: SourceDocument ensemble_id: str - ensemble_name: str | None coordinates: tuple[tuple[str, JsonValue], ...] domain: DomainConfig time: NormalizedTime @@ -224,8 +220,6 @@ class ArtifactSummary: class ResolvedSimulation: """Spawn-serializable execution payload with one final core configuration.""" - source: SourceDocument - ensemble_id: str simulation_id: str coordinates: tuple[tuple[str, JsonValue], ...] grass_params: GrassParams @@ -241,25 +235,11 @@ class ResolvedSimulation: class ValidationFailure: """A coordinate that could not become an executable resolved simulation.""" - source: SourceDocument - ensemble_id: str coordinates: tuple[tuple[str, JsonValue], ...] phase: str detail: str -@dataclass(frozen=True) -class ResolvedEnsemble: - """Resolved member payloads and local validation failures for one ensemble.""" - - source: SourceDocument - ensemble_id: str - ensemble_name: str | None - manifest_file: Path - simulations: tuple[ResolvedSimulation, ...] - failures: tuple[ValidationFailure, ...] - - @dataclass(frozen=True) class LoadedYamlStream: """Valid ensembles and document-local failures from one YAML file.""" @@ -310,14 +290,6 @@ def format_iso_duration(value: timedelta) -> str: return "".join(parts) -def _semantic_value(value: Any) -> Any: - if isinstance(value, BaseModel): - dumped = value.model_dump(mode="python") - dumped.pop("label", None) - return dumped - return value - - def _canonical_json(value: Any) -> str: return json.dumps( _canonical_value(value), diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble_schema.py index 0f1654d3..b1f1da9f 100644 --- a/src/itzi/ensemble_schema.py +++ b/src/itzi/ensemble_schema.py @@ -15,7 +15,6 @@ from __future__ import annotations import math -from datetime import timedelta from typing import Annotated, Any, Literal from pydantic import ( @@ -33,8 +32,6 @@ DomainConfig, StrictModel, _canonical_json, - _semantic_value, - parse_iso_duration, ) type SweepString = StrictStr | list[StrictStr] @@ -83,18 +80,15 @@ class AbsoluteEndTimeConfig(StrictModel): class NoInfiltration(StrictModel): type: Literal["none"] - label: StrictStr | None = None class ConstantInfiltration(StrictModel): type: Literal["constant"] - label: StrictStr | None = None rate: StrictStr class GreenAmptInfiltration(StrictModel): type: Literal["green-ampt"] - label: StrictStr | None = None effective_porosity: StrictStr capillary_pressure: StrictStr hydraulic_conductivity: StrictStr @@ -234,25 +228,6 @@ class OutputConfig(StrictModel): manifest: ManifestOutputs | None = None -class HotstartOutputConfig(StrictModel): - wallclock_step: StrictStr - file: StrictStr - - @field_validator("wallclock_step") - @classmethod - def validate_step(cls, value: str) -> str: - if parse_iso_duration(value) <= timedelta(): - raise ValueError("must be strictly positive") - return value - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value - - class YamlEnsembleDocumentV1(StrictModel): schema_version: Literal[1] ensemble: EnsembleMetadata @@ -262,7 +237,6 @@ class YamlEnsembleDocumentV1(StrictModel): options: OptionSweepConfig drainage: DrainageSweepConfig | None = None outputs: OutputConfig - hotstart: HotstartOutputConfig | None = None @model_validator(mode="after") def validate_drainage_output(self) -> YamlEnsembleDocumentV1: @@ -286,7 +260,7 @@ def _validate_sweep( if numeric and (not isinstance(item, float) or not math.isfinite(item)): raise ValueError("must contain finite numbers") - semantic_values = [_canonical_json(_semantic_value(item)) for item in values] + semantic_values = [_canonical_json(item) for item in values] if len(semantic_values) != len(set(semantic_values)): raise ValueError("contains duplicate semantic values") return value diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 5a94d360..6c79b6fe 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -331,8 +331,6 @@ def _run_ensemble_batch(cli_args) -> None: except EnsembleError as error: failures.extend( ValidationFailure( - source=simulation.source, - ensemble_id=ensemble.ensemble_id, coordinates=simulation.coordinates, phase="artifact_validation", detail=str(error), @@ -436,8 +434,6 @@ def _resolve_in_spawn(expanded: ExpandedSimulation) -> ResolvedSimulation | Vali if status == "resolved" and isinstance(payload, ResolvedSimulation): return payload return ValidationFailure( - source=expanded.source, - ensemble_id=expanded.ensemble_id, coordinates=expanded.coordinates, phase="input_resolution", detail=str(payload), diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index 88b16b58..8a1113ef 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -89,8 +89,6 @@ def _resolve_in_active_context( swmm_path, ) return ResolvedSimulation( - source=expanded.source, - ensemble_id=expanded.ensemble_id, simulation_id=simulation_id, coordinates=expanded.coordinates, grass_params=actual_params, diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index c0ee5c1f..71f8bdbb 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -72,7 +72,6 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble: source = SourceDocument( path=source_path, document_index=0, - start_line=1, file_digest=hashlib.blake2b(source_bytes, digest_size=32).hexdigest(), document_digest=hashlib.blake2b(b"legacy-v1\0" + source_bytes, digest_size=32).hexdigest(), ) @@ -127,7 +126,6 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble: expanded = ExpandedSimulation( source=source, ensemble_id=ensemble_id, - ensemble_name=source_path.stem, coordinates=(), domain=DomainConfig(grass=context, region=grass.region, mask=grass.mask), time=normalized_time, @@ -142,8 +140,6 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble: raster_variables=tuple(reader.out_values), statistics_file=stats_file, drainage_dataset=config.drainage_output, - hotstart_file=None, - hotstart_interval=None, ), ) return ExpandedEnsemble(source, ensemble_id, source_path.stem, None, (expanded,)) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 8d253467..83ad53c2 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -157,7 +157,7 @@ def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] -def test_infiltration_alternatives_are_explicit_and_labels_are_not_semantic(tmp_path): +def test_infiltration_alternatives_are_explicit(tmp_path): content = document( extra="""\ input: @@ -165,7 +165,6 @@ def test_infiltration_alternatives_are_explicit_and_labels_are_not_semantic(tmp_ friction: manning infiltration: - type: none - label: "first" - type: constant rate: infiltration_rate options: {} @@ -181,13 +180,25 @@ def test_infiltration_alternatives_are_explicit_and_labels_are_not_semantic(tmp_ InfiltrationModelType.CONSTANT, ] - duplicate = content.replace( - " - type: constant\n rate: infiltration_rate", - ' - type: none\n label: "second"', + labelled = content.replace(" - type: none", ' - type: none\n label: "first"') + labelled_stream = load_yaml_stream(write_yaml(tmp_path, labelled)) + assert labelled_stream.failures[0].phase == "schema" + + +def test_yaml_rejects_unsupported_hotstart(tmp_path): + content = ( + document() + + """\ +hotstart: + wallclock_step: "PT1M" + file: "checkpoint.zip" +""" ) - duplicate_stream = load_yaml_stream(write_yaml(tmp_path, duplicate)) - assert duplicate_stream.failures[0].phase == "schema" - assert "duplicate semantic values" in duplicate_stream.failures[0].detail + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.ensembles == () + assert stream.failures[0].phase == "schema" def test_absolute_offsets_keep_wall_clock_values(tmp_path): From 270798179b1dc866e243b009121b1e2949415fe9 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 17:32:39 -0600 Subject: [PATCH 08/85] simplify ConfigReader --- src/itzi/bmi_itzi.py | 6 ++--- src/itzi/configreader.py | 32 +++++++------------------- src/itzi/itzi.py | 6 ++--- src/itzi/run_plan.py | 6 ++--- tests/cli/test_cli.py | 13 +++-------- tests/cli/test_configreader.py | 22 +++++++++--------- tests/grass/conftest.py | 6 ++--- tests/grass/test_itzi.py | 18 +++++++-------- tests/grass/test_temporal_overwrite.py | 6 ++--- tests/grass/test_tutorial.py | 18 +++++++-------- 10 files changed, 55 insertions(+), 78 deletions(-) diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 4e3e973a..5afe2de8 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -70,15 +70,15 @@ def initialize(self, filename=None) -> None: Path to name of input file. """ conf_data = ConfigReader(filename) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params self.grass_session_manager = GrassSessionManager(grass_params) self.grass_session_manager.open() self.itzi = SimulationRunner( sim_params, grass_params, - stats_file=conf_data.get_stats_file(), + stats_file=conf_data.stats_file, ) def update(self): diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index ce94054c..0efe470b 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -297,7 +297,6 @@ class SimulationTimes(BaseModel): start: datetime end: datetime - duration: timedelta record_step: timedelta | None temporal_type: TemporalType @@ -323,7 +322,6 @@ def from_raw_values(cls, raw_values: dict[str, str | None]) -> SimulationTimes: return cls( start=start, end=end, - duration=duration, record_step=record_step, temporal_type=temporal_type, ) @@ -389,16 +387,14 @@ def __init__(self, filename: str | None) -> None: msgr.fatal("Not a valid configuration file") self.config_file = filename - self.ga_list = list(GREEN_AMPT_KEYS) - self.grass_mandatory = list(GRASS_MANDATORY_KEYS) params = _read_parser(filename) self.hotstart_config = _read_hotstart_values(params) - self.raw_input_times = _read_time_values(params) + raw_time_values = _read_time_values(params) self.input_map_names = _read_input_map_names(params) self.out_prefix, self.out_values, self.output_map_names = _read_output_config(params) self.stats_file = params.get("statistics", "stats_file", fallback=None) or None - self.sim_times = SimulationTimes.from_raw_values(self.raw_input_times) + self.sim_times = SimulationTimes.from_raw_values(raw_time_values) self._check_general_input(self.input_map_names) infiltration_model = self._resolve_infiltration_model(self.input_map_names) @@ -434,18 +430,18 @@ def __init__(self, filename: str | None) -> None: def _check_grass_params(self, grass_params: GrassParams) -> None: """Ensure mandatory GRASS settings are provided together.""" grass_values = grass_params.model_dump() - grass_any = any(grass_values[key] for key in self.grass_mandatory) - grass_all = all(grass_values[key] for key in self.grass_mandatory) + grass_any = any(grass_values[key] for key in GRASS_MANDATORY_KEYS) + grass_all = all(grass_values[key] for key in GRASS_MANDATORY_KEYS) if grass_any and not grass_all: - msgr.fatal(f"{self.grass_mandatory} are mutualy inclusive") + msgr.fatal(f"{GRASS_MANDATORY_KEYS} are mutualy inclusive") def _resolve_infiltration_model( self, input_map_names: dict[str, str | None] ) -> InfiltrationModelType: """Infer the infiltration model from the configured input maps.""" infiltration_input = input_map_names["infiltration"] - ga_any = any(input_map_names[key] for key in self.ga_list) - ga_all = all(input_map_names[key] for key in self.ga_list) + ga_any = any(input_map_names[key] for key in GREEN_AMPT_KEYS) + ga_all = all(input_map_names[key] for key in GREEN_AMPT_KEYS) if not infiltration_input and not ga_any: return InfiltrationModelType.NULL @@ -454,7 +450,7 @@ def _resolve_infiltration_model( if infiltration_input and ga_any: msgr.fatal("Infiltration model incompatible with user-defined rate") if ga_any and not ga_all: - msgr.fatal(f"{self.ga_list} are mutualy inclusive") + msgr.fatal(f"{GREEN_AMPT_KEYS} are mutualy inclusive") return InfiltrationModelType.GREEN_AMPT def _check_general_input(self, input_map_names: dict[str, str | None]) -> None: @@ -471,15 +467,3 @@ def _check_general_input(self, input_map_names: dict[str, str | None]) -> None: msgr.fatal( "inputs and are mutually exclusive." ) - - def get_sim_params(self) -> SimulationConfig: - """Return validated simulation parameters.""" - return self.sim_config - - def get_grass_params(self) -> GrassParams: - """Return validated GRASS GIS session parameters.""" - return self.grass_params - - def get_stats_file(self) -> str | None: - """Return the CSV statistics output file name, if configured.""" - return self.stats_file diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 6c79b6fe..5945d867 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -83,14 +83,14 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: # Run the simulation msgr.message(f"Starting simulation of {os.path.basename(conf_file)}...") conf_data = ConfigReader(conf_file) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params with GrassSessionManager(grass_params): sim_runner = SimulationRunner( sim_params, grass_params, hotstart_path=hotstart_file, - stats_file=conf_data.get_stats_file(), + stats_file=conf_data.stats_file, ) sim_runner.run().finalize() except msgr.FatalError: diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index 71f8bdbb..e0ebb56d 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -63,10 +63,10 @@ def load_batch( def _legacy_ensemble(path: Path) -> ExpandedEnsemble: """Adapt the public one-member INI reader to the scalar batch model.""" reader = ConfigReader(str(path)) - config = reader.get_sim_params() + config = reader.sim_config if config.hotstart_config is not None: raise EnsembleError("hotstart output requires Stage 2 checkpoint support in mixed batches") - grass = reader.get_grass_params() + grass = reader.grass_params source_path = path.expanduser().resolve() source_bytes = source_path.read_bytes() source = SourceDocument( @@ -119,7 +119,7 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble: config.record_step, False, ) - stats_file = reader.get_stats_file() + stats_file = reader.stats_file if stats_file is not None: # The INI deprecation release preserves cwd-relative destinations. stats_file = str((Path.cwd() / stats_file).resolve()) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 3a9d0dd3..d9217a7c 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -151,16 +151,9 @@ def test_worker_passes_statistics_file_to_simulation_runner(monkeypatch): class FakeConfigReader: def __init__(self, _): - pass - - def get_sim_params(self): - return sim_params - - def get_grass_params(self): - return grass_params - - def get_stats_file(self): - return "statistics.csv" + self.sim_config = sim_params + self.grass_params = grass_params + self.stats_file = "statistics.csv" class FakeGrassSessionManager: def __init__(self, received_grass_params): diff --git a/tests/cli/test_configreader.py b/tests/cli/test_configreader.py index 9c9e2d08..90999465 100644 --- a/tests/cli/test_configreader.py +++ b/tests/cli/test_configreader.py @@ -58,12 +58,12 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): config_file = write_config_file(tmp_path, make_config_dict()) reader = ConfigReader(config_file) - sim_config = reader.get_sim_params() - grass_params = reader.get_grass_params() + sim_config = reader.sim_config + grass_params = reader.grass_params assert sim_config.hotstart_config is None assert sim_config.surface_flow_parameters == SurfaceFlowParameters() - assert reader.get_stats_file() is None + assert reader.stats_file is None assert sim_config.dtinf == DefaultValues.DTINF assert sim_config.swmm_inp is None assert sim_config.drainage_output is None @@ -102,8 +102,8 @@ def test_reader_exposes_stats_file_separately_from_simulation_config( reader = ConfigReader(config_file) - assert reader.get_stats_file() == expected_stats_file - assert "stats_file" not in type(reader.get_sim_params()).model_fields + assert reader.stats_file == expected_stats_file + assert "stats_file" not in type(reader.sim_config).model_fields def test_reader_normalizes_deprecated_aliases(tmp_path, caplog): @@ -127,7 +127,7 @@ def test_reader_normalizes_deprecated_aliases(tmp_path, caplog): with caplog.at_level(logging.WARNING, logger="itzi"): itzi_logger.addHandler(caplog.handler) try: - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config finally: itzi_logger.removeHandler(caplog.handler) @@ -189,7 +189,7 @@ def test_reader_prefers_canonical_names_over_input_aliases(tmp_path): ), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.input_map_names["ground_elevation"] == "canonical_elevation" @@ -217,7 +217,7 @@ def test_reader_normalizes_legacy_created_volume_aliases(tmp_path, caplog, legac with caplog.at_level(logging.WARNING, logger="itzi"): itzi_logger.addHandler(caplog.handler) try: - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config finally: itzi_logger.removeHandler(caplog.handler) @@ -268,7 +268,7 @@ def test_reader_accepts_supported_time_combinations( ): config_file = write_config_file(tmp_path, make_config_dict(time=time_section)) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.temporal_type == expected_temporal_type assert sim_config.start_time == expected_start @@ -331,7 +331,7 @@ def test_reader_infers_green_ampt_model_from_complete_parameter_set(tmp_path): ), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.infiltration_model == InfiltrationModelType.GREEN_AMPT @@ -368,7 +368,7 @@ def test_reader_falls_back_to_config_dir_for_relative_swmm_inp(tmp_path, monkeyp make_config_dict(drainage={"swmm_inp": "swmm_config.inp"}), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.swmm_inp == Path(config_swmm_inp) diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index 1fc5512f..57b1df68 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -161,9 +161,9 @@ def grass_5by5_sim(grass_5by5, test_data_path): config_file = os.path.join(test_data_path, "5by5", "5by5.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) assert isinstance(sim_runner, SimulationRunner) sim_runner.run().finalize() diff --git a/tests/grass/test_itzi.py b/tests/grass/test_itzi.py index 47f41cb3..ecfa1f34 100644 --- a/tests/grass/test_itzi.py +++ b/tests/grass/test_itzi.py @@ -90,9 +90,9 @@ def _build_timed_rain_runner( conf_data = ConfigReader(config_file) return SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) @@ -155,12 +155,12 @@ def test_region_mask(test_data_path): # Set simulation (should set region and mask) config_file = os.path.join(test_data_path, "5by5", "5by5_mask.ini") conf_data = ConfigReader(config_file) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params sim_runner = SimulationRunner( sim_params, grass_params, - stats_file=conf_data.get_stats_file(), + stats_file=conf_data.stats_file, ) # Run simulation sim_runner.run().finalize() @@ -216,9 +216,9 @@ def test_fails_when_region_has_no_dem_data(test_data_temp_path): RuntimeError, match=r"input map contains only NULL/NaN cells" ): SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index 954baa64..281b34e6 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -65,9 +65,9 @@ def _build_runner( conf_data = ConfigReader(config_file) return SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) diff --git a/tests/grass/test_tutorial.py b/tests/grass/test_tutorial.py index 8bed70c6..7657e613 100644 --- a/tests/grass/test_tutorial.py +++ b/tests/grass/test_tutorial.py @@ -102,9 +102,9 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) sim_runner.run().finalize() # Check the results @@ -164,9 +164,9 @@ def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, h # Run the simulation conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) sim_runner.run().finalize() @@ -275,9 +275,9 @@ def test_tutorial_drainage_without_vector_output(itzi_tutorial, test_data_path): config_file = os.path.join(test_data_path, "tutorial_files", "tutorial_drainage.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) try: From 7067f39b7f9d400d7f4b98de2367112ea4c825f3 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 17:38:22 -0600 Subject: [PATCH 09/85] remove unused paths --- pyproject.toml | 1 - src/itzi/configreader.py | 1 - src/itzi/data/colortable/default.txt | 11 --- src/itzi/data/example.ini | 97 --------------------------- src/itzi/data/tutorial.ini | 20 ------ src/itzi/ensemble_models.py | 5 -- src/itzi/providers/grass_interface.py | 1 - 7 files changed, 136 deletions(-) delete mode 100644 src/itzi/data/colortable/default.txt delete mode 100644 src/itzi/data/example.ini delete mode 100644 src/itzi/data/tutorial.ini diff --git a/pyproject.toml b/pyproject.toml index aff8c4da..9ff6afa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,6 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -"itzi.data" = ["*.ini", "*.yaml"] "itzi.data.colortable" = ["*.txt"] [tool.pytest.ini_options] diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index 0efe470b..5130b415 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -73,7 +73,6 @@ ) TIME_OPTION_KEYS = ("start_time", "end_time", "duration", "record_step") -HOTSTART_OPTION_KEYS = ("wallclock_step", "save_file") GREEN_AMPT_KEYS = ( "effective_porosity", "capillary_pressure", diff --git a/src/itzi/data/colortable/default.txt b/src/itzi/data/colortable/default.txt deleted file mode 100644 index cdd70fb9..00000000 --- a/src/itzi/data/colortable/default.txt +++ /dev/null @@ -1,11 +0,0 @@ -0% black --10 blue --1 aqua --.1 cyan -0 white -.1 yellow -1 orange -10 red -100% magenta -nv white -default white diff --git a/src/itzi/data/example.ini b/src/itzi/data/example.ini deleted file mode 100644 index 52754f68..00000000 --- a/src/itzi/data/example.ini +++ /dev/null @@ -1,97 +0,0 @@ -[time] -# Start of the simulation. Format yyyy-mm-dd HH:MM -start_time = -# End of the simulation. Format yyyy-mm-dd HH:MM -end_time = -# Duration of the simulation. Format HH:MM:SS -duration = -# Duration between two records. Format HH:MM:SS -record_step = - -[hotstart] -# Wallclock duration between records. Format HH:MM:SS -wallclock_step = 00:15:00 -# File name for the hotstart file -save_file = - -[input] -# ground elevation (raster map/stds) -ground_elevation = -# friction coefficient (raster map/stds) -friction = -# starting water depth (raster map/stds) -water_depth = -# starting water surface elevation (raster map/stds) -water_surface_elevation = - -# rainfall in mm/h (raster map/stds) -rainfall_rate = -# Input user flow in m/s (raster map/stds) -inflow = -# Input boundary conditions type (raster map/stds) -boundary_type = -# Input boundary conditions values (raster map/stds) -boundary_value = - -# Fixed input infiltration rate in mm/h (raster map/stds) -infiltration = -# Effective porosity in mm/mm (raster map/stds) -effective_porosity = -# Wetting front capillary pressure head in mm(raster map/stds) -capillary_pressure = -# Hydraulic conductivity in mm/h (raster map/stds) -hydraulic_conductivity = -# Relative soil water content in mm (raster map/stds) -soil_water_content = - -# user-input losses in mm/h (raster map/strds) -losses = - -[output] -# prefix of output strds -prefix = -# strds to be saved, separated by comma -values = water_depth, max_water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, froude, flow_rate_x, flow_rate_y, created_volume, mean_boundary_flow, mean_infiltration, mean_rainfall, mean_inflow, mean_losses, mean_drainage_flow - -[statistics] -# statistic file -stats_file = - -[options] -# Water depth threshold in metres -hmin = 0.005 -# Slope threshold in m/m -slope_threshold = 0.8 -# Max slope in m/m -max_slope = 0.8 -# CFL coefficient used to calculate time-step -cfl = 0.7 -# Inertia weighting coefficient -theta = 0.9 -# Maximum surface flow time-step in seconds -dtmax = 5 -# Infiltration time-step in seconds -dtinf = 60 -# Maximum created-volume ratio. -max_error = 0.05 - -[drainage] -swmm_inp = -output = -orifice_coeff = 0.167 -free_weir_coeff = 0.54 -submerged_weir_coeff = 0.056 - -[grass] -# Path to the grass binary -grass_bin = -# Path to the GIS DataBase -grassdata = -# name of the location -location = -# name of the mapset -mapset = -# name of the region definition -region = -# name of raster map to be used as a mask -mask = diff --git a/src/itzi/data/tutorial.ini b/src/itzi/data/tutorial.ini deleted file mode 100644 index f3ae8c85..00000000 --- a/src/itzi/data/tutorial.ini +++ /dev/null @@ -1,20 +0,0 @@ -[time] -duration = 02:00:00 -record_step = 00:05:00 - -[input] -ground_elevation = elev_lid792_5m@PERMANENT -friction = n@PERMANENT -rainfall_rate = rain@PERMANENT -boundary_type = bctype@PERMANENT -boundary_value = bcvalue@PERMANENT - -[output] -prefix = nc_itzi_tutorial -values = water_depth, max_water_depth, water_surface_elevation, flow_speed, flow_velocity_direction, mean_boundary_flow - -[statistics] -stats_file = nc_itzi_tutorial.csv - -[options] -cfl = 0.5 diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index eaab2dc9..830d2043 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -65,11 +65,6 @@ "max_slope": DefaultValues.MAX_SLOPE, "max_error": DefaultValues.MAX_ERROR, } -DRAINAGE_DEFAULTS = { - "orifice_coeff": DefaultValues.ORIFICE_COEFF, - "free_weir_coeff": DefaultValues.FREE_WEIR_COEFF, - "submerged_weir_coeff": DefaultValues.SUBMERGED_WEIR_COEFF, -} MAX_ENSEMBLE_MEMBERS = 100 MAX_BATCH_MEMBERS = 200 DURATION_RE = re.compile( diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index f9a4ab03..ff87ac1b 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -48,7 +48,6 @@ RULE_V = _DIR / "velocity.txt" RULE_VDIR = _DIR / "vdir.txt" RULE_FR = _DIR / "froude.txt" -RULE_DEF = _DIR / "default.txt" colors_rules_dict = { "water_depth": str(RULE_H), "max_water_depth": str(RULE_H), From 6fc674334d24a789495b62c8f891647c09dbb25f Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 18:17:43 -0600 Subject: [PATCH 10/85] use HH:MM:SS for durations --- docs/cli.rst | 2 +- docs/conf_file.rst | 7 ++-- src/itzi/ensemble.py | 6 +-- src/itzi/ensemble_models.py | 61 +++++++++++-------------------- src/itzi/resolution.py | 6 +-- tests/cli/test_ensemble_config.py | 43 +++++++++++++++------- tests/grass/test_ensemble.py | 4 +- 7 files changed, 65 insertions(+), 64 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 9d349606..3538be5b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -9,7 +9,7 @@ Run a simulation :func: build_parser :prog: itzi :path: run - :nodefault: + :nodefault: YAML ensembles ~~~~~~~~~~~~~~ diff --git a/docs/conf_file.rst b/docs/conf_file.rst index 9ef76026..f3120e93 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -37,8 +37,8 @@ Basic YAML example mask: "central_city_mask" time: start: "2026-09-01T00:00:00" - duration: "PT2H" - record_step: "PT5M" + duration: "02:00:00" + record_step: "00:05:00" input: ground_elevation: "elevation_5m@PERMANENT" friction: "manning_n@PERMANENT" @@ -64,7 +64,8 @@ contain at most 200. All simulations in an ensemble use one geographical domain, time form, and ``record_step``. Time accepts exactly one of a relative ``duration``, absolute ``start`` plus ``duration``, or absolute ``start`` plus ``end``. Durations use -the supported non-calendar ISO 8601 subset, such as ``"PT5M"`` and ``"P1DT2H"``. +the legacy ``HH:MM:SS`` form, such as ``"00:05:00"`` and ``"26:00:00"``. +Hours may exceed 23; minutes and seconds must be between 0 and 59. Timezone offsets are retained as source metadata but execute as naive GRASS wall-clock times without conversion. diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py index edf24530..54d04b50 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble.py @@ -44,7 +44,7 @@ SourceDocument, _canonical_json, _canonical_value, - parse_iso_duration, + parse_duration, render_template, ) from itzi.ensemble_schema import ( @@ -105,8 +105,8 @@ def _construct_mapping( def normalize_time(value: TimeConfig) -> NormalizedTime: """Validate a public time form without leaking core's relative sentinels.""" - duration = parse_iso_duration(value.duration) if value.duration is not None else None - record_step = parse_iso_duration(value.record_step) + duration = parse_duration(value.duration) if value.duration is not None else None + record_step = parse_duration(value.record_step) start = _parse_datetime(value.start) if value.start is not None else None end = _parse_datetime(value.end) if value.end is not None else None aware = [timestamp for timestamp in (start, end) if timestamp is not None and timestamp.tzinfo] diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 830d2043..0e59a14a 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -16,7 +16,6 @@ import json import math -import re import string from dataclasses import dataclass from datetime import date, datetime, timedelta @@ -67,10 +66,6 @@ } MAX_ENSEMBLE_MEMBERS = 100 MAX_BATCH_MEMBERS = 200 -DURATION_RE = re.compile( - r"P(?:(?P\d+)D)?(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?" - r"(?:(?P\d+(?:\.\d+)?)S)?)?\Z" -) ENSEMBLE_ID_START_CHARS = string.ascii_letters + string.digits ENSEMBLE_ID_CHARS = ENSEMBLE_ID_START_CHARS + "._-" @@ -243,46 +238,34 @@ class LoadedYamlStream: failures: tuple[DocumentFailure, ...] -def parse_iso_duration(value: str) -> timedelta: - """Parse the supported, non-calendar ISO 8601 duration subset.""" - match = DURATION_RE.fullmatch(value) - if match is None or not any(match.groupdict().values()): - raise ValueError("must be an ISO 8601 duration without years or months") - groups = match.groupdict(default="0") - if "T" in value and not any(groups[key] != "0" for key in ("hours", "minutes", "seconds")): - raise ValueError("must include a time component after T") - duration = timedelta( - days=int(groups["days"]), - hours=int(groups["hours"]), - minutes=int(groups["minutes"]), - seconds=float(groups["seconds"]), - ) +def parse_duration(value: str) -> timedelta: + """Parse a strictly positive ``HH:MM:SS`` duration.""" + try: + hours_str, minutes_str, seconds_str = value.split(":") + hours = int(hours_str) + minutes = int(minutes_str) + seconds = int(seconds_str) + except ValueError as error: + raise ValueError("must use HH:MM:SS") from error + if hours < 0 or not 0 <= minutes <= 59 or not 0 <= seconds <= 59: + raise ValueError("must use HH:MM:SS") + duration = timedelta(hours=hours, minutes=minutes, seconds=seconds) if duration <= timedelta(): raise ValueError("must be strictly positive") return duration -def format_iso_duration(value: timedelta) -> str: - """Render a positive duration in a stable ISO 8601 representation.""" - total_seconds = value.total_seconds() - if total_seconds <= 0: +def format_duration(value: timedelta) -> str: + """Render a positive duration for canonical internal data.""" + if value <= timedelta(): raise ValueError("duration must be strictly positive") - days, remainder = divmod(total_seconds, 86400) - hours, remainder = divmod(remainder, 3600) + total_seconds = value.days * 86400 + value.seconds + hours, remainder = divmod(total_seconds, 3600) minutes, seconds = divmod(remainder, 60) - parts = ["P"] - if days: - parts.append(f"{int(days)}D") - if hours or minutes or seconds or not days: - parts.append("T") - if hours: - parts.append(f"{int(hours)}H") - if minutes: - parts.append(f"{int(minutes)}M") - if seconds or (not hours and not minutes): - rendered_seconds = format(seconds, ".6f").rstrip("0").rstrip(".") - parts.append(f"{rendered_seconds}S") - return "".join(parts) + rendered = f"{hours:02}:{minutes:02}:{seconds:02}" + if value.microseconds: + return f"{rendered}.{value.microseconds:06}".rstrip("0") + return rendered def _canonical_json(value: Any) -> str: @@ -307,7 +290,7 @@ def _canonical_value(value: Any) -> Any: if isinstance(value, date): return {"$date": value.isoformat()} if isinstance(value, timedelta): - return {"$duration": format_iso_duration(value)} + return {"$duration": format_duration(value)} if isinstance(value, float) and not math.isfinite(value): raise EnsembleError("NaN and infinity are not supported") return value diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index 8a1113ef..13858566 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -22,7 +22,7 @@ EnsembleError, ExpandedSimulation, ResolvedSimulation, - format_iso_duration, + format_duration, render_template, ) from itzi.grass_session import GrassParams, GrassSessionManager @@ -232,8 +232,8 @@ def _simulation_identity( "temporal_type": str(expanded.time.temporal_type), "start": expanded.time.start.isoformat() if expanded.time.start is not None else None, "end": expanded.time.end.isoformat() if expanded.time.end is not None else None, - "duration": format_iso_duration(expanded.time.duration), - "record_step": format_iso_duration(expanded.time.record_step), + "duration": format_duration(expanded.time.duration), + "record_step": format_duration(expanded.time.record_step), } payload = { "identity_version": 1, diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 83ad53c2..031005f6 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -4,13 +4,14 @@ from datetime import datetime, timedelta +import pytest from itzi_core import InfiltrationModelType, TemporalType from itzi.ensemble import load_yaml_stream from itzi.ensemble_models import ( MAX_ENSEMBLE_MEMBERS, - format_iso_duration, - parse_iso_duration, + format_duration, + parse_duration, render_template, ) @@ -41,8 +42,8 @@ def document(*, ensemble_id: str = "study", extra: str = "") -> str: id: {ensemble_id} domain: {{}} time: - duration: "PT2H" - record_step: "PT5M" + duration: "02:00:00" + record_step: "00:05:00" {body}""" @@ -96,8 +97,8 @@ def test_loader_continues_after_failed_explicit_document(tmp_path): id: first domain: {} time: - duration: "PT1H" - record_step: "PT5M" + duration: "01:00:00" + record_step: "00:05:00" input: ground_elevation: elevation friction: manning @@ -140,8 +141,8 @@ def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): id: invalid-merge domain: {} time: - duration: "PT1H" - record_step: "PT5M" + duration: "01:00:00" + record_step: "00:05:00" input: ground_elevation: elevation friction: manning @@ -190,7 +191,7 @@ def test_yaml_rejects_unsupported_hotstart(tmp_path): document() + """\ hotstart: - wallclock_step: "PT1M" + wallclock_step: "00:01:00" file: "checkpoint.zip" """ ) @@ -203,8 +204,8 @@ def test_yaml_rejects_unsupported_hotstart(tmp_path): def test_absolute_offsets_keep_wall_clock_values(tmp_path): content = document().replace( - ' duration: "PT2H"\n record_step: "PT5M"', - ' start: "2026-09-01T00:00:00+02:00"\n duration: "PT2H"\n record_step: "PT5M"', + ' duration: "02:00:00"\n record_step: "00:05:00"', + ' start: "2026-09-01T00:00:00+02:00"\n duration: "02:00:00"\n record_step: "00:05:00"', ) stream = load_yaml_stream(write_yaml(tmp_path, content)) @@ -237,8 +238,24 @@ def test_expansion_limit_is_checked_before_materialization(tmp_path): def test_duration_and_template_helpers_are_strict(): - assert parse_iso_duration("P1DT2H3M4.5S") == timedelta(days=1, hours=2, minutes=3, seconds=4.5) - assert format_iso_duration(timedelta(minutes=5)) == "PT5M" + for str_value, expected in ( + ("25:02:03", timedelta(hours=25, minutes=2, seconds=3)), + ("123:55:20", timedelta(hours=123, minutes=55, seconds=20)), + ): + assert parse_duration(str_value) == expected + assert format_duration(timedelta(days=1, hours=2, minutes=3, seconds=4)) == "26:03:04" + for value in ( + "PT1H", + "01:00", + "01:60:00", + "00:00:00", + "-01:00:00", + "2:31:00:00", + "une:heure:trente", + "00:90:00", + ): + with pytest.raises(ValueError): + parse_duration(value) assert ( render_template("{{{ensemble}}}-{simulation}", ensemble="study", simulation="sim-a") == "{study}-sim-a" diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 670217a3..1e4023d0 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -22,8 +22,8 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): id: basic domain: {} time: - duration: "PT1M" - record_step: "PT30S" + duration: "00:01:00" + record_step: "00:00:30" input: ground_elevation: "z" friction: "n" From e405df3f47ce45a677ed059fd7c218dde8381ccb Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 19:07:16 -0600 Subject: [PATCH 11/85] make sure stat file names are rendered, and reduce digest size --- src/itzi/resolution.py | 11 +++++++++-- tests/cli/test_ensemble_config.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index 13858566..73b0adb3 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -251,7 +251,7 @@ def _simulation_identity( "drainage": drainage, } canonical = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) - digest = hashlib.blake2b(canonical.encode(), digest_size=16).hexdigest() + digest = hashlib.blake2b(canonical.encode(), digest_size=4).hexdigest() return f"sim-{digest}", canonical @@ -280,7 +280,14 @@ def _render_artifacts( else None ) stats_file = ( - _source_relative_path(expanded.outputs.statistics_file, expanded.source.path.parent) + _source_relative_path( + render_template( + expanded.outputs.statistics_file, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ), + expanded.source.path.parent, + ) if expanded.outputs.statistics_file is not None else None ) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 031005f6..a5c99e95 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import datetime, timedelta +from types import SimpleNamespace import pytest from itzi_core import InfiltrationModelType, TemporalType @@ -14,6 +15,7 @@ parse_duration, render_template, ) +from itzi.resolution import _render_artifacts def write_yaml(tmp_path, content: str): @@ -260,3 +262,22 @@ def test_duration_and_template_helpers_are_strict(): render_template("{{{ensemble}}}-{simulation}", ensemble="study", simulation="sim-a") == "{study}-sim-a" ) + + +def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): + expanded = SimpleNamespace( + ensemble_id="study", + source=SimpleNamespace(path=tmp_path / "study.yaml"), + outputs=SimpleNamespace( + raster_prefix=None, + raster_variables=(), + statistics_file="results/{simulation}.csv", + drainage_dataset=None, + ), + drainage=None, + ) + monkeypatch.setattr("itzi.resolution._validate_output_names", lambda *_: None) + + artifacts = _render_artifacts(expanded, "sim-a", None) + + assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" From 18db9f463c1cf35a1c69c46f5f93d4fb02c0cfac Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 19:22:19 -0600 Subject: [PATCH 12/85] resolve ensemble per ensemble --- src/itzi/itzi.py | 66 +++++++++----- src/itzi/resolution.py | 142 ++++++++++++++++++++++++++---- tests/cli/test_cli.py | 21 +++++ tests/cli/test_ensemble_config.py | 29 +++++- tests/grass/test_ensemble.py | 11 ++- 5 files changed, 226 insertions(+), 43 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 5945d867..40cd038c 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -32,6 +32,7 @@ from datetime import timedelta from importlib.metadata import version from multiprocessing import Process, get_context +from multiprocessing.connection import wait from pathlib import Path from queue import Empty @@ -50,7 +51,7 @@ from itzi.grass_session import GrassSessionManager from itzi.messenger import VerbosityLevel from itzi.resolution import ( - resolve_simulation, + resolve_ensemble, validate_resolved_ensemble, verify_resolved_simulation, ) @@ -133,14 +134,16 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation, result_queue) -> result_queue.put(("execution_failed", f"{type(error).__name__}: {error}")) -def resolver_worker(expanded: ExpandedSimulation, result_queue) -> None: - """Resolve one scalar member in a short-lived spawned GRASS process.""" +def resolver_worker(expanded: tuple[ExpandedSimulation, ...], result_connection) -> None: + """Resolve one ensemble in a short-lived spawned GRASS process.""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) try: - result_queue.put(("resolved", resolve_simulation(expanded))) + result_connection.send(("resolved", resolve_ensemble(expanded))) except Exception as error: - result_queue.put(("validation_failed", f"{type(error).__name__}: {error}")) + result_connection.send(("validation_failed", f"{type(error).__name__}: {error}")) + finally: + result_connection.close() def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: @@ -319,17 +322,16 @@ def _run_ensemble_batch(cli_args) -> None: ] = [] for ensemble in ensembles: successful: list[ResolvedSimulation] = [] - failures: list[ValidationFailure] = [] - for expanded in ensemble.simulations: - result = _resolve_in_spawn(expanded) + member_failures: list[ValidationFailure] = [] + for result in _resolve_ensemble_in_spawn(ensemble.simulations): if isinstance(result, ResolvedSimulation): successful.append(result) else: - failures.append(result) + member_failures.append(result) try: validate_resolved_ensemble(tuple(successful)) except EnsembleError as error: - failures.extend( + member_failures.extend( ValidationFailure( coordinates=simulation.coordinates, phase="artifact_validation", @@ -338,7 +340,7 @@ def _run_ensemble_batch(cli_args) -> None: for simulation in successful ) successful = [] - resolved.append((ensemble, tuple(successful), tuple(failures))) + resolved.append((ensemble, tuple(successful), tuple(member_failures))) selected = _select_members(resolved, getattr(cli_args, "member", [])) if getattr(cli_args, "dry", False): @@ -417,26 +419,44 @@ def _run_ensemble_batch(cli_args) -> None: msgr.message("Simulation(s) complete.") -def _resolve_in_spawn(expanded: ExpandedSimulation) -> ResolvedSimulation | ValidationFailure: +def _resolve_ensemble_in_spawn( + expanded: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: context = get_context("spawn") - result_queue = context.Queue() - process = context.Process(target=resolver_worker, args=(expanded, result_queue)) + receiver, sender = context.Pipe(duplex=False) + process = context.Process(target=resolver_worker, args=(expanded, sender)) process.start() - process.join() + sender.close() try: - status, payload = result_queue.get(timeout=1) - except Empty: + ready = wait((receiver, process.sentinel)) + if receiver in ready: + status, payload = receiver.recv() + else: + process.join() + status, payload = ( + receiver.recv() + if receiver.poll() + else ( + "validation_failed", + f"resolver exited with status {process.exitcode} without returning a result", + ) + ) + except EOFError: status = "validation_failed" payload = f"resolver exited with status {process.exitcode} without returning a result" finally: - result_queue.close() + process.join() + receiver.close() process.close() - if status == "resolved" and isinstance(payload, ResolvedSimulation): + if status == "resolved" and isinstance(payload, tuple): return payload - return ValidationFailure( - coordinates=expanded.coordinates, - phase="input_resolution", - detail=str(payload), + return tuple( + ValidationFailure( + coordinates=simulation.coordinates, + phase="input_resolution", + detail=str(payload), + ) + for simulation in expanded ) diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index 73b0adb3..b229687d 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -22,6 +22,7 @@ EnsembleError, ExpandedSimulation, ResolvedSimulation, + ValidationFailure, format_duration, render_template, ) @@ -35,6 +36,63 @@ def resolve_simulation(expanded: ExpandedSimulation) -> ResolvedSimulation: return _resolve_in_active_context(expanded, requested_params) +def resolve_ensemble( + simulations: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: + """Resolve one ensemble in one GRASS session, retaining member-local failures.""" + if not simulations: + return () + requested_params = _requested_grass_params(simulations[0]) + if any( + _requested_grass_params(simulation) != requested_params for simulation in simulations[1:] + ): + raise EnsembleError("all simulations in an ensemble must use the same GRASS context") + + with GrassSessionManager(requested_params): + import grass.script as gscript + from grass.pygrass.gis import Mapset + from grass.pygrass.gis.region import Region + from grass.pygrass.utils import getenv + + actual_params = _active_grass_params(requested_params, getenv) + domain = _read_domain(simulations[0].domain.region, Region, gscript) + effective_mask = _resolve_effective_mask(simulations[0].domain.mask, getenv) + _initialize_temporal() + input_cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] = {} + swmm_cache: dict[Path, str] = {} + results: list[ResolvedSimulation | ValidationFailure] = [] + for expanded in simulations: + try: + input_names, input_kinds = _resolve_inputs( + dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), + Mapset, + cache=input_cache, + initialize_temporal=False, + ) + swmm_path, swmm_digest = _resolve_swmm_input(expanded, swmm_cache) + results.append( + _build_resolved_simulation( + expanded, + actual_params, + domain, + effective_mask, + input_names, + input_kinds, + swmm_path, + swmm_digest, + ) + ) + except Exception as error: + results.append( + ValidationFailure( + coordinates=expanded.coordinates, + phase="input_resolution", + detail=f"{type(error).__name__}: {error}", + ) + ) + return tuple(results) + + def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams: context = expanded.domain.grass if context is None or context.database is None: @@ -58,14 +116,7 @@ def _resolve_in_active_context( from grass.pygrass.gis.region import Region from grass.pygrass.utils import getenv - actual_params = GrassParams( - grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), - location=getenv("LOCATION_NAME"), - mapset=getenv("MAPSET"), - region=requested_params.region, - mask=requested_params.mask, - grass_bin=requested_params.grass_bin, - ) + actual_params = _active_grass_params(requested_params, getenv) domain = _read_domain(expanded.domain.region, Region, gscript) effective_mask = _resolve_effective_mask(expanded.domain.mask, getenv) input_names, input_kinds = _resolve_inputs( @@ -73,6 +124,39 @@ def _resolve_in_active_context( Mapset, ) swmm_path, swmm_digest = _resolve_swmm_input(expanded) + return _build_resolved_simulation( + expanded, + actual_params, + domain, + effective_mask, + input_names, + input_kinds, + swmm_path, + swmm_digest, + ) + + +def _active_grass_params(requested_params: GrassParams, getenv) -> GrassParams: + return GrassParams( + grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), + location=getenv("LOCATION_NAME"), + mapset=getenv("MAPSET"), + region=requested_params.region, + mask=requested_params.mask, + grass_bin=requested_params.grass_bin, + ) + + +def _build_resolved_simulation( + expanded: ExpandedSimulation, + actual_params: GrassParams, + domain: DomainData, + effective_mask: EffectiveMask, + input_names: dict[str, str], + input_kinds: dict[str, Literal["raster", "strds"]], + swmm_path: Path | None, + swmm_digest: str | None, +) -> ResolvedSimulation: simulation_id, normalized_payload = _simulation_identity( expanded, actual_params, @@ -137,25 +221,46 @@ def _split_identifier(identifier: str) -> tuple[str, str | None]: def _resolve_inputs( inputs: dict[str, str], mapset_type: type, + *, + cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] | None = None, + initialize_temporal: bool = True, ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: + if initialize_temporal: + _initialize_temporal() resolved: dict[str, str] = {} kinds: dict[str, Literal["raster", "strds"]] = {} for key, identifier in inputs.items(): - source, kind = _resolve_input_identifier(identifier, mapset_type) + result = cache.get(identifier) if cache is not None else None + if result is None: + try: + result = _resolve_input_identifier(identifier, mapset_type) + except Exception as error: + if cache is not None: + cache[identifier] = error + raise + if cache is not None: + cache[identifier] = result + if isinstance(result, Exception): + raise result + source, kind = result resolved[key] = source kinds[key] = kind return resolved, kinds +def _initialize_temporal() -> None: + import grass.temporal as tgis + + tgis.init(raise_fatal_error=True) + tgis.set_raise_on_error(True) + + def _resolve_input_identifier( identifier: str, mapset_type: type ) -> tuple[str, Literal["raster", "strds"]]: """Resolve both input namespaces and reject a visible ambiguity.""" - import grass.temporal as tgis from grass.pygrass import utils as gutils - - tgis.init(raise_fatal_error=True) - tgis.set_raise_on_error(True) + import grass.temporal as tgis name, requested_mapset = _split_identifier(identifier) raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") @@ -201,7 +306,9 @@ def _resolve_effective_mask(mask: str | None, getenv) -> EffectiveMask: return EffectiveMask("none", None) -def _resolve_swmm_input(expanded: ExpandedSimulation) -> tuple[Path | None, str | None]: +def _resolve_swmm_input( + expanded: ExpandedSimulation, cache: dict[Path, str] | None = None +) -> tuple[Path | None, str | None]: if expanded.drainage is None: return None, None drainage = dict(expanded.drainage) @@ -210,7 +317,12 @@ def _resolve_swmm_input(expanded: ExpandedSimulation) -> tuple[Path | None, str path = path.resolve() if not path.is_file(): raise EnsembleError(f"SWMM input file <{path}> not found") - return path, f"blake2b:{hashlib.blake2b(path.read_bytes(), digest_size=32).hexdigest()}" + if cache is not None and path in cache: + return path, cache[path] + digest = f"blake2b:{hashlib.blake2b(path.read_bytes(), digest_size=32).hexdigest()}" + if cache is not None: + cache[path] = digest + return path, digest def _simulation_identity( diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index d9217a7c..3fdae395 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2,6 +2,7 @@ import argparse import os +from types import SimpleNamespace import pytest @@ -9,6 +10,7 @@ from itzi.cli_parser import build_parser from itzi.itzi import ( VerbosityLevel, + _run_ensemble_batch, itzi_run, itzi_run_one, main, @@ -49,6 +51,25 @@ def test_run_parser_accepts_yaml_dry_run_and_member_selection(): assert args.member == ["study#sim-a", "study#sim-b"] +def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): + simulations = (object(), object()) + ensemble = SimpleNamespace(ensemble_id="study", simulations=simulations) + calls = [] + + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: ((ensemble,), ())) + monkeypatch.setattr( + "itzi.itzi._resolve_ensemble_in_spawn", + lambda received: calls.append(received) or (), + ) + monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_: None) + + _run_ensemble_batch( + SimpleNamespace(config_file=["study.yaml"], resume_from=[], member=[], dry=True) + ) + + assert calls == [simulations] + + def test_run_parser_rejects_v_and_q_together(): with pytest.raises(SystemExit): build_parser().parse_args(["run", "a.ini", "-v", "-q"]) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index a5c99e95..0aad61d6 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -15,7 +15,7 @@ parse_duration, render_template, ) -from itzi.resolution import _render_artifacts +from itzi.resolution import _render_artifacts, _resolve_inputs def write_yaml(tmp_path, content: str): @@ -281,3 +281,30 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): artifacts = _render_artifacts(expanded, "sim-a", None) assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" + + +def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): + calls = [] + + def resolve(identifier, _mapset_type): + calls.append(identifier) + return f"{identifier}@PERMANENT", "raster" + + monkeypatch.setattr("itzi.resolution._resolve_input_identifier", resolve) + cache = {} + + _resolve_inputs( + {"ground_elevation": "shared", "rainfall_rate": "first"}, + object, + cache=cache, + initialize_temporal=False, + ) + resolved, _ = _resolve_inputs( + {"ground_elevation": "shared", "rainfall_rate": "second"}, + object, + cache=cache, + initialize_temporal=False, + ) + + assert calls == ["shared", "first", "second"] + assert resolved["ground_elevation"] == "shared@PERMANENT" diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 1e4023d0..0817428e 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -28,7 +28,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): ground_elevation: "z" friction: "n" options: - cfl: 0.2 + cfl: [0.2, 0.3] dtmax: 0.3 outputs: rasters: @@ -53,6 +53,9 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): manifest_path = Path(test_data_temp_path) / "basic.manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - member = manifest["members"][0] - assert member["status"] == "completed" - assert member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") + assert len(manifest["members"]) == 2 + assert all(member["status"] == "completed" for member in manifest["members"]) + assert all( + member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") + for member in manifest["members"] + ) From 3d61ab092f847176d896ea20ea9647e0c3081f2e Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 19:46:52 -0600 Subject: [PATCH 13/85] resolve ensemble per ensemble --- src/itzi/simulation_runner.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index d29b9525..be65777d 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -14,6 +14,7 @@ from collections.abc import Mapping from datetime import datetime +from time import monotonic from typing import TYPE_CHECKING, Literal import numpy as np @@ -136,15 +137,19 @@ def __init__( def run(self): """Run a full simulation""" sim_start_time = datetime.now() + last_progress_update = monotonic() - 0.5 msgr.verbose("Starting time-stepping...") while self.sim.sim_time < self.sim.end_time: # display advance of simulation - msgr.percent( - self.sim.start_time, - self.sim.end_time, - self.sim.sim_time, - sim_start_time, - ) + now = monotonic() + if now - last_progress_update >= 0.5: + msgr.percent( + self.sim.start_time, + self.sim.end_time, + self.sim.sim_time, + sim_start_time, + ) + last_progress_update = now # step models self.step() return self From 518b7e31e75b8777b50a5bdd9ad4d2a151c26564 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 20:37:51 -0600 Subject: [PATCH 14/85] fix grass env for exec --- src/itzi/grass_session.py | 9 ++--- src/itzi/simulation_runner.py | 2 +- tests/grass/test_ensemble.py | 66 +++++++++++++++++++++++++++++++++++ tests/test_grass_session.py | 21 +++++++++-- 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/itzi/grass_session.py b/src/itzi/grass_session.py index e5ba6b7d..8f4d19f5 100644 --- a/src/itzi/grass_session.py +++ b/src/itzi/grass_session.py @@ -133,10 +133,11 @@ def open(self): path=gisdb, location=location, mapset=mapset, grass_path=grassbin ) self._owns_session = True - # GRASS 8.4 setup establishes GISRC. The fallback retains support for - # embedders whose setup object deliberately hides its environment. - if self.current_context() is not None: - self._validate_requested_context() + if not os.environ.get("GISRC"): + session_env = getattr(self.grass_session, "env", None) + if session_env and session_env.get("GISRC"): + os.environ.update(session_env) + self._validate_requested_context() def close(self): """Stop GRASS session.""" diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index be65777d..7e5937c4 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -120,7 +120,7 @@ def __init__( sim_builder.with_vector_output_provider(vector_output_provider) if stats_file: if exclusive_stats: - from itzi.providers import ExclusiveCSVMassBalanceOutputProvider + from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider stats_provider = ExclusiveCSVMassBalanceOutputProvider( stats_file, overwrite=self.g_interface.overwrite diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 0817428e..bd1fe0fa 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -4,8 +4,10 @@ import argparse import json +import os from pathlib import Path +import grass.script as gscript import pytest from itzi.itzi import itzi_run @@ -59,3 +61,67 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") for member in manifest["members"] ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_explicit_grass_context_runs_four_members_without_parent_session(test_data_temp_path): + context = gscript.gisenv() + config_path = Path(test_data_temp_path) / "explicit.yaml" + config_path.write_text( + f"""\ +schema_version: 1 +ensemble: + id: explicit +domain: + grass: + database: "{context["GISDBASE"]}" + project: "{context["LOCATION_NAME"]}" + mapset: "{context["MAPSET"]}" + executable: "grass" +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" + water_depth: "start_h" +options: + cfl: [0.2, 0.3] + theta: [0.8, 0.9] + dtmax: 0.3 +outputs: + rasters: + prefix: "explicit_{{simulation}}" + variables: [water_depth] + manifest: + file: "explicit.manifest.json" +""", + encoding="utf-8", + ) + args = argparse.Namespace( + config_file=[str(config_path)], + o=True, + v=None, + q=None, + resume_from=[], + member=[], + dry=False, + ) + + gisrc = os.environ.pop("GISRC") + try: + itzi_run(args) + finally: + os.environ["GISRC"] = gisrc + + manifest_path = Path(test_data_temp_path) / "explicit.manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert len(manifest["members"]) == 4 + assert all(member["status"] == "completed" for member in manifest["members"]) + mapset_path = ( + Path(context["GISDBASE"]) / context["LOCATION_NAME"] / context["MAPSET"] / "cellhd" + ) + assert { + f"{member['artifacts']['rasters']['water_depth']}_0000" for member in manifest["members"] + } <= {path.name for path in mapset_path.iterdir()} diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index 99e5e9c1..c305227f 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -1,3 +1,4 @@ +import os import sys from types import ModuleType, SimpleNamespace @@ -5,15 +6,16 @@ class FakeGrassSession: - def __init__(self) -> None: + def __init__(self, env: dict[str, str]) -> None: + self.env = env self.finished = False def finish(self) -> None: self.finished = True -def test_close_finishes_session_created_by_manager(monkeypatch) -> None: - session = FakeGrassSession() +def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: + session = FakeGrassSession({"GISRC": "/tmp/worker-gisrc"}) grass_script = ModuleType("grass.script") grass_script.setup = SimpleNamespace(init=lambda **_: session) grass_package = ModuleType("grass") @@ -24,6 +26,18 @@ def test_close_finishes_session_created_by_manager(monkeypatch) -> None: monkeypatch.setattr( "itzi.grass_session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" ) + monkeypatch.delenv("GISRC", raising=False) + monkeypatch.setattr( + GrassSessionManager, + "current_context", + staticmethod( + lambda: ( + ("/grassdata", "location", "mapset") + if os.environ.get("GISRC") == session.env["GISRC"] + else None + ) + ), + ) monkeypatch.setitem(sys.modules, "grass", grass_package) monkeypatch.setitem(sys.modules, "grass.script", grass_script) @@ -38,4 +52,5 @@ def test_close_finishes_session_created_by_manager(monkeypatch) -> None: manager.open() manager.close() + assert os.environ["GISRC"] == "/tmp/worker-gisrc" assert session.finished From a98ed7d8fd14adc37e74b139d802624c7f56dafa Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sat, 26 Sep 2026 23:28:09 -0600 Subject: [PATCH 15/85] change manifest file name and save it to yaml --- docs/conf_file.rst | 4 ++++ src/itzi/itzi.py | 28 +++++----------------------- tests/grass/test_ensemble.py | 18 +++++++++--------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/docs/conf_file.rst b/docs/conf_file.rst index f3120e93..c43119b4 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -69,6 +69,10 @@ Hours may exceed 23; minutes and seconds must be between 0 and 59. Timezone offsets are retained as source metadata but execute as naive GRASS wall-clock times without conversion. +The ensemble manifest is written as YAML to +``results/.manifest.yaml`` relative to the configuration file. +Set ``outputs.manifest.file`` to override this path. + YAML infiltration alternatives are explicit under ``input.infiltration``: ``type: "none"``, ``type: "constant"`` with a ``rate`` map, or ``type: "green-ampt"`` with ``effective_porosity``, ``capillary_pressure``, diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 40cd038c..255b704f 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -36,6 +36,8 @@ from pathlib import Path from queue import Empty +import yaml + import itzi.messenger as msgr from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader @@ -522,10 +524,7 @@ def _manifest_path(ensemble: ExpandedEnsemble) -> Path: ) path = Path(rendered).expanduser() return (path if path.is_absolute() else ensemble.source.path.parent / path).resolve() - return ( - ensemble.source.path.parent - / f"{ensemble.source.path.stem}.{ensemble.ensemble_id}.manifest.json" - ) + return ensemble.source.path.parent / "results" / f"{ensemble.ensemble_id}.manifest.yaml" def _validate_manifest_destination( @@ -605,34 +604,17 @@ def _create_manifest( path.parent.mkdir(parents=True, exist_ok=True) mode = "w" if overwrite else "x" with path.open(mode, encoding="utf-8") as file_obj: - import json - - json.dump( - _manifest_document(ensemble, states), - file_obj, - allow_nan=False, - indent=2, - sort_keys=True, - ) - file_obj.write("\n") + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=True) def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: - import json import tempfile with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False ) as file_obj: temporary_path = Path(file_obj.name) - json.dump( - _manifest_document(ensemble, states), - file_obj, - allow_nan=False, - indent=2, - sort_keys=True, - ) - file_obj.write("\n") + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=True) file_obj.flush() os.fsync(file_obj.fileno()) try: diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index bd1fe0fa..d06640d5 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -3,12 +3,12 @@ from __future__ import annotations import argparse -import json import os from pathlib import Path import grass.script as gscript import pytest +import yaml from itzi.itzi import itzi_run @@ -16,7 +16,7 @@ @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): - config_path = Path(test_data_temp_path) / "basic.yaml" + config_path = Path(test_data_temp_path) / "source-file-name.yaml" config_path.write_text( """\ schema_version: 1 @@ -36,8 +36,6 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): rasters: prefix: "basic_{simulation}" variables: [water_depth] - manifest: - file: "basic.manifest.json" """, encoding="utf-8", ) @@ -53,8 +51,10 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): itzi_run(args) - manifest_path = Path(test_data_temp_path) / "basic.manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest_path = Path(test_data_temp_path) / "results" / "basic.manifest.yaml" + manifest_text = manifest_path.read_text(encoding="utf-8") + assert not manifest_text.lstrip().startswith("{") + manifest = yaml.safe_load(manifest_text) assert len(manifest["members"]) == 2 assert all(member["status"] == "completed" for member in manifest["members"]) assert all( @@ -95,7 +95,7 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da prefix: "explicit_{{simulation}}" variables: [water_depth] manifest: - file: "explicit.manifest.json" + file: "explicit-manifest.yaml" """, encoding="utf-8", ) @@ -115,8 +115,8 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da finally: os.environ["GISRC"] = gisrc - manifest_path = Path(test_data_temp_path) / "explicit.manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest_path = Path(test_data_temp_path) / "explicit-manifest.yaml" + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) assert len(manifest["members"]) == 4 assert all(member["status"] == "completed" for member in manifest["members"]) mapset_path = ( From 593e0326985e402e89e9d64c870d3f368b5803cc Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 00:06:57 -0600 Subject: [PATCH 16/85] order manifest --- src/itzi/itzi.py | 34 +++++++++++++++++++++++++++------- tests/grass/test_ensemble.py | 13 +++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 255b704f..690bdc1b 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -35,6 +35,7 @@ from multiprocessing.connection import wait from pathlib import Path from queue import Empty +from typing import cast import yaml @@ -46,6 +47,7 @@ EnsembleError, ExpandedEnsemble, ExpandedSimulation, + JsonValue, ResolvedSimulation, ValidationFailure, render_template, @@ -400,7 +402,14 @@ def _run_ensemble_batch(cli_args) -> None: started = time.monotonic() status, detail = _run_resolved_in_spawn(simulation) states[simulation.simulation_id]["status"] = status - states[simulation.simulation_id]["elapsed_seconds"] = time.monotonic() - started + state = states[simulation.simulation_id] + states[simulation.simulation_id] = { + "simulation_id": state["simulation_id"], + "selected": state["selected"], + "status": state["status"], + "elapsed_seconds": time.monotonic() - started, + **state, + } if detail is not None: states[simulation.simulation_id]["failure"] = detail failure_count += 1 @@ -561,9 +570,9 @@ def _initial_member_states( selected = simulation.simulation_id in selected_ids states[simulation.simulation_id] = { "simulation_id": simulation.simulation_id, - "coordinates": simulation.coordinates, - "status": "planned" if selected else "not_selected", "selected": selected, + "status": "planned" if selected else "not_selected", + "coordinates": _manifest_coordinates(simulation.coordinates), "artifacts": { "rasters": dict(simulation.artifacts.output_map_names), "drainage": simulation.artifacts.drainage_output, @@ -576,14 +585,25 @@ def _initial_member_states( key = f"validation-{index}" states[key] = { "simulation_id": None, - "coordinates": failure.coordinates, - "status": "not_selected" if has_selectors else "validation_failed", "selected": not has_selectors, + "status": "not_selected" if has_selectors else "validation_failed", + "coordinates": _manifest_coordinates(failure.coordinates), "failure": {"phase": failure.phase, "detail": failure.detail}, } return states +def _manifest_coordinates( + coordinates: tuple[tuple[str, JsonValue], ...], +) -> dict[str, JsonValue | dict[str, JsonValue]]: + return { + path: dict(cast(tuple[tuple[str, JsonValue], ...], value)) + if isinstance(value, tuple) + else value + for path, value in coordinates + } + + def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: return { "manifest_version": 1, @@ -604,7 +624,7 @@ def _create_manifest( path.parent.mkdir(parents=True, exist_ok=True) mode = "w" if overwrite else "x" with path.open(mode, encoding="utf-8") as file_obj: - yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=True) + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: @@ -614,7 +634,7 @@ def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, d mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False ) as file_obj: temporary_path = Path(file_obj.name) - yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=True) + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) file_obj.flush() os.fsync(file_obj.fileno()) try: diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index d06640d5..28c897e1 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -29,6 +29,9 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): input: ground_elevation: "z" friction: "n" + infiltration: + - type: "constant" + rate: "infiltration_rate" options: cfl: [0.2, 0.3] dtmax: 0.3 @@ -57,6 +60,16 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): manifest = yaml.safe_load(manifest_text) assert len(manifest["members"]) == 2 assert all(member["status"] == "completed" for member in manifest["members"]) + assert [member["coordinates"] for member in manifest["members"]] == [ + { + "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, + "options.cfl": 0.2, + }, + { + "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, + "options.cfl": 0.3, + }, + ] assert all( member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") for member in manifest["members"] From 3eb0203d858e7cba005da55574cbb1d1e979e295 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 00:14:21 -0600 Subject: [PATCH 17/85] add datetime to manifest --- src/itzi/itzi.py | 3 ++- tests/grass/test_ensemble.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 690bdc1b..fdd074bc 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -29,7 +29,7 @@ import sys import time from collections.abc import Callable -from datetime import timedelta +from datetime import datetime, timedelta from importlib.metadata import version from multiprocessing import Process, get_context from multiprocessing.connection import wait @@ -607,6 +607,7 @@ def _manifest_coordinates( def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: return { "manifest_version": 1, + "last_updated_at": datetime.now().astimezone().isoformat(), "ensemble": {"id": ensemble.ensemble_id, "name": ensemble.ensemble_name}, "source": { "path": str(ensemble.source.path), diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 28c897e1..e7008e9b 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -4,6 +4,7 @@ import argparse import os +from datetime import datetime from pathlib import Path import grass.script as gscript @@ -57,7 +58,11 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): manifest_path = Path(test_data_temp_path) / "results" / "basic.manifest.yaml" manifest_text = manifest_path.read_text(encoding="utf-8") assert not manifest_text.lstrip().startswith("{") + assert manifest_text.splitlines()[1].startswith("last_updated_at: ") manifest = yaml.safe_load(manifest_text) + last_updated_at = manifest["last_updated_at"] + assert isinstance(last_updated_at, str) + assert datetime.fromisoformat(last_updated_at).utcoffset() is not None assert len(manifest["members"]) == 2 assert all(member["status"] == "completed" for member in manifest["members"]) assert [member["coordinates"] for member in manifest["members"]] == [ From f956b719e26cf420b4e8bae4d6afafb02eb5eece Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 07:26:06 -0600 Subject: [PATCH 18/85] move manifest functions into their own file --- src/itzi/itzi.py | 141 ++++------------------------------------ src/itzi/manifest.py | 149 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 129 deletions(-) create mode 100644 src/itzi/manifest.py diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index fdd074bc..3a422ffe 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -29,15 +29,12 @@ import sys import time from collections.abc import Callable -from datetime import datetime, timedelta +from datetime import timedelta from importlib.metadata import version from multiprocessing import Process, get_context from multiprocessing.connection import wait from pathlib import Path from queue import Empty -from typing import cast - -import yaml import itzi.messenger as msgr from itzi.cli_parser import build_parser @@ -47,12 +44,17 @@ EnsembleError, ExpandedEnsemble, ExpandedSimulation, - JsonValue, ResolvedSimulation, ValidationFailure, - render_template, ) from itzi.grass_session import GrassSessionManager +from itzi.manifest import ( + _create_manifest, + _initial_member_states, + _manifest_path, + _update_manifest, + _validate_manifest_destination, +) from itzi.messenger import VerbosityLevel from itzi.resolution import ( resolve_ensemble, @@ -305,10 +307,10 @@ def _run_legacy_batch(cli_args) -> None: else: msgr.message("Simulation(s) complete. Elapsed times:") for f, t in times_list: - msgr.message("{}: {}".format(f, t)) - msgr.message("Total: {}".format(total_elapsed_time)) + msgr.message(f"{f}: {t}") + msgr.message(f"Total: {total_elapsed_time}") avg_time_s = int(total_elapsed_time.total_seconds() / len(times_list)) - msgr.message("Average: {}".format(timedelta(seconds=avg_time_s))) + msgr.message(f"Average: {timedelta(seconds=avg_time_s)}") if failed_files: msgr.fatal(f"{len(failed_files)} simulation(s) failed") @@ -316,7 +318,7 @@ def _run_legacy_batch(cli_args) -> None: def _run_ensemble_batch(cli_args) -> None: """Resolve and execute an ordered YAML/INI batch through spawn boundaries.""" if getattr(cli_args, "resume_from", []): - msgr.fatal("Resume is not available for YAML ensembles until Stage 2") + msgr.fatal("Resume is not available for YAML ensembles.") ensembles, document_failures = load_batch(cli_args.config_file) for failure in document_failures: msgr.warning(failure.format()) @@ -526,125 +528,6 @@ def _select_members( return selected -def _manifest_path(ensemble: ExpandedEnsemble) -> Path: - if ensemble.manifest_template is not None: - rendered = render_template( - ensemble.manifest_template, ensemble=ensemble.ensemble_id, simulation=None - ) - path = Path(rendered).expanduser() - return (path if path.is_absolute() else ensemble.source.path.parent / path).resolve() - return ensemble.source.path.parent / "results" / f"{ensemble.ensemble_id}.manifest.yaml" - - -def _validate_manifest_destination( - manifest_path: Path, - ensemble: ExpandedEnsemble, - simulations: tuple[ResolvedSimulation, ...], -) -> None: - """Keep the parent-owned manifest away from files required by the run.""" - protected = {ensemble.source.path.resolve()} - for simulation in simulations: - if simulation.simulation_config.swmm_inp is not None: - protected.add(simulation.simulation_config.swmm_inp.resolve()) - if simulation.artifacts.statistics_file is not None: - protected.add(simulation.artifacts.statistics_file.resolve()) - if manifest_path in protected: - raise EnsembleError( - f"manifest <{manifest_path}> aliases a protected input or member artifact" - ) - for path in protected: - if manifest_path.exists() and path.exists() and manifest_path.samefile(path): - raise EnsembleError( - f"manifest <{manifest_path}> aliases a protected input or member artifact" - ) - - -def _initial_member_states( - simulations: tuple[ResolvedSimulation, ...], - failures: tuple[ValidationFailure, ...], - selected_ids: set[str], - has_selectors: bool, -) -> dict[str, dict]: - states: dict[str, dict] = {} - for simulation in simulations: - selected = simulation.simulation_id in selected_ids - states[simulation.simulation_id] = { - "simulation_id": simulation.simulation_id, - "selected": selected, - "status": "planned" if selected else "not_selected", - "coordinates": _manifest_coordinates(simulation.coordinates), - "artifacts": { - "rasters": dict(simulation.artifacts.output_map_names), - "drainage": simulation.artifacts.drainage_output, - "statistics": str(simulation.artifacts.statistics_file) - if simulation.artifacts.statistics_file is not None - else None, - }, - } - for index, failure in enumerate(failures): - key = f"validation-{index}" - states[key] = { - "simulation_id": None, - "selected": not has_selectors, - "status": "not_selected" if has_selectors else "validation_failed", - "coordinates": _manifest_coordinates(failure.coordinates), - "failure": {"phase": failure.phase, "detail": failure.detail}, - } - return states - - -def _manifest_coordinates( - coordinates: tuple[tuple[str, JsonValue], ...], -) -> dict[str, JsonValue | dict[str, JsonValue]]: - return { - path: dict(cast(tuple[tuple[str, JsonValue], ...], value)) - if isinstance(value, tuple) - else value - for path, value in coordinates - } - - -def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: - return { - "manifest_version": 1, - "last_updated_at": datetime.now().astimezone().isoformat(), - "ensemble": {"id": ensemble.ensemble_id, "name": ensemble.ensemble_name}, - "source": { - "path": str(ensemble.source.path), - "document_index": ensemble.source.document_index, - "file_digest": ensemble.source.file_digest, - "document_digest": ensemble.source.document_digest, - }, - "members": list(states.values()), - } - - -def _create_manifest( - path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict], *, overwrite: bool -) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - mode = "w" if overwrite else "x" - with path.open(mode, encoding="utf-8") as file_obj: - yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) - - -def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: - import tempfile - - with tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False - ) as file_obj: - temporary_path = Path(file_obj.name) - yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) - file_obj.flush() - os.fsync(file_obj.fileno()) - try: - temporary_path.replace(path) - except Exception: - temporary_path.unlink(missing_ok=True) - raise - - def _display_dry_plan( resolved: list[ tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py new file mode 100644 index 00000000..14902cfd --- /dev/null +++ b/src/itzi/manifest.py @@ -0,0 +1,149 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import os +import tempfile +from datetime import datetime +from pathlib import Path +from typing import cast + +import yaml + +from itzi.ensemble_models import ( + EnsembleError, + ExpandedEnsemble, + JsonValue, + ResolvedSimulation, + ValidationFailure, + render_template, +) + + +def _manifest_path(ensemble: ExpandedEnsemble) -> Path: + if ensemble.manifest_template is not None: + rendered = render_template( + ensemble.manifest_template, ensemble=ensemble.ensemble_id, simulation=None + ) + path = Path(rendered).expanduser() + return (path if path.is_absolute() else ensemble.source.path.parent / path).resolve() + return ensemble.source.path.parent / "results" / f"{ensemble.ensemble_id}.manifest.yaml" + + +def _validate_manifest_destination( + manifest_path: Path, + ensemble: ExpandedEnsemble, + simulations: tuple[ResolvedSimulation, ...], +) -> None: + """Keep the parent-owned manifest away from files required by the run.""" + protected = {ensemble.source.path.resolve()} + for simulation in simulations: + if simulation.simulation_config.swmm_inp is not None: + protected.add(simulation.simulation_config.swmm_inp.resolve()) + if simulation.artifacts.statistics_file is not None: + protected.add(simulation.artifacts.statistics_file.resolve()) + if manifest_path in protected: + raise EnsembleError( + f"manifest <{manifest_path}> aliases a protected input or member artifact" + ) + for path in protected: + if manifest_path.exists() and path.exists() and manifest_path.samefile(path): + raise EnsembleError( + f"manifest <{manifest_path}> aliases a protected input or member artifact" + ) + + +def _initial_member_states( + simulations: tuple[ResolvedSimulation, ...], + failures: tuple[ValidationFailure, ...], + selected_ids: set[str], + has_selectors: bool, +) -> dict[str, dict]: + states: dict[str, dict] = {} + for simulation in simulations: + selected = simulation.simulation_id in selected_ids + states[simulation.simulation_id] = { + "simulation_id": simulation.simulation_id, + "selected": selected, + "status": "planned" if selected else "not_selected", + "coordinates": _manifest_coordinates(simulation.coordinates), + "artifacts": { + "rasters": dict(simulation.artifacts.output_map_names), + "drainage": simulation.artifacts.drainage_output, + "statistics": str(simulation.artifacts.statistics_file) + if simulation.artifacts.statistics_file is not None + else None, + }, + } + for index, failure in enumerate(failures): + key = f"validation-{index}" + states[key] = { + "simulation_id": None, + "selected": not has_selectors, + "status": "not_selected" if has_selectors else "validation_failed", + "coordinates": _manifest_coordinates(failure.coordinates), + "failure": {"phase": failure.phase, "detail": failure.detail}, + } + return states + + +def _manifest_coordinates( + coordinates: tuple[tuple[str, JsonValue], ...], +) -> dict[str, JsonValue | dict[str, JsonValue]]: + return { + path: dict(cast(tuple[tuple[str, JsonValue], ...], value)) + if isinstance(value, tuple) + else value + for path, value in coordinates + } + + +def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: + return { + "manifest_version": 1, + "last_updated_at": datetime.now().astimezone().isoformat(), + "ensemble": {"id": ensemble.ensemble_id, "name": ensemble.ensemble_name}, + "source": { + "path": str(ensemble.source.path), + "document_index": ensemble.source.document_index, + "file_digest": ensemble.source.file_digest, + "document_digest": ensemble.source.document_digest, + }, + "members": list(states.values()), + } + + +def _create_manifest( + path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict], *, overwrite: bool +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + mode = "w" if overwrite else "x" + with path.open(mode, encoding="utf-8") as file_obj: + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) + + +def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as file_obj: + temporary_path = Path(file_obj.name) + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) + file_obj.flush() + os.fsync(file_obj.fileno()) + try: + temporary_path.replace(path) + except Exception: + temporary_path.unlink(missing_ok=True) + raise From ba993ea91bd8d5330221e0908b8067411d386b15 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 08:41:34 -0600 Subject: [PATCH 19/85] simplify ensemble validation --- src/itzi/ensemble.py | 14 +------ src/itzi/ensemble_models.py | 2 - src/itzi/ensemble_schema.py | 68 +++++++------------------------ tests/cli/test_ensemble_config.py | 25 ++++++++++++ 4 files changed, 41 insertions(+), 68 deletions(-) diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py index 54d04b50..f9483a03 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble.py @@ -16,6 +16,7 @@ import hashlib import itertools +import math from datetime import datetime, timedelta from pathlib import Path from typing import Any @@ -307,7 +308,7 @@ def expand_yaml_document( """Expand sweep dimensions in canonical path order into scalar simulations.""" normalized_time = normalize_time(document.time) dimensions = _collect_dimensions(document) - count = _checked_product(tuple(len(values) for _, values in dimensions)) + count = math.prod(len(values) for _, values in dimensions) if count > MAX_ENSEMBLE_MEMBERS: sizes = ", ".join(f"{path}={len(values)}" for path, values in dimensions) raise EnsembleError( @@ -377,17 +378,6 @@ def _collect_dimensions( return tuple(sorted(dimensions, key=lambda item: item[0])) -def _checked_product(sizes: tuple[int, ...]) -> int: - product = 1 - for size in sizes: - if size <= 0: - raise EnsembleError("sweep lists must not be empty") - product *= size - if product > MAX_ENSEMBLE_MEMBERS: - return product - return product - - def _selected_input_maps( input_config: InputSweepConfig, selected: dict[str, Any] ) -> dict[str, str]: diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 0e59a14a..615bfe58 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -66,8 +66,6 @@ } MAX_ENSEMBLE_MEMBERS = 100 MAX_BATCH_MEMBERS = 200 -ENSEMBLE_ID_START_CHARS = string.ascii_letters + string.digits -ENSEMBLE_ID_CHARS = ENSEMBLE_ID_START_CHARS + "._-" type JsonValue = ( None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble_schema.py index b1f1da9f..8ef051f2 100644 --- a/src/itzi/ensemble_schema.py +++ b/src/itzi/ensemble_schema.py @@ -14,7 +14,6 @@ from __future__ import annotations -import math from typing import Annotated, Any, Literal from pydantic import ( @@ -26,33 +25,21 @@ ) from itzi.ensemble_models import ( - ENSEMBLE_ID_CHARS, - ENSEMBLE_ID_START_CHARS, OUTPUT_KEYS, DomainConfig, StrictModel, _canonical_json, ) -type SweepString = StrictStr | list[StrictStr] -type SweepFloat = StrictFloat | list[StrictFloat] +type SweepString = StrictStr | Annotated[list[StrictStr], Field(min_length=1)] +type FiniteFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] +type SweepFloat = FiniteFloat | Annotated[list[FiniteFloat], Field(min_length=1)] class EnsembleMetadata(StrictModel): - id: StrictStr + id: StrictStr = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$") name: StrictStr | None = None - @field_validator("id") - @classmethod - def validate_id(cls, value: str) -> str: - if ( - not value - or value[0] not in ENSEMBLE_ID_START_CHARS - or any(character not in ENSEMBLE_ID_CHARS for character in value[1:]) - ): - raise ValueError("must match [A-Za-z0-9][A-Za-z0-9._-]*") - return value - class RelativeTimeConfig(StrictModel): duration: StrictStr @@ -99,6 +86,9 @@ class GreenAmptInfiltration(StrictModel): NoInfiltration | ConstantInfiltration | GreenAmptInfiltration, Field(discriminator="type"), ] +type SweepInfiltration = ( + InfiltrationAlternative | Annotated[list[InfiltrationAlternative], Field(min_length=1)] +) class InputSweepConfig(StrictModel): @@ -111,9 +101,7 @@ class InputSweepConfig(StrictModel): inflow: SweepString | None = None boundary_value: SweepString | None = None boundary_type: SweepString | None = None - infiltration: InfiltrationAlternative | list[InfiltrationAlternative] = NoInfiltration( - type="none" - ) + infiltration: SweepInfiltration = NoInfiltration(type="none") @field_validator( "ground_elevation", @@ -132,9 +120,7 @@ def validate_string_sweep(cls, value: SweepString | None) -> SweepString | None: @field_validator("infiltration") @classmethod - def validate_infiltration_sweep( - cls, value: InfiltrationAlternative | list[InfiltrationAlternative] - ) -> InfiltrationAlternative | list[InfiltrationAlternative]: + def validate_infiltration_sweep(cls, value: SweepInfiltration) -> SweepInfiltration: return _validate_sweep(value) @model_validator(mode="after") @@ -158,7 +144,7 @@ class OptionSweepConfig(StrictModel): @field_validator("*") @classmethod def validate_numeric_sweep(cls, value: SweepFloat | None) -> SweepFloat | None: - return _validate_sweep(value, numeric=True) + return _validate_sweep(value) class DrainageSweepConfig(StrictModel): @@ -175,7 +161,7 @@ def validate_input_sweep(cls, value: SweepString) -> SweepString: @field_validator("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") @classmethod def validate_coeff_sweep(cls, value: SweepFloat) -> SweepFloat: - return _validate_sweep(value, numeric=True) + return _validate_sweep(value) class RasterOutputs(StrictModel): @@ -196,14 +182,7 @@ def validate_variables(cls, value: list[str]) -> list[str]: class StatisticsOutputs(StrictModel): - file: StrictStr - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value + file: StrictStr = Field(min_length=1) class DrainageOutputs(StrictModel): @@ -211,14 +190,7 @@ class DrainageOutputs(StrictModel): class ManifestOutputs(StrictModel): - file: StrictStr - - @field_validator("file") - @classmethod - def validate_file(cls, value: str) -> str: - if not value: - raise ValueError("must not be empty") - return value + file: StrictStr = Field(min_length=1) class OutputConfig(StrictModel): @@ -245,21 +217,9 @@ def validate_drainage_output(self) -> YamlEnsembleDocumentV1: return self -def _validate_sweep( - value: Any, - *, - numeric: bool = False, -) -> Any: +def _validate_sweep(value: Any) -> Any: """Validate one scalar-or-list sweep field after Pydantic's type validation.""" values = value if isinstance(value, list) else [value] - if not values: - raise ValueError("sweep lists must not be empty") - for item in values: - if isinstance(item, list): - raise ValueError("nested sweep lists are not supported") # noqa: TRY004 - if numeric and (not isinstance(item, float) or not math.isfinite(item)): - raise ValueError("must contain finite numbers") - semantic_values = [_canonical_json(item) for item in values] if len(semantic_values) != len(set(semantic_values)): raise ValueError("contains duplicate semantic values") diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 0aad61d6..dd0acb15 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -7,6 +7,7 @@ import pytest from itzi_core import InfiltrationModelType, TemporalType +from pydantic import ValidationError from itzi.ensemble import load_yaml_stream from itzi.ensemble_models import ( @@ -15,6 +16,13 @@ parse_duration, render_template, ) +from itzi.ensemble_schema import ( + EnsembleMetadata, + InputSweepConfig, + ManifestOutputs, + OptionSweepConfig, + StatisticsOutputs, +) from itzi.resolution import _render_artifacts, _resolve_inputs @@ -62,6 +70,23 @@ def test_scalar_document_expands_to_one_member(tmp_path): assert simulation.time.duration == timedelta(hours=2) +@pytest.mark.parametrize( + ("model", "value"), + ( + (EnsembleMetadata, {"id": "invalid!"}), + (InputSweepConfig, {"ground_elevation": [], "friction": "manning"}), + (InputSweepConfig, {"ground_elevation": ["elevation"], "friction": [["manning"]]}), + (OptionSweepConfig, {"cfl": [float("nan")]}), + (OptionSweepConfig, {"cfl": [0.5, 0.5]}), + (StatisticsOutputs, {"file": ""}), + (ManifestOutputs, {"file": ""}), + ), +) +def test_schema_constraints_reject_invalid_values(model, value): + with pytest.raises(ValidationError): + model.model_validate(value) + + def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): content = document( extra="""\ From 405660fc4d43e90ef73b9af895151a2a86351495 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 08:46:43 -0600 Subject: [PATCH 20/85] simplify argument parsing in tests --- tests/grass/test_ensemble.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index e7008e9b..6a1aa856 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -2,7 +2,6 @@ from __future__ import annotations -import argparse import os from datetime import datetime from pathlib import Path @@ -11,6 +10,7 @@ import pytest import yaml +from itzi.cli_parser import build_parser from itzi.itzi import itzi_run @@ -43,15 +43,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): """, encoding="utf-8", ) - args = argparse.Namespace( - config_file=[str(config_path)], - o=True, - v=None, - q=None, - resume_from=[], - member=[], - dry=False, - ) + args = build_parser().parse_args(["run", str(config_path), "-o"]) itzi_run(args) @@ -117,15 +109,7 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da """, encoding="utf-8", ) - args = argparse.Namespace( - config_file=[str(config_path)], - o=True, - v=None, - q=None, - resume_from=[], - member=[], - dry=False, - ) + args = build_parser().parse_args(["run", str(config_path), "-o"]) gisrc = os.environ.pop("GISRC") try: From d8714bbb9857c63686887dc451996ccc32ebbbe1 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 08:59:05 -0600 Subject: [PATCH 21/85] simplify ensemble resolution and manifest update --- src/itzi/itzi.py | 16 +++---------- src/itzi/resolution.py | 39 +------------------------------ src/itzi/run_plan.py | 26 +++++++-------------- tests/cli/test_ensemble_config.py | 2 +- 4 files changed, 13 insertions(+), 70 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 3a422ffe..c5c339ea 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -40,7 +40,6 @@ from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader from itzi.ensemble_models import ( - DocumentFailure, EnsembleError, ExpandedEnsemble, ExpandedSimulation, @@ -350,7 +349,7 @@ def _run_ensemble_batch(cli_args) -> None: selected = _select_members(resolved, getattr(cli_args, "member", [])) if getattr(cli_args, "dry", False): - _display_dry_plan(resolved, selected, document_failures) + _display_dry_plan(resolved, selected) if document_failures or any(failures for _, _, failures in resolved): msgr.fatal("YAML batch validation failed") return @@ -403,15 +402,9 @@ def _run_ensemble_batch(cli_args) -> None: break started = time.monotonic() status, detail = _run_resolved_in_spawn(simulation) - states[simulation.simulation_id]["status"] = status state = states[simulation.simulation_id] - states[simulation.simulation_id] = { - "simulation_id": state["simulation_id"], - "selected": state["selected"], - "status": state["status"], - "elapsed_seconds": time.monotonic() - started, - **state, - } + state["status"] = status + state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") if detail is not None: states[simulation.simulation_id]["failure"] = detail failure_count += 1 @@ -533,7 +526,6 @@ def _display_dry_plan( tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] ], selected: dict[str, set[str]], - document_failures: tuple[DocumentFailure, ...], ) -> None: for ensemble, simulations, failures in resolved: msgr.message(f"Ensemble {ensemble.ensemble_id}:") @@ -546,8 +538,6 @@ def _display_dry_plan( msgr.message(f" {simulation.simulation_id} ({selection})") for failure in failures: msgr.warning(f" validation failed: {failure.detail}") - for failure in document_failures: - msgr.warning(failure.format()) def itzi_version(cli_args): diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index b229687d..e452689b 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -29,13 +29,6 @@ from itzi.grass_session import GrassParams, GrassSessionManager -def resolve_simulation(expanded: ExpandedSimulation) -> ResolvedSimulation: - """Resolve one scalar member inside an already selected GRASS context.""" - requested_params = _requested_grass_params(expanded) - with GrassSessionManager(requested_params): - return _resolve_in_active_context(expanded, requested_params) - - def resolve_ensemble( simulations: tuple[ExpandedSimulation, ...], ) -> tuple[ResolvedSimulation | ValidationFailure, ...]: @@ -107,35 +100,6 @@ def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams: ) -def _resolve_in_active_context( - expanded: ExpandedSimulation, requested_params: GrassParams -) -> ResolvedSimulation: - """Resolve GRASS sources and create the final core configuration.""" - import grass.script as gscript - from grass.pygrass.gis import Mapset - from grass.pygrass.gis.region import Region - from grass.pygrass.utils import getenv - - actual_params = _active_grass_params(requested_params, getenv) - domain = _read_domain(expanded.domain.region, Region, gscript) - effective_mask = _resolve_effective_mask(expanded.domain.mask, getenv) - input_names, input_kinds = _resolve_inputs( - dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), - Mapset, - ) - swmm_path, swmm_digest = _resolve_swmm_input(expanded) - return _build_resolved_simulation( - expanded, - actual_params, - domain, - effective_mask, - input_names, - input_kinds, - swmm_path, - swmm_digest, - ) - - def _active_grass_params(requested_params: GrassParams, getenv) -> GrassParams: return GrassParams( grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), @@ -165,7 +129,7 @@ def _build_resolved_simulation( input_names, swmm_digest, ) - artifacts = _render_artifacts(expanded, simulation_id, actual_params.mapset) + artifacts = _render_artifacts(expanded, simulation_id) simulation_config = _build_simulation_config( expanded, input_names, @@ -370,7 +334,6 @@ def _simulation_identity( def _render_artifacts( expanded: ExpandedSimulation, simulation_id: str, - mapset: str | None, ) -> ArtifactSummary: output_map_names: dict[str, str] = {} if expanded.outputs.raster_prefix is not None: diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index e0ebb56d..adf76c1c 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -101,24 +101,14 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble: "free_weir_coeff": config.free_weir_coeff, "submerged_weir_coeff": config.submerged_weir_coeff, } - if config.temporal_type == TemporalType.RELATIVE: - normalized_time = NormalizedTime( - TemporalType.RELATIVE, - None, - None, - config.end_time - config.start_time, - config.record_step, - False, - ) - else: - normalized_time = NormalizedTime( - TemporalType.ABSOLUTE, - config.start_time, - config.end_time, - config.end_time - config.start_time, - config.record_step, - False, - ) + normalized_time = NormalizedTime( + config.temporal_type, + None if config.temporal_type == TemporalType.RELATIVE else config.start_time, + None if config.temporal_type == TemporalType.RELATIVE else config.end_time, + config.end_time - config.start_time, + config.record_step, + False, + ) stats_file = reader.stats_file if stats_file is not None: # The INI deprecation release preserves cwd-relative destinations. diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index dd0acb15..3f035c36 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -303,7 +303,7 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): ) monkeypatch.setattr("itzi.resolution._validate_output_names", lambda *_: None) - artifacts = _render_artifacts(expanded, "sim-a", None) + artifacts = _render_artifacts(expanded, "sim-a") assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" From c184523711e0c6f6960ebc4cc8138b624c454608 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 09:14:37 -0600 Subject: [PATCH 22/85] simplify grass input and interface --- src/itzi/providers/grass_input.py | 36 +++++++++++---------------- src/itzi/providers/grass_interface.py | 9 ------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index d6569bf8..30874f7c 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -81,30 +81,24 @@ def _get_map_lists( map_list = None continue map_id = self.grass_interface.format_id(map_name) - expected_kind = self.input_kinds.get(k) if self.input_kinds is not None else None - if expected_kind == "strds": + kind = self.input_kinds.get(k) if self.input_kinds is not None else None + if kind == "strds": if not self.grass_interface.name_is_stds(map_id): - msgr.fatal(f"Resolved STRDS input <{map_id}> is no longer available") - strds_id = map_id - if not self.grass_interface.stds_temporal_sanity(strds_id): - msgr.fatal(f"{map_name}: inadequate temporal format") - map_list = self.grass_interface.raster_list_from_strds(strds_id) - elif expected_kind == "raster": + msgr.fatal(f"STRDS input <{map_id}> is no longer available") + elif kind == "raster": if not self.grass_interface.name_is_map(map_id): - msgr.fatal(f"Resolved raster input <{map_id}> is no longer available") - map_list = [ - MapData( - id=map_id, - start_time=self.start_time, - end_time=self.end_time, - ) - ] + msgr.fatal(f"raster input <{map_id}> is no longer available") elif self.grass_interface.name_is_stds(map_id): - strds_id = map_id - if not self.grass_interface.stds_temporal_sanity(strds_id): - msgr.fatal(f"{map_name}: inadequate temporal format") - map_list = self.grass_interface.raster_list_from_strds(strds_id) + kind = "strds" elif self.grass_interface.name_is_map(map_id): + kind = "raster" + else: + msgr.fatal(f"{map_name} not found!") + if kind == "strds": + if not self.grass_interface.stds_temporal_sanity(map_id): + msgr.fatal(f"{map_name}: inadequate temporal format") + map_list = self.grass_interface.raster_list_from_strds(map_id) + else: map_list = [ MapData( id=map_id, @@ -112,8 +106,6 @@ def _get_map_lists( end_time=self.end_time, ) ] - else: - msgr.fatal(f"{map_name} not found!") map_lists[k] = map_list return map_lists diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index ff87ac1b..2543c597 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -320,15 +320,6 @@ def get_npmask(self) -> np.ndarray: # A mapset MASK uses CELL semantics: zero and NULL are outside. return np.isnan(grass_mask) | (grass_mask == 0) - def set_temp_mask(self) -> Self: - """Retained API: masks are now applied in NumPy without GRASS mutation.""" - msgr.fatal("Itzi never installs a temporary mapset MASK") - return self - - def del_temp_mask(self) -> Self: - """Retained API: Itzi never installs a temporary mapset MASK.""" - return self - def coor2pixel(self, coor: tuple[float, float]) -> tuple[int, int]: """convert coordinates easting and northing to pixel row and column""" row, col = gutils.coor2pixel(coor, self.region) From 9179243f095d1723331892656360279343ed4d0b Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 09:29:55 -0600 Subject: [PATCH 23/85] use ProcessPoolExecutor instead of manual processes --- src/itzi/itzi.py | 79 +++++++++++------------------------------------- 1 file changed, 18 insertions(+), 61 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index c5c339ea..bfa48778 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -29,12 +29,11 @@ import sys import time from collections.abc import Callable +from concurrent.futures import ProcessPoolExecutor from datetime import timedelta from importlib.metadata import version from multiprocessing import Process, get_context -from multiprocessing.connection import wait from pathlib import Path -from queue import Empty import itzi.messenger as msgr from itzi.cli_parser import build_parser @@ -110,8 +109,8 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: raise SystemExit(1) from None -def resolved_sim_runner_worker(simulation: ResolvedSimulation, result_queue) -> None: - """Run one resolved member and return a small structured worker result.""" +def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str | None]: + """Run one resolved member and return its status.""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) runner: SimulationRunner | None = None @@ -129,26 +128,23 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation, result_queue) -> exclusive_stats=True, ) runner.run().finalize() - result_queue.put(("completed", None)) + return "completed", None except Exception as error: if runner is not None: try: runner.finalize() except Exception: pass - result_queue.put(("execution_failed", f"{type(error).__name__}: {error}")) + return "execution_failed", f"{type(error).__name__}: {error}" -def resolver_worker(expanded: tuple[ExpandedSimulation, ...], result_connection) -> None: +def resolver_worker( + expanded: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: """Resolve one ensemble in a short-lived spawned GRASS process.""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) - try: - result_connection.send(("resolved", resolve_ensemble(expanded))) - except Exception as error: - result_connection.send(("validation_failed", f"{type(error).__name__}: {error}")) - finally: - result_connection.close() + return resolve_ensemble(expanded) def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: @@ -428,66 +424,27 @@ def _run_ensemble_batch(cli_args) -> None: def _resolve_ensemble_in_spawn( expanded: tuple[ExpandedSimulation, ...], ) -> tuple[ResolvedSimulation | ValidationFailure, ...]: - context = get_context("spawn") - receiver, sender = context.Pipe(duplex=False) - process = context.Process(target=resolver_worker, args=(expanded, sender)) - process.start() - sender.close() try: - ready = wait((receiver, process.sentinel)) - if receiver in ready: - status, payload = receiver.recv() - else: - process.join() - status, payload = ( - receiver.recv() - if receiver.poll() - else ( - "validation_failed", - f"resolver exited with status {process.exitcode} without returning a result", - ) - ) - except EOFError: - status = "validation_failed" - payload = f"resolver exited with status {process.exitcode} without returning a result" - finally: - process.join() - receiver.close() - process.close() - if status == "resolved" and isinstance(payload, tuple): - return payload + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(resolver_worker, expanded).result() + except Exception as error: + detail = f"{type(error).__name__}: {error}" return tuple( ValidationFailure( coordinates=simulation.coordinates, phase="input_resolution", - detail=str(payload), + detail=detail, ) for simulation in expanded ) def _run_resolved_in_spawn(simulation: ResolvedSimulation) -> tuple[str, str | None]: - context = get_context("spawn") - result_queue = context.Queue() - process = context.Process(target=resolved_sim_runner_worker, args=(simulation, result_queue)) - process.start() - process.join() try: - status, detail = result_queue.get(timeout=1) - except Empty: - reason = ( - f"signal {-process.exitcode}" - if process.exitcode and process.exitcode < 0 - else str(process.exitcode) - ) - status, detail = ( - "execution_failed", - f"worker exited with {reason} without returning a result", - ) - finally: - result_queue.close() - process.close() - return status, detail + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(resolved_sim_runner_worker, simulation).result() + except Exception as error: + return "execution_failed", f"{type(error).__name__}: {error}" def _select_members( From 274bab2dd7332361e19a30a2843e4c8b3112a758 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 10:02:14 -0600 Subject: [PATCH 24/85] remove Any and object type annotations --- AGENTS.md | 7 +- src/itzi/configreader.py | 33 +++++--- src/itzi/ensemble.py | 132 +++++++++++++++++++----------- src/itzi/ensemble_models.py | 54 ++++++------ src/itzi/ensemble_schema.py | 9 +- tests/cli/test_ensemble_config.py | 13 +++ tests/grass/test_grass_input.py | 54 ++++++++++++ 7 files changed, 211 insertions(+), 91 deletions(-) create mode 100644 tests/grass/test_grass_input.py diff --git a/AGENTS.md b/AGENTS.md index 7a185e4b..1882180a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,13 +2,14 @@ ## Common commands - Run a single test: `uv run pytest tests/my_test.py` -- Due to a bug in GRASS, tests will fails if the mapset is changed between tests. Run each test file independently to prevent this. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. +- Due to a bug in GRASS<8.5, a change of mapset change cannot occur within the same process. Run each test file independently to prevent this type of failure. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. - Enforce code formatting: `uvx ruff format .` ## Code style -- Use python type hints. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. +- Use python type annotation. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. - Do not quote class names in hints. Use `from __future__ import annotations` when necessary. -- Since the arguments types and return types are already documented by the hints, there's no need to duplicate this information in the docstrings. +- Do not use the `Any` type, and avoid the use of the general `object` type. Define the actual type instead. +- Since the arguments types and return types are already documented by the annotations, there's no need to duplicate this information in the docstrings. - Use pydantic BaseModel when validation is needed, and dataclass for internal data exchange. - Place imports at the top of the file. Only break this rule to prevent heavy imports in a rarely used function (for example, CLI options). diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index 5130b415..52dab9a9 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -17,7 +17,7 @@ from configparser import ConfigParser from datetime import datetime, timedelta from pathlib import Path -from typing import Any, NoReturn +from typing import NoReturn from itzi_core import ( ARRAY_DEFINITIONS, @@ -91,6 +91,19 @@ DRAINAGE_STRING_KEYS = ("swmm_inp", "output") DRAINAGE_FLOAT_KEYS = ("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") +type SimulationConfigValue = ( + datetime + | timedelta + | TemporalType + | HotstartRunConfig + | dict[str, str | None] + | SurfaceFlowParameters + | InfiltrationModelType + | float + | str + | Path +) + def _read_parser(filename: str) -> ConfigParser: """Read the INI file or fail fast if it is missing.""" @@ -162,7 +175,7 @@ def _read_hotstart_values(params: ConfigParser) -> HotstartRunConfig | None: return None try: - return HotstartRunConfig(**hotstart_values) + return HotstartRunConfig.model_validate(hotstart_values) except ValidationError as error: _fatal_validation_error(error) @@ -281,10 +294,10 @@ def _read_grass_params(params: ConfigParser) -> GrassParams: return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) -def _build_simulation_config(**kwargs: Any) -> SimulationConfig: +def _build_simulation_config(values: dict[str, SimulationConfigValue]) -> SimulationConfig: """Build a validated simulation config from normalized values.""" try: - return SimulationConfig(**kwargs) + return SimulationConfig.model_validate(values) except ValidationError as error: _fatal_validation_error(error) @@ -403,7 +416,7 @@ def __init__(self, filename: str | None) -> None: surface_flow_parameters = _read_surface_flow_parameters(params) assert self.sim_times.record_step is not None - simulation_kwargs: dict[str, Any] = { + simulation_kwargs: dict[str, SimulationConfigValue] = { "start_time": self.sim_times.start, "end_time": self.sim_times.end, "record_step": self.sim_times.record_step, @@ -419,12 +432,12 @@ def __init__(self, filename: str | None) -> None: simulation_kwargs.update(_read_simulation_option_values(params)) simulation_kwargs.update(_read_simulation_drainage_values(params)) - if "swmm_inp" in simulation_kwargs: - simulation_kwargs["swmm_inp"] = _resolve_swmm_input_path( - simulation_kwargs["swmm_inp"], self.config_file - ) + swmm_inp = simulation_kwargs.get("swmm_inp") + if swmm_inp is not None: + assert isinstance(swmm_inp, str) + simulation_kwargs["swmm_inp"] = _resolve_swmm_input_path(swmm_inp, self.config_file) - self.sim_config = _build_simulation_config(**simulation_kwargs) + self.sim_config = _build_simulation_config(simulation_kwargs) def _check_grass_params(self, grass_params: GrassParams) -> None: """Ensure mandatory GRASS settings are provided together.""" diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble.py index f9483a03..a3f96617 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble.py @@ -17,14 +17,15 @@ import hashlib import itertools import math -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from pathlib import Path -from typing import Any +from typing import cast import yaml from itzi_core import InfiltrationModelType, TemporalType from itzi_core.const import DefaultValues -from pydantic import BaseModel, ValidationError +from pydantic import JsonValue as PydanticJsonValue +from pydantic import TypeAdapter, ValidationError from yaml.constructor import ConstructorError from yaml.resolver import BaseResolver @@ -44,7 +45,6 @@ OutputTemplates, SourceDocument, _canonical_json, - _canonical_value, parse_duration, render_template, ) @@ -56,10 +56,27 @@ InputSweepConfig, NoInfiltration, OptionSweepConfig, + SweepFloat, + SweepString, TimeConfig, YamlEnsembleDocumentV1, ) +type RawYamlScalar = None | bool | int | float | str | bytes | date | datetime +type RawYamlKey = RawYamlScalar | tuple[RawYamlKey, ...] +type RawYamlValue = ( + RawYamlScalar + | list[RawYamlValue] + | tuple[RawYamlValue, ...] + | dict[RawYamlKey, RawYamlValue] + | set[RawYamlKey] +) +type RawYamlMapping = dict[RawYamlKey, RawYamlValue] +type YamlDocument = dict[str, PydanticJsonValue] +type SweepScalar = str | float | InfiltrationAlternative + +_YAML_DOCUMENT_ADAPTER = TypeAdapter(YamlDocument) + class _StrictSafeLoader(yaml.SafeLoader): """SafeLoader variant that rejects duplicate and merge mapping keys.""" @@ -67,8 +84,8 @@ class _StrictSafeLoader(yaml.SafeLoader): def _construct_mapping( loader: _StrictSafeLoader, node: yaml.MappingNode, deep: bool = False -) -> dict: - mapping: dict[Any, Any] = {} +) -> RawYamlMapping: + mapping: RawYamlMapping = {} for key_node, value_node in node.value: if key_node.tag == "tag:yaml.org,2002:merge": raise ConstructorError( @@ -77,7 +94,7 @@ def _construct_mapping( "YAML merge keys are not supported", key_node.start_mark, ) - key = loader.construct_object(key_node, deep=deep) + key = cast(RawYamlKey, loader.construct_object(key_node, deep=deep)) try: duplicate = key in mapping except TypeError as error: @@ -94,7 +111,7 @@ def _construct_mapping( f"duplicate mapping key {key!r}", key_node.start_mark, ) - mapping[key] = loader.construct_object(value_node, deep=deep) + mapping[key] = cast(RawYamlValue, loader.construct_object(value_node, deep=deep)) return mapping @@ -159,8 +176,11 @@ def _parse_datetime(value: str) -> datetime: raise EnsembleError(f"invalid ISO 8601 timestamp {value!r}") from error -def _document_digest(document: Any) -> str: - payload = {"schema": "itzi-yaml-document-v1", "document": _canonical_value(document)} +def _document_digest(document: YamlDocument) -> str: + payload: dict[str, PydanticJsonValue] = { + "schema": "itzi-yaml-document-v1", + "document": document, + } return hashlib.blake2b(_canonical_json(payload).encode(), digest_size=32).hexdigest() @@ -199,15 +219,18 @@ def _segments(text: str) -> tuple[tuple[int, int, str], ...]: return tuple(segments) -def _parse_segment(segment: str) -> dict[str, Any]: +def _parse_segment(segment: str) -> YamlDocument: if any(line.startswith("%") for line in segment.split("\n")): raise EnsembleError("YAML directives are not supported") - value = yaml.load(segment, Loader=_StrictSafeLoader) + value = cast(RawYamlValue, yaml.load(segment, Loader=_StrictSafeLoader)) if value is None: raise EnsembleError("empty YAML documents are not supported") if not isinstance(value, dict): raise EnsembleError("document root must be a mapping") - return value + try: + return _YAML_DOCUMENT_ADAPTER.validate_python(value, strict=True) + except ValidationError as error: + raise EnsembleError("document must contain JSON-compatible values") from error def load_yaml_stream(path: str | Path) -> LoadedYamlStream: @@ -228,6 +251,7 @@ def load_yaml_stream(path: str | Path) -> LoadedYamlStream: for document_index, (start_line, _offset, segment) in enumerate(_segments(text)): try: raw_document = _parse_segment(segment) + document_digest = _document_digest(raw_document) except yaml.YAMLError as error: marker = getattr(error, "problem_mark", None) failures.append( @@ -258,7 +282,7 @@ def load_yaml_stream(path: str | Path) -> LoadedYamlStream: path=source_path, document_index=document_index, file_digest=file_digest, - document_digest=_document_digest(raw_document), + document_digest=document_digest, ) try: document = YamlEnsembleDocumentV1.model_validate(raw_document) @@ -317,16 +341,19 @@ def expand_yaml_document( simulations = [] for selected_values in itertools.product(*(values for _, values in dimensions)): - selected = { + selected: dict[str, SweepScalar] = { path: value for (path, _), value in zip(dimensions, selected_values, strict=True) } coordinates = tuple( (path, _freeze_json(value)) for path, value in sorted(selected.items()) ) input_maps = _selected_input_maps(document.input, selected) - infiltration = _normalize_infiltration( - selected.get("input.infiltration", document.input.infiltration) + infiltration_value = selected.get("input.infiltration", document.input.infiltration) + assert isinstance( + infiltration_value, + (NoInfiltration, ConstantInfiltration, GreenAmptInfiltration), ) + infiltration = _normalize_infiltration(infiltration_value) options = _selected_options(document.options, selected) drainage = _selected_drainage(document.drainage, selected) outputs = _output_templates(document) @@ -358,42 +385,44 @@ def expand_yaml_document( def _collect_dimensions( document: YamlEnsembleDocumentV1, -) -> tuple[tuple[str, tuple[Any, ...]], ...]: - dimensions: list[tuple[str, tuple[Any, ...]]] = [] +) -> tuple[tuple[str, tuple[SweepScalar, ...]], ...]: + dimensions: list[tuple[str, tuple[SweepScalar, ...]]] = [] for key in DIRECT_INPUT_KEYS: - value = getattr(document.input, key) + value = cast(SweepString | None, getattr(document.input, key)) if isinstance(value, list): - dimensions.append((f"input.{key}", tuple(value))) + dimensions.append((f"input.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) if isinstance(document.input.infiltration, list): - dimensions.append(("input.infiltration", tuple(document.input.infiltration))) + dimensions.append( + ( + "input.infiltration", + cast(tuple[SweepScalar, ...], tuple(document.input.infiltration)), + ) + ) for key in type(document.options).model_fields: - value = getattr(document.options, key) + value = cast(SweepFloat | None, getattr(document.options, key)) if isinstance(value, list): - dimensions.append((f"options.{key}", tuple(value))) + dimensions.append((f"options.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) if document.drainage is not None: for key in type(document.drainage).model_fields: - value = getattr(document.drainage, key) + value = cast(SweepString | SweepFloat, getattr(document.drainage, key)) if isinstance(value, list): - dimensions.append((f"drainage.{key}", tuple(value))) + dimensions.append((f"drainage.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) return tuple(sorted(dimensions, key=lambda item: item[0])) def _selected_input_maps( - input_config: InputSweepConfig, selected: dict[str, Any] + input_config: InputSweepConfig, selected: dict[str, SweepScalar] ) -> dict[str, str]: values: dict[str, str] = {} for key in DIRECT_INPUT_KEYS: - value = selected.get(f"input.{key}", getattr(input_config, key)) + value = selected.get(f"input.{key}", cast(SweepString | None, getattr(input_config, key))) if value is not None: assert isinstance(value, str) values[key] = value return values -def _normalize_infiltration( - value: InfiltrationAlternative | list[InfiltrationAlternative], -) -> NormalizedInfiltration: - assert not isinstance(value, list), "infiltration must be scalar after expansion" +def _normalize_infiltration(value: InfiltrationAlternative) -> NormalizedInfiltration: if isinstance(value, NoInfiltration): return NormalizedInfiltration(InfiltrationModelType.NULL, ()) if isinstance(value, ConstantInfiltration): @@ -411,11 +440,13 @@ def _normalize_infiltration( return NormalizedInfiltration(InfiltrationModelType.GREEN_AMPT, tuple(sorted(maps.items()))) -def _selected_options(options: OptionSweepConfig, selected: dict[str, Any]) -> dict[str, float]: +def _selected_options( + options: OptionSweepConfig, selected: dict[str, SweepScalar] +) -> dict[str, float]: values = dict(SURFACE_FLOW_DEFAULTS) values["dtinf"] = DefaultValues.DTINF for key in type(options).model_fields: - value = selected.get(f"options.{key}", getattr(options, key)) + value = selected.get(f"options.{key}", cast(SweepFloat | None, getattr(options, key))) if value is not None: assert isinstance(value, float) values[key] = value @@ -423,14 +454,16 @@ def _selected_options(options: OptionSweepConfig, selected: dict[str, Any]) -> d def _selected_drainage( - drainage: DrainageSweepConfig | None, selected: dict[str, Any] + drainage: DrainageSweepConfig | None, selected: dict[str, SweepScalar] ) -> dict[str, str | float] | None: if drainage is None: return None values: dict[str, str | float] = {} for key in type(drainage).model_fields: - value = selected.get(f"drainage.{key}", getattr(drainage, key)) - assert not isinstance(value, list), "drainage must be scalar after expansion" + value = selected.get( + f"drainage.{key}", cast(SweepString | SweepFloat, getattr(drainage, key)) + ) + assert isinstance(value, (str, float)), "drainage must be scalar after expansion" values[key] = value return values @@ -453,13 +486,20 @@ def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: ) -def _freeze_json(value: Any) -> JsonValue: - if isinstance(value, BaseModel): - return _freeze_json(value.model_dump(mode="python")) - if isinstance(value, dict): - return tuple((str(key), _freeze_json(item)) for key, item in sorted(value.items())) - if isinstance(value, list | tuple): - return tuple(_freeze_json(item) for item in value) - if isinstance(value, (str, int, float, bool)) or value is None: +def _freeze_json(value: SweepScalar) -> JsonValue: + if isinstance(value, (str, float)): return value - raise TypeError(f"unsupported coordinate value {value!r}") + if isinstance(value, NoInfiltration): + return (("type", value.type),) + if isinstance(value, ConstantInfiltration): + return (("rate", value.rate), ("type", value.type)) + assert isinstance(value, GreenAmptInfiltration) + values: list[tuple[str, JsonValue]] = [ + ("capillary_pressure", value.capillary_pressure), + ("effective_porosity", value.effective_porosity), + ("hydraulic_conductivity", value.hydraulic_conductivity), + ("type", value.type), + ] + if value.soil_water_content is not None: + values.insert(3, ("soil_water_content", value.soil_water_content)) + return tuple(values) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 615bfe58..453263c8 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -15,12 +15,11 @@ from __future__ import annotations import json -import math import string from dataclasses import dataclass -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Literal +from typing import Literal from itzi_core import ( ARRAY_DEFINITIONS, @@ -31,7 +30,14 @@ TemporalType, ) from itzi_core.const import DefaultValues -from pydantic import BaseModel, ConfigDict, StrictStr, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + JsonValue as PydanticJsonValue, + StrictStr, + TypeAdapter, + model_validator, +) from itzi.grass_session import GrassParams @@ -71,6 +77,8 @@ None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] ) +_JSON_VALUE_ADAPTER = TypeAdapter(PydanticJsonValue) + class EnsembleError(ValueError): """Configuration error associated with a single YAML document.""" @@ -266,32 +274,20 @@ def format_duration(value: timedelta) -> str: return rendered -def _canonical_json(value: Any) -> str: - return json.dumps( - _canonical_value(value), - allow_nan=False, - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ) - - -def _canonical_value(value: Any) -> Any: +def _canonical_json(value: PydanticJsonValue | BaseModel) -> str: + """Serialize a JSON-compatible value or model in a stable form.""" if isinstance(value, BaseModel): - return _canonical_value(value.model_dump(mode="python")) - if isinstance(value, dict): - return {str(key): _canonical_value(item) for key, item in value.items()} - if isinstance(value, list | tuple): - return [_canonical_value(item) for item in value] - if isinstance(value, datetime): - return {"$datetime": value.isoformat()} - if isinstance(value, date): - return {"$date": value.isoformat()} - if isinstance(value, timedelta): - return {"$duration": format_duration(value)} - if isinstance(value, float) and not math.isfinite(value): - raise EnsembleError("NaN and infinity are not supported") - return value + value = _JSON_VALUE_ADAPTER.validate_json(value.model_dump_json()) + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + except ValueError as error: + raise EnsembleError("NaN and infinity are not supported") from error def render_template(template: str, *, ensemble: str, simulation: str | None) -> str: diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble_schema.py index 8ef051f2..1dcc87fe 100644 --- a/src/itzi/ensemble_schema.py +++ b/src/itzi/ensemble_schema.py @@ -14,10 +14,11 @@ from __future__ import annotations -from typing import Annotated, Any, Literal +from typing import Annotated, Literal, cast from pydantic import ( Field, + JsonValue as PydanticJsonValue, StrictFloat, StrictStr, field_validator, @@ -217,10 +218,12 @@ def validate_drainage_output(self) -> YamlEnsembleDocumentV1: return self -def _validate_sweep(value: Any) -> Any: +def _validate_sweep[T](value: T) -> T: """Validate one scalar-or-list sweep field after Pydantic's type validation.""" values = value if isinstance(value, list) else [value] - semantic_values = [_canonical_json(item) for item in values] + semantic_values = [ + _canonical_json(cast(PydanticJsonValue | InfiltrationAlternative, item)) for item in values + ] if len(semantic_values) != len(set(semantic_values)): raise ValueError("contains duplicate semantic values") return value diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 3f035c36..56a9ff35 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -185,6 +185,19 @@ def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] +def test_loader_rejects_non_json_yaml_and_keeps_later_document(tmp_path): + content = ( + document(ensemble_id="binary") + + "extra: !!binary AQI=\n---\n" + + document(ensemble_id="valid") + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert [failure.phase for failure in stream.failures] == ["parse"] + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] + + def test_infiltration_alternatives_are_explicit(tmp_path): content = document( extra="""\ diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py new file mode 100644 index 00000000..b6a8401f --- /dev/null +++ b/tests/grass/test_grass_input.py @@ -0,0 +1,54 @@ +from datetime import datetime, timedelta +from typing import Literal + +import pytest + +from itzi.providers.grass_input import GrassRasterInputProvider +from itzi.providers.grass_interface import MapData + + +class FakeGrassInterface: + def __init__(self) -> None: + self.temporal_checks: list[str] = [] + + @staticmethod + def format_id(name: str) -> str: + return f"{name}@test" + + @staticmethod + def name_is_stds(map_id: str) -> bool: + return map_id == "series@test" + + @staticmethod + def name_is_map(map_id: str) -> bool: + return map_id == "raster@test" + + def stds_temporal_sanity(self, strds_id: str) -> bool: + self.temporal_checks.append(strds_id) + return True + + @staticmethod + def raster_list_from_strds(strds_id: str) -> list[MapData]: + start = datetime(2020, 1, 1) + return [MapData(strds_id, start, start + timedelta(hours=1))] + + +@pytest.mark.parametrize("input_kinds", [None, {"raster": "raster", "series": "strds"}]) +def test_input_kinds_share_raster_and_strds_construction( + input_kinds: dict[str, Literal["raster", "strds"]] | None, +) -> None: + start = datetime(2020, 1, 1) + end = start + timedelta(hours=1) + interface = FakeGrassInterface() + + provider = GrassRasterInputProvider( + interface, + {"raster": "raster", "series": "series"}, + start, + end, + input_kinds, + ) + + assert provider.map_lists["raster"] == [MapData("raster@test", start, end)] + assert provider.map_lists["series"] == [MapData("series@test", start, end)] + assert interface.temporal_checks == ["series@test"] From 590cb958419e2cc62029437c2ceae173d5cb5bf6 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 10:13:08 -0600 Subject: [PATCH 25/85] ConfigReader: build SimulationConfig directly --- src/itzi/configreader.py | 103 ++++++++++++++------------------------- 1 file changed, 37 insertions(+), 66 deletions(-) diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index 52dab9a9..4adb6a00 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -28,6 +28,7 @@ SurfaceFlowParameters, TemporalType, ) +from itzi_core.const import DefaultValues from pydantic import BaseModel, ConfigDict, ValidationError import itzi.messenger as msgr @@ -87,22 +88,6 @@ arr_def.key for arr_def in ARRAY_DEFINITIONS if ArrayCategory.OUTPUT in arr_def.category ) SURFACE_FLOW_OPTION_KEYS = tuple(SurfaceFlowParameters.model_fields) -SIMULATION_OPTION_KEYS = ("dtinf",) -DRAINAGE_STRING_KEYS = ("swmm_inp", "output") -DRAINAGE_FLOAT_KEYS = ("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") - -type SimulationConfigValue = ( - datetime - | timedelta - | TemporalType - | HotstartRunConfig - | dict[str, str | None] - | SurfaceFlowParameters - | InfiltrationModelType - | float - | str - | Path -) def _read_parser(filename: str) -> ConfigParser: @@ -243,28 +228,6 @@ def _read_surface_flow_parameters(params: ConfigParser) -> SurfaceFlowParameters _fatal_validation_error(error) -def _read_simulation_option_values(params: ConfigParser) -> dict[str, float]: - """Read simulation options that live outside the surface flow model.""" - return _read_float_options(params, "options", SIMULATION_OPTION_KEYS) - - -def _read_simulation_drainage_values(params: ConfigParser) -> dict[str, str | float]: - """Read drainage settings using SimulationConfig field names.""" - drainage_values: dict[str, str | float] = {} - - for option_name in DRAINAGE_STRING_KEYS: - value = params.get("drainage", option_name, fallback=None) - if value is None: - continue - if option_name == "output": - drainage_values["drainage_output"] = value - else: - drainage_values[option_name] = value - - drainage_values.update(_read_float_options(params, "drainage", DRAINAGE_FLOAT_KEYS)) - return drainage_values - - def _resolve_swmm_input_path(swmm_inp: str, config_file: str) -> Path: """Resolve a SWMM input path, searching cwd first and then the config directory.""" configured_path = Path(swmm_inp) @@ -294,14 +257,6 @@ def _read_grass_params(params: ConfigParser) -> GrassParams: return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) -def _build_simulation_config(values: dict[str, SimulationConfigValue]) -> SimulationConfig: - """Build a validated simulation config from normalized values.""" - try: - return SimulationConfig.model_validate(values) - except ValidationError as error: - _fatal_validation_error(error) - - class SimulationTimes(BaseModel): """Parsed and validated simulation time settings.""" @@ -416,28 +371,44 @@ def __init__(self, filename: str | None) -> None: surface_flow_parameters = _read_surface_flow_parameters(params) assert self.sim_times.record_step is not None - simulation_kwargs: dict[str, SimulationConfigValue] = { - "start_time": self.sim_times.start, - "end_time": self.sim_times.end, - "record_step": self.sim_times.record_step, - "temporal_type": self.sim_times.temporal_type, - "input_map_names": self.input_map_names, - "output_map_names": self.output_map_names, - "surface_flow_parameters": surface_flow_parameters, - "infiltration_model": infiltration_model, - } - - if self.hotstart_config is not None: - simulation_kwargs["hotstart_config"] = self.hotstart_config - - simulation_kwargs.update(_read_simulation_option_values(params)) - simulation_kwargs.update(_read_simulation_drainage_values(params)) - swmm_inp = simulation_kwargs.get("swmm_inp") + swmm_inp = params.get("drainage", "swmm_inp", fallback=None) if swmm_inp is not None: - assert isinstance(swmm_inp, str) - simulation_kwargs["swmm_inp"] = _resolve_swmm_input_path(swmm_inp, self.config_file) + resolved_swmm_inp = _resolve_swmm_input_path(swmm_inp, self.config_file) + else: + resolved_swmm_inp = None - self.sim_config = _build_simulation_config(simulation_kwargs) + try: + self.sim_config = SimulationConfig( + start_time=self.sim_times.start, + end_time=self.sim_times.end, + record_step=self.sim_times.record_step, + temporal_type=self.sim_times.temporal_type, + hotstart_config=self.hotstart_config, + input_map_names={ + key: value for key, value in self.input_map_names.items() if value is not None + }, + output_map_names={ + key: value for key, value in self.output_map_names.items() if value is not None + }, + surface_flow_parameters=surface_flow_parameters, + dtinf=params.getfloat("options", "dtinf", fallback=DefaultValues.DTINF), + infiltration_model=infiltration_model, + swmm_inp=resolved_swmm_inp, + drainage_output=params.get("drainage", "output", fallback=None), + orifice_coeff=params.getfloat( + "drainage", "orifice_coeff", fallback=DefaultValues.ORIFICE_COEFF + ), + free_weir_coeff=params.getfloat( + "drainage", "free_weir_coeff", fallback=DefaultValues.FREE_WEIR_COEFF + ), + submerged_weir_coeff=params.getfloat( + "drainage", + "submerged_weir_coeff", + fallback=DefaultValues.SUBMERGED_WEIR_COEFF, + ), + ) + except ValidationError as error: + _fatal_validation_error(error) def _check_grass_params(self, grass_params: GrassParams) -> None: """Ensure mandatory GRASS settings are provided together.""" From 105abfea85331cafbb404090cf573907f244f4f5 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 10:52:27 -0600 Subject: [PATCH 26/85] more explicit dry-run cli argument --- docs/cli.rst | 4 ++-- docs/conf_file.rst | 2 +- src/itzi/cli_parser.py | 8 +++++--- tests/cli/test_cli.py | 5 +++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3538be5b..26c8e8a3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -14,13 +14,13 @@ Run a simulation YAML ensembles ~~~~~~~~~~~~~~ -Each YAML document is one ensemble. Use ``--dry`` to perform parsing, +Each YAML document is one ensemble. Use ``--dry-run`` (or ``-d``) to perform parsing, expansion, GRASS input resolution, member-ID calculation, and artifact validation without writing manifests or model outputs: .. code-block:: bash - itzi run --dry studies.yaml + itzi run --dry-run studies.yaml Select resolved members with a qualified ensemble and simulation ID. The unqualified ID form is accepted only when the batch contains one ensemble. diff --git a/docs/conf_file.rst b/docs/conf_file.rst index c43119b4..b4852769 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -82,7 +82,7 @@ A list of complete alternatives is one sweep dimension. ``{ensemble}`` and ``{simulation}`` are the only output-template substitutions. Output maps are always bare names in the active execution mapset; qualified output names are rejected. YAML filesystem paths are relative to the YAML -source file. Use ``itzi run --dry study.yaml`` to resolve and validate members +source file. Use ``itzi run --dry-run study.yaml`` to resolve and validate members without creating output artifacts. Legacy INI reference diff --git a/src/itzi/cli_parser.py b/src/itzi/cli_parser.py index 91ee3625..e384d093 100644 --- a/src/itzi/cli_parser.py +++ b/src/itzi/cli_parser.py @@ -28,7 +28,7 @@ def build_parser() -> argparse.ArgumentParser: run_parser.add_argument( "config_file", nargs="+", - help=("An Itzï configuration file or YAML stream (several files form one batch)."), + help="An Itzï configuration file. ini and yaml are allowed, several files form one batch.", ) run_parser.add_argument("-o", action="store_true", help="Overwrite files if exist.") verbosity_parser = run_parser.add_mutually_exclusive_group() @@ -51,10 +51,12 @@ def build_parser() -> argparse.ArgumentParser: action="append", default=[], metavar="[ENSEMBLE#]SIMULATION", - help="Run only a resolved ensemble member. Repeat to select multiple members.", + help="Run only one ensemble member. Repeat to select multiple members.", ) run_parser.add_argument( - "--dry", + "-d", + "--dry-run", + dest="dry", action="store_true", help="Resolve and validate selected members without writing user artifacts.", ) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 3fdae395..ac93d4a4 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -42,9 +42,10 @@ def test_run_parser_accepts_resume_from_args(): assert args.resume_from == [("a.ini", "restart_a.zip"), ("b.ini", "restart_b.zip")] -def test_run_parser_accepts_yaml_dry_run_and_member_selection(): +@pytest.mark.parametrize("dry_option", ["--dry-run", "-d"]) +def test_run_parser_accepts_yaml_dry_run_and_member_selection(dry_option): args = build_parser().parse_args( - ["run", "study.yaml", "--dry", "--member", "study#sim-a", "--member", "study#sim-b"] + ["run", "study.yaml", dry_option, "--member", "study#sim-a", "--member", "study#sim-b"] ) assert args.dry is True From 0becaec8ce1f3a4f4893feea0136d33ae8d7f043 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 11:19:05 -0600 Subject: [PATCH 27/85] add copyright notice to csv writer --- src/itzi/providers/csv_output.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/itzi/providers/csv_output.py b/src/itzi/providers/csv_output.py index b2b68de5..5300cd63 100644 --- a/src/itzi/providers/csv_output.py +++ b/src/itzi/providers/csv_output.py @@ -1,4 +1,16 @@ -"""Application-owned CSV output with explicit first-creation ownership.""" +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" from __future__ import annotations From f8761b4b6a5bf1427b55ce5da46b25bc6dffbad4 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Sun, 27 Sep 2026 11:19:27 -0600 Subject: [PATCH 28/85] AGENTS: clearer mapset change description --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1882180a..a2209c13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,8 +2,8 @@ ## Common commands - Run a single test: `uv run pytest tests/my_test.py` -- Due to a bug in GRASS<8.5, a change of mapset change cannot occur within the same process. Run each test file independently to prevent this type of failure. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. -- Enforce code formatting: `uvx ruff format .` +- Due to a bug in GRASS<8.5, a change of mapset cannot occur within the same process when using the temporal framework. Each test file must therefore be run independently. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. +- Enforce code formatting: `uv run ruff format .` ## Code style - Use python type annotation. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. From 26cd4ad0eec1d34f5b9e442557204f35ddaab877 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 13:19:57 -0600 Subject: [PATCH 29/85] improve bmi class definitions --- src/itzi/bmi_itzi.py | 46 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 5afe2de8..61982ced 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -15,14 +15,15 @@ """ from datetime import timedelta +from types import MappingProxyType import numpy as np from bmipy import Bmi +from itzi_core import ARRAY_DEFINITIONS, ArrayCategory -from itzi import SimulationRunner from itzi.configreader import ConfigReader from itzi.grass_session import GrassSessionManager -from itzi_core.array_definitions import ARRAY_DEFINITIONS, ArrayCategory +from itzi.simulation_runner import SimulationRunner class BmiItzi(Bmi): @@ -30,29 +31,38 @@ class BmiItzi(Bmi): _name = "Itzï" # Mappings of CSDMS name: internal key for input and output arrays - _input_var_names = { - arr_def.csdms_name: arr_def.key - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in arr_def.category - } - _output_var_names = { - arr_def.csdms_name: arr_def.key - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.OUTPUT in arr_def.category - } - + _input_var_names = MappingProxyType( + { + arr_def.csdms_name: arr_def.key + for arr_def in ARRAY_DEFINITIONS + if ArrayCategory.INPUT in arr_def.category + } + ) + _output_var_names = MappingProxyType( + { + arr_def.csdms_name: arr_def.key + for arr_def in ARRAY_DEFINITIONS + if ArrayCategory.OUTPUT in arr_def.category + } + ) # A list of array definition for both input and output, without overlaps - input_output_array_definition = [ + input_output_array_definition = ( arr_def for arr_def in ARRAY_DEFINITIONS if any(cat in arr_def.category for cat in [ArrayCategory.OUTPUT, ArrayCategory.INPUT]) - ] + ) # Mapping of csdms_name: internal key for both input and output - _var_names = {arr_def.csdms_name: arr_def.key for arr_def in input_output_array_definition} + _var_names = MappingProxyType( + {arr_def.csdms_name: arr_def.key for arr_def in input_output_array_definition} + ) # Mapping of csdms_name: unit - _var_units = {arr_def.csdms_name: arr_def.unit for arr_def in input_output_array_definition} + _var_units = MappingProxyType( + {arr_def.csdms_name: arr_def.unit for arr_def in input_output_array_definition} + ) # Mapping of csdms_name: var_loc - _var_loc = {arr_def.csdms_name: arr_def.var_loc for arr_def in input_output_array_definition} + _var_loc = MappingProxyType( + {arr_def.csdms_name: arr_def.var_loc for arr_def in input_output_array_definition} + ) def __init__(self): """Create a BmiItzi model that is ready for initialization.""" From 9f194d7aece22acf582fbe37829b21d148cc6b23 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 13:37:31 -0600 Subject: [PATCH 30/85] use itzi_core imports in configreader --- src/itzi/configreader.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index 4adb6a00..2354ae14 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -20,8 +20,8 @@ from typing import NoReturn from itzi_core import ( - ARRAY_DEFINITIONS, - ArrayCategory, + INPUT_ARRAY_KEYS, + OUTPUT_ARRAY_KEYS, HotstartRunConfig, InfiltrationModelType, SimulationConfig, @@ -81,12 +81,6 @@ ) GRASS_MANDATORY_KEYS = ("grassdata", "location", "mapset") GRASS_OPTION_KEYS = (*GRASS_MANDATORY_KEYS, "region", "mask", "grass_bin") -INPUT_MAP_KEYS = tuple( - arr_def.key for arr_def in ARRAY_DEFINITIONS if ArrayCategory.INPUT in arr_def.category -) -OUTPUT_MAP_KEYS = tuple( - arr_def.key for arr_def in ARRAY_DEFINITIONS if ArrayCategory.OUTPUT in arr_def.category -) SURFACE_FLOW_OPTION_KEYS = tuple(SurfaceFlowParameters.model_fields) @@ -167,14 +161,14 @@ def _read_hotstart_values(params: ConfigParser) -> HotstartRunConfig | None: def _read_input_map_names(params: ConfigParser) -> dict[str, str | None]: """Read input map names and normalize deprecated aliases.""" - map_names: dict[str, str | None] = dict.fromkeys(INPUT_MAP_KEYS) + map_names: dict[str, str | None] = dict.fromkeys(INPUT_ARRAY_KEYS) for old_input_name, new_input_name in DEPRECATED_INPUT_ALIASES: if params.has_option("input", old_input_name): _warn_about_deprecated_alias("Input", old_input_name, new_input_name) map_names[new_input_name] = params.get("input", old_input_name) - for input_name in INPUT_MAP_KEYS: + for input_name in INPUT_ARRAY_KEYS: if params.has_option("input", input_name): map_names[input_name] = params.get("input", input_name) @@ -202,7 +196,7 @@ def _normalize_output_values(raw_values: str | None) -> list[str]: def _generate_output_map_names(prefix: str, output_values: list[str]) -> dict[str, str | None]: """Build the output map dictionary from the selected outputs.""" - output_map_names: dict[str, str | None] = dict.fromkeys(OUTPUT_MAP_KEYS) + output_map_names: dict[str, str | None] = dict.fromkeys(OUTPUT_ARRAY_KEYS) for value in output_values: if value in output_map_names: output_map_names[value] = f"{prefix}_{value}" From af360a19484f167e25b37ea850be2bc4a5a2bb62 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 13:38:04 -0600 Subject: [PATCH 31/85] reorder imports in __init__ --- src/itzi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/itzi/__init__.py b/src/itzi/__init__.py index 34598c83..5b349d75 100644 --- a/src/itzi/__init__.py +++ b/src/itzi/__init__.py @@ -1,2 +1,2 @@ -from itzi.itzi import SimulationRunner as SimulationRunner from itzi.bmi_itzi import BmiItzi as BmiItzi +from itzi.itzi import SimulationRunner as SimulationRunner From f5c62b82cb4cbef8c2619968ebf019862f687fcf Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 13:39:26 -0600 Subject: [PATCH 32/85] use itzi_core imports in enememble_schema and _models --- src/itzi/ensemble_models.py | 15 ++------------- src/itzi/ensemble_schema.py | 6 +++--- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 453263c8..9f0a6d18 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -22,8 +22,7 @@ from typing import Literal from itzi_core import ( - ARRAY_DEFINITIONS, - ArrayCategory, + INPUT_ARRAY_KEYS, DomainData, InfiltrationModelType, SimulationConfig, @@ -41,16 +40,6 @@ from itzi.grass_session import GrassParams -INPUT_KEYS = tuple( - definition.key - for definition in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in definition.category -) -OUTPUT_KEYS = tuple( - definition.key - for definition in ARRAY_DEFINITIONS - if ArrayCategory.OUTPUT in definition.category -) GREEN_AMPT_KEYS = ( "effective_porosity", "capillary_pressure", @@ -58,7 +47,7 @@ "soil_water_content", ) DIRECT_INPUT_KEYS = tuple( - key for key in INPUT_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} + key for key in INPUT_ARRAY_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} ) SURFACE_FLOW_DEFAULTS = { "hmin": DefaultValues.HFMIN, diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble_schema.py index 1dcc87fe..2b53c692 100644 --- a/src/itzi/ensemble_schema.py +++ b/src/itzi/ensemble_schema.py @@ -16,17 +16,17 @@ from typing import Annotated, Literal, cast +from itzi_core import OUTPUT_ARRAY_KEYS from pydantic import ( Field, - JsonValue as PydanticJsonValue, StrictFloat, StrictStr, field_validator, model_validator, ) +from pydantic import JsonValue as PydanticJsonValue from itzi.ensemble_models import ( - OUTPUT_KEYS, DomainConfig, StrictModel, _canonical_json, @@ -174,7 +174,7 @@ class RasterOutputs(StrictModel): def validate_variables(cls, value: list[str]) -> list[str]: if not value: raise ValueError("must not be empty") - unknown = sorted(set(value) - set(OUTPUT_KEYS)) + unknown = sorted(set(value) - set(OUTPUT_ARRAY_KEYS)) if unknown: raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") if len(value) != len(set(value)): From 4ea231b437ac76cf7766774824cdf8bfe05f1a82 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 14:58:59 -0600 Subject: [PATCH 33/85] add preflight checks --- src/itzi/itzi.py | 155 +++++++++++++++++---- src/itzi/manifest.py | 20 +++ src/itzi/preflight.py | 161 +++++++++++++++++++++ src/itzi/providers/grass_input.py | 102 +++++++------- src/itzi/providers/grass_interface.py | 38 +++-- src/itzi/resolution.py | 78 +++++++---- tests/cli/test_cli.py | 39 +++++- tests/cli/test_ensemble_config.py | 57 +++++++- tests/grass/test_ensemble.py | 193 +++++++++++++++++++++++++- tests/grass/test_grass_input.py | 16 ++- 10 files changed, 725 insertions(+), 134 deletions(-) create mode 100644 src/itzi/preflight.py diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index bfa48778..9192b276 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -147,6 +147,20 @@ def resolver_worker( return resolve_ensemble(expanded) +def preflight_worker(simulation: ResolvedSimulation) -> str | None: + """Validate one resolved member in a read-only spawned GRASS process.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + try: + with GrassSessionManager(simulation.grass_params): + from itzi.preflight import preflight_simulation + + preflight_simulation(simulation) + except Exception as error: + return f"{type(error).__name__}: {error}" + return None + + def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: """Run a simulation in a subprocess""" worker_args = (conf_file, hotstart_file) @@ -321,6 +335,7 @@ def _run_ensemble_batch(cli_args) -> None: resolved: list[ tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] ] = [] + artifact_failures: dict[str, str] = {} for ensemble in ensembles: successful: list[ResolvedSimulation] = [] member_failures: list[ValidationFailure] = [] @@ -330,39 +345,42 @@ def _run_ensemble_batch(cli_args) -> None: else: member_failures.append(result) try: - validate_resolved_ensemble(tuple(successful)) + validate_resolved_ensemble(tuple(successful), (ensemble.source.path,)) except EnsembleError as error: - member_failures.extend( - ValidationFailure( - coordinates=simulation.coordinates, - phase="artifact_validation", - detail=str(error), - ) - for simulation in successful - ) - successful = [] + artifact_failures[ensemble.ensemble_id] = str(error) resolved.append((ensemble, tuple(successful), tuple(member_failures))) - selected = _select_members(resolved, getattr(cli_args, "member", [])) + selectors = getattr(cli_args, "member", []) + has_selectors = bool(selectors) + selected = _select_members(resolved, selectors) + preflight_failures, manifest_failures = _preflight_selected( + resolved, + selected, + artifact_failures, + has_selectors=has_selectors, + overwrite=bool(getattr(cli_args, "o", False)), + ) if getattr(cli_args, "dry", False): - _display_dry_plan(resolved, selected) - if document_failures or any(failures for _, _, failures in resolved): + _display_dry_plan( + resolved, selected, preflight_failures, manifest_failures, has_selectors=has_selectors + ) + unresolved_selected = not has_selectors and any(failures for _, _, failures in resolved) + if document_failures or unresolved_selected or preflight_failures or manifest_failures: msgr.fatal("YAML batch validation failed") return failure_count = len(document_failures) for ensemble, simulations, failures in resolved: selected_ids = selected.get(ensemble.ensemble_id, set()) - if not selected_ids and not (not getattr(cli_args, "member", []) and failures): + if not selected_ids and not (not has_selectors and failures): continue manifest_path = _manifest_path(ensemble) - states = _initial_member_states( - simulations, failures, selected_ids, bool(getattr(cli_args, "member", [])) - ) - try: - _validate_manifest_destination(manifest_path, ensemble, simulations) - except EnsembleError as error: - msgr.warning(str(error)) + states = _initial_member_states(simulations, failures, selected_ids, has_selectors) + for simulation_id, failure in preflight_failures.get(ensemble.ensemble_id, {}).items(): + states[simulation_id]["status"] = "validation_failed" + states[simulation_id]["failure"] = failure + if ensemble.ensemble_id in manifest_failures: + msgr.warning(manifest_failures[ensemble.ensemble_id]) failure_count += max(1, len(selected_ids)) continue try: @@ -377,15 +395,12 @@ def _run_ensemble_batch(cli_args) -> None: continue if states[simulation.simulation_id]["status"] != "planned": continue - if ( - simulation.artifacts.statistics_file is not None - and simulation.artifacts.statistics_file.exists() - and not cli_args.o - ): + detail = _preflight_resolved_in_spawn(simulation) + if detail is not None: states[simulation.simulation_id]["status"] = "validation_failed" states[simulation.simulation_id]["failure"] = { - "phase": "artifact_validation", - "detail": f"statistics file <{simulation.artifacts.statistics_file}> already exists", + "phase": "preflight", + "detail": detail, } _update_manifest(manifest_path, ensemble, states) continue @@ -402,7 +417,10 @@ def _run_ensemble_batch(cli_args) -> None: state["status"] = status state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") if detail is not None: - states[simulation.simulation_id]["failure"] = detail + states[simulation.simulation_id]["failure"] = { + "phase": "execution", + "detail": detail, + } failure_count += 1 try: _update_manifest(manifest_path, ensemble, states) @@ -447,6 +465,65 @@ def _run_resolved_in_spawn(simulation: ResolvedSimulation) -> tuple[str, str | N return "execution_failed", f"{type(error).__name__}: {error}" +def _preflight_resolved_in_spawn(simulation: ResolvedSimulation) -> str | None: + try: + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(preflight_worker, simulation).result() + except Exception as error: + return f"{type(error).__name__}: {error}" + + +def _preflight_selected( + resolved: list[ + tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] + ], + selected: dict[str, set[str]], + artifact_failures: dict[str, str], + *, + has_selectors: bool, + overwrite: bool, +) -> tuple[dict[str, dict[str, dict[str, str]]], dict[str, str]]: + """Collect selected-member and manifest failures before creating artifacts.""" + member_failures: dict[str, dict[str, dict[str, str]]] = {} + manifest_failures: dict[str, str] = {} + for ensemble, simulations, _ in resolved: + selected_ids = selected[ensemble.ensemble_id] + if not selected_ids and has_selectors: + continue + failures = member_failures.setdefault(ensemble.ensemble_id, {}) + artifact_detail = artifact_failures.get(ensemble.ensemble_id) + if artifact_detail is not None: + for simulation_id in selected_ids: + failures[simulation_id] = { + "phase": "artifact_validation", + "detail": artifact_detail, + } + try: + _validate_manifest_destination( + _manifest_path(ensemble), + ensemble, + simulations, + overwrite=overwrite, + ) + except EnsembleError as error: + manifest_failures[ensemble.ensemble_id] = str(error) + for simulation in simulations: + if ( + simulation.simulation_id not in selected_ids + or simulation.simulation_id in failures + ): + continue + detail = _preflight_resolved_in_spawn(simulation) + if detail is not None: + failures[simulation.simulation_id] = { + "phase": "preflight", + "detail": detail, + } + if not failures: + member_failures.pop(ensemble.ensemble_id) + return member_failures, manifest_failures + + def _select_members( resolved: list[ tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] @@ -483,6 +560,10 @@ def _display_dry_plan( tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] ], selected: dict[str, set[str]], + preflight_failures: dict[str, dict[str, dict[str, str]]], + manifest_failures: dict[str, str], + *, + has_selectors: bool, ) -> None: for ensemble, simulations, failures in resolved: msgr.message(f"Ensemble {ensemble.ensemble_id}:") @@ -493,8 +574,22 @@ def _display_dry_plan( else "not selected" ) msgr.message(f" {simulation.simulation_id} ({selection})") + if simulation.simulation_id in preflight_failures.get(ensemble.ensemble_id, {}): + failure = preflight_failures[ensemble.ensemble_id][simulation.simulation_id] + msgr.warning(f" {failure['phase']} failed: {failure['detail']}") + for variable, name in simulation.artifacts.output_map_names: + msgr.message(f" raster {variable}: {name}") + if simulation.artifacts.drainage_output is not None: + msgr.message(f" drainage: {simulation.artifacts.drainage_output}") + if simulation.artifacts.statistics_file is not None: + msgr.message(f" statistics: {simulation.artifacts.statistics_file}") for failure in failures: - msgr.warning(f" validation failed: {failure.detail}") + status = "not selected" if has_selectors else "validation failed" + msgr.warning(f" {status}: {failure.detail}") + if ensemble.ensemble_id in manifest_failures: + msgr.warning( + f" manifest validation failed: {manifest_failures[ensemble.ensemble_id]}" + ) def itzi_version(cli_args): diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py index 14902cfd..bf76d153 100644 --- a/src/itzi/manifest.py +++ b/src/itzi/manifest.py @@ -42,10 +42,29 @@ def _manifest_path(ensemble: ExpandedEnsemble) -> Path: return ensemble.source.path.parent / "results" / f"{ensemble.ensemble_id}.manifest.yaml" +def validate_file_destination(path: Path, *, overwrite: bool, description: str) -> None: + """Check a filesystem output destination without creating it or its parents.""" + if path.exists() or path.is_symlink(): + if not path.is_file(): + raise EnsembleError(f"{description} <{path}> is not a regular file") + if not overwrite: + raise EnsembleError(f"{description} <{path}> already exists") + if not os.access(path, os.W_OK): + raise EnsembleError(f"{description} <{path}> is not writable") + + parent = path.parent + while not parent.exists(): + parent = parent.parent + if not parent.is_dir() or not os.access(parent, os.W_OK | os.X_OK): + raise EnsembleError(f"parent directory <{parent}> for {description} is not writable") + + def _validate_manifest_destination( manifest_path: Path, ensemble: ExpandedEnsemble, simulations: tuple[ResolvedSimulation, ...], + *, + overwrite: bool, ) -> None: """Keep the parent-owned manifest away from files required by the run.""" protected = {ensemble.source.path.resolve()} @@ -63,6 +82,7 @@ def _validate_manifest_destination( raise EnsembleError( f"manifest <{manifest_path}> aliases a protected input or member artifact" ) + validate_file_destination(manifest_path, overwrite=overwrite, description="manifest") def _initial_member_states( diff --git a/src/itzi/preflight.py b/src/itzi/preflight.py new file mode 100644 index 00000000..15e8c789 --- /dev/null +++ b/src/itzi/preflight.py @@ -0,0 +1,161 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import closing +from pathlib import Path +from typing import TYPE_CHECKING + +import numpy as np + +from itzi.ensemble_models import EnsembleError +from itzi.manifest import validate_file_destination +from itzi.providers.grass_input import resolve_input_map_lists +from itzi.providers.grass_interface import GrassInterface +from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name +from itzi.resolution import _last_record_index, verify_resolved_simulation + +if TYPE_CHECKING: + from itzi.ensemble_models import ResolvedSimulation + + +def preflight_simulation(simulation: ResolvedSimulation) -> None: + """Validate one resolved simulation without creating user artifacts.""" + import grass.script as gscript + + verify_resolved_simulation(simulation) + grass_version = gscript.parse_command("g.version", flags="g")["version"] + major, minor = (int(part) for part in grass_version.split(".")[:2]) + if (major, minor) < (8, 4): + raise EnsembleError( + f"Itzi requires at least GRASS 8.4.0, version {grass_version} detected" + ) + config = simulation.simulation_config + interface = GrassInterface( + start_time=config.start_time, + end_time=config.end_time, + dtype=np.float32, + region_id=simulation.grass_params.region, + raster_mask_id=simulation.grass_params.mask, + effective_mask=(simulation.effective_mask.mode, simulation.effective_mask.source), + non_blocking_write=False, + ) + try: + _validate_inputs(simulation, interface) + _validate_grass_outputs(simulation, interface) + if simulation.artifacts.statistics_file is not None: + validate_file_destination( + simulation.artifacts.statistics_file, + overwrite=interface.overwrite, + description="statistics file", + ) + finally: + interface.cleanup() + + +def _validate_inputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + config = simulation.simulation_config + mask = interface.get_npmask() + if np.all(mask): + raise EnsembleError("effective mask excludes the complete simulation domain") + + map_lists = resolve_input_map_lists( + interface, + config.input_map_names, + config.start_time, + config.end_time, + dict(simulation.input_kinds), + ) + for key in ("ground_elevation", "friction"): + for map_data in map_lists[key] or (): + array = interface.read_raster_map(map_data.id) + active = array[~mask] + if not np.any(np.isfinite(active)): + raise EnsembleError( + f"input map <{key}> contains only NULL/NaN cells inside the active domain" + ) + + +def _validate_grass_outputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + import grass.temporal as tgis + from grass.pygrass import utils as gutils + + config = simulation.simulation_config + last_index = _last_record_index( + config.end_time - config.start_time, + config.record_step, + ) + + mapset = interface.get_current_mapset() + + def map_exists(name: str, element: str) -> bool: + lookup = gutils.get_mapset_raster if element == "cell" else gutils.get_mapset_vector + return bool(lookup(name, mapset)) + + def stds_exists(name: str, stds_type: str) -> bool: + dataset = tgis.dataset_factory(stds_type, interface.format_id(name)) + return dataset is not None and bool(dataset.is_in_db()) + + def reject_existing(name: str, exists: bool, kind: str) -> None: + if exists and not interface.overwrite: + raise EnsembleError(f"{kind} <{name}> already exists") + + for name in config.output_map_names.values(): + if not interface.overwrite: + reject_existing( + name, + map_exists(name, "cell") or stds_exists(name, "strds"), + "raster output", + ) + interface.validate_output_stds_temporal_type(name, "strds", config.temporal_type) + if not interface.overwrite: + for index in range(last_index + 1): + child = derived_record_name(name, index) + reject_existing(child, map_exists(child, "cell"), "raster output") + + if config.drainage_output is None: + return + name = config.drainage_output + if not interface.overwrite: + reject_existing( + name, + map_exists(name, "vector") or stds_exists(name, "stvds"), + "drainage output", + ) + interface.validate_output_stds_temporal_type(name, "stvds", config.temporal_type) + if interface.overwrite: + return + + # Drainage DB links use PyGRASS Link's default mapset-local SQLite database. + database = ( + Path(gutils.getenv("GISDBASE")) + / gutils.getenv("LOCATION_NAME") + / mapset + / "sqlite" + / "sqlite.db" + ) + existing_tables: set[str] = set() + if database.is_file(): + with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as connection: + existing_tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + for index in range(last_index + 1): + child = derived_record_name(name, index) + reject_existing(child, map_exists(child, "vector"), "drainage output") + for table_name in derived_drainage_table_names(child): + reject_existing(table_name, table_name in existing_tables, "drainage table") diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 30874f7c..de4b0ef8 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Literal import numpy as np -from itzi_core import ARRAY_DEFINITIONS, ArrayCategory, DomainData +from itzi_core import INPUT_ARRAY_KEYS, DomainData from itzi_core.providers import RasterInputProvider import itzi.messenger as msgr @@ -42,8 +42,13 @@ def __init__( self.grass_interface = grass_interface self.start_time = default_start_time self.end_time = default_end_time - self.input_kinds = input_kinds - self.map_lists = self._get_map_lists(input_map_names) + self.map_lists = resolve_input_map_lists( + grass_interface, + input_map_names, + default_start_time, + default_end_time, + input_kinds, + ) def get_domain_data(self) -> DomainData: """Return a DomainData object""" @@ -57,58 +62,6 @@ def get_domain_data(self) -> DomainData: crs_wkt=self.grass_interface.get_crs_wkt(), ) - def _get_map_lists( - self, map_names: Mapping[str, str | None] - ) -> Mapping[str, list[MapData] | None]: - """Read maps names from GIS. - input map_names is a dictionary of maps/STDS names - for each entry in map_names: - if the name is empty or None, store None - if a strds, load all maps in the instance's time extend, - store them as a list - if a single map, set the start and end time to fit simulation. - store it in a list for consistency - each map is stored as a MapData namedtuple - store result in instance's dictionary - """ - map_lists: dict[str, list[MapData] | None] = { - arr_def.key: None - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in arr_def.category - } - for k, map_name in map_names.items(): - if not map_name: - map_list = None - continue - map_id = self.grass_interface.format_id(map_name) - kind = self.input_kinds.get(k) if self.input_kinds is not None else None - if kind == "strds": - if not self.grass_interface.name_is_stds(map_id): - msgr.fatal(f"STRDS input <{map_id}> is no longer available") - elif kind == "raster": - if not self.grass_interface.name_is_map(map_id): - msgr.fatal(f"raster input <{map_id}> is no longer available") - elif self.grass_interface.name_is_stds(map_id): - kind = "strds" - elif self.grass_interface.name_is_map(map_id): - kind = "raster" - else: - msgr.fatal(f"{map_name} not found!") - if kind == "strds": - if not self.grass_interface.stds_temporal_sanity(map_id): - msgr.fatal(f"{map_name}: inadequate temporal format") - map_list = self.grass_interface.raster_list_from_strds(map_id) - else: - map_list = [ - MapData( - id=map_id, - start_time=self.start_time, - end_time=self.end_time, - ) - ] - map_lists[k] = map_list - return map_lists - def get_array( self, map_key: str, current_time: datetime ) -> tuple[np.ndarray | None, datetime, datetime]: @@ -132,3 +85,42 @@ def get_array( # No gap is expected here: GRASS temporal sanity is checked at init. # An in-range lookup should always hit one map. raise ValueError(f"No map found for {map_key} at time {current_time}") + + +def resolve_input_map_lists( + grass_interface: GrassInterface, + map_names: Mapping[str, str | None], + start_time: datetime, + end_time: datetime, + input_kinds: Mapping[str, Literal["raster", "strds"]] | None, +) -> Mapping[str, list[MapData] | None]: + """Resolve provider-ready raster lists without creating an output provider.""" + + invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) + if invalid_input_keys: + raise ValueError(f"Invalid input keys found: {', '.join(invalid_input_keys)}") + + map_lists: dict[str, list[MapData] | None] = {arr_key: None for arr_key in INPUT_ARRAY_KEYS} + for key, map_name in map_names.items(): + if not map_name: + continue + map_id = grass_interface.format_id(map_name) + kind = input_kinds.get(key) if input_kinds is not None else None + if kind == "strds": + if not grass_interface.name_is_stds(map_id, initialize=False): + msgr.fatal(f"STRDS input <{map_id}> is no longer available") + elif kind == "raster": + if not grass_interface.name_is_map(map_id): + msgr.fatal(f"raster input <{map_id}> is no longer available") + elif grass_interface.name_is_stds(map_id, initialize=False): + kind = "strds" + elif grass_interface.name_is_map(map_id): + kind = "raster" + else: + msgr.fatal(f"{map_name} not found!") + if kind == "strds": + map_list = grass_interface.raster_list_from_strds(map_id) + else: + map_list = [MapData(id=map_id, start_time=start_time, end_time=end_time)] + map_lists[key] = map_list + return map_lists diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index 2543c597..d8a38441 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -70,19 +70,23 @@ def _init_temporal() -> None: assert Path(f).is_file() -def file_exists(name): +def file_exists(name: str, *, initialize: bool = True) -> bool: """Return True if name is an existing map or stds, False otherwise""" if not name: return False - else: - _id = GrassInterface.format_id(name) - return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds(_id) + _id = GrassInterface.format_id(name) + return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds( + _id, initialize=initialize + ) -def check_output_files(file_list) -> None: +def check_output_files(file_list: list[str]) -> None: """Check if the output files exist""" + if gscript.overwrite() or not file_list: + return + _init_temporal() for map_name in file_list: - if file_exists(map_name) and not gscript.overwrite(): + if file_exists(map_name, initialize=False): msgr.fatal(f"File {map_name} exists and will not be overwritten") @@ -348,12 +352,13 @@ def get_current_mapset() -> str: return gutils.getenv("MAPSET") @staticmethod - def name_is_stds(name: str) -> bool: + def name_is_stds(name: str, *, initialize: bool = True) -> bool: """return True if the name given as input is a registered strds False if not """ # make sure temporal module is initialized - _init_temporal() + if initialize: + _init_temporal() return bool(tgis.SpaceTimeRasterDataset(name).is_in_db()) @staticmethod @@ -365,7 +370,8 @@ def name_is_map(map_id: str) -> bool: """return True if the given name is a map in the grass database False if not """ - return bool(gscript.find_file(name=map_id, element="cell").get("file")) + name, _, mapset = map_id.partition("@") + return bool(gutils.get_mapset_raster(name, mapset)) @staticmethod def set_null(map_id: str, threshold: float) -> None: @@ -398,7 +404,9 @@ def get_sim_extend_in_stds_unit(self, strds) -> tuple[int | datetime, int | date assert False, "unknown temporal type" return start_time_in_stds_unit, end_time_in_stds_unit - def stds_temporal_sanity(self, stds_id: str) -> bool: + def stds_temporal_sanity( + self, stds_id: str, stds: tgis.SpaceTimeRasterDataset | None = None + ) -> bool: """Make the following check on the given stds: - Topology is valid - No gap @@ -406,19 +414,21 @@ def stds_temporal_sanity(self, stds_id: str) -> bool: return True if all the above is True, False otherwise """ out = True - stds = tgis.open_stds.open_old_stds(stds_id, "strds") + if stds is None: + stds = tgis.open_stds.open_old_stds(stds_id, "strds") stds_start, stds_end = stds.get_temporal_extent_as_tuple() if stds_start is None or stds_end is None: msgr.fatal( f"STRDS <{stds_id}> has no temporal extent; " "make sure it contains registered raster maps" ) + maps = stds.get_registered_maps_as_objects(order="start_time") # valid topology - if not stds.check_temporal_topology(): + if not stds.check_temporal_topology(maps=maps): out = False msgr.warning(f"{stds_id}: invalid topology") # no gap - if stds.count_gaps() != 0: + if stds.count_gaps(maps=maps) != 0: out = False msgr.warning(f"{stds_id}: gaps found") # cover all simulation time @@ -440,6 +450,8 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: # transform simulation start and end time in strds unit strds = tgis.open_stds.open_old_stds(strds_name, "strds") + if not self.stds_temporal_sanity(strds_name, strds): + msgr.fatal(f"{strds_name}: inadequate temporal format") sim_start, sim_end = self.get_sim_extend_in_stds_unit(strds) # retrieve data from DB diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index e452689b..b847ab7a 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -9,7 +9,8 @@ import hashlib import json -from datetime import datetime +from collections.abc import Iterable +from datetime import datetime, timedelta from pathlib import Path from typing import Literal @@ -366,8 +367,12 @@ def _render_artifacts( if expanded.outputs.statistics_file is not None else None ) - _validate_output_names(output_map_names, drainage_output) - _validate_protected_file_output(stats_file, expanded.source.path, _drainage_path(expanded)) + _validate_output_names( + output_map_names, + drainage_output, + _last_record_index(expanded.time.duration, expanded.time.record_step), + ) + _validate_protected_file_output(stats_file, (expanded.source.path, _drainage_path(expanded))) return ArtifactSummary(tuple(sorted(output_map_names.items())), drainage_output, stats_file) @@ -384,18 +389,25 @@ def _drainage_path(expanded: ExpandedSimulation) -> Path | None: ) -def _validate_protected_file_output( - output: Path | None, config_path: Path, swmm_path: Path | None -) -> None: +def _validate_protected_file_output(output: Path | None, protected: Iterable[Path | None]) -> None: if output is None: return - protected = (config_path.resolve(), *(path for path in (swmm_path,) if path is not None)) - for source in protected: + for path in protected: + if path is None: + continue + source = path.resolve() if output == source or (output.exists() and source.exists() and output.samefile(source)): raise EnsembleError(f"output file <{output}> aliases protected input <{source}>") -def _validate_output_names(output_map_names: dict[str, str], drainage_output: str | None) -> None: +def _last_record_index(duration: timedelta, record_step: timedelta) -> int: + """Return the final output index, including the initial record at index zero.""" + return -(-duration // record_step) + + +def _validate_output_names( + output_map_names: dict[str, str], drainage_output: str | None, last_record_index: int +) -> None: from grass.pygrass import utils as gutils from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name @@ -410,11 +422,11 @@ def _validate_output_names(output_map_names: dict[str, str], drainage_output: st raise EnsembleError("MASK cannot be used as an output name") if not gutils.is_clean_name(name): raise EnsembleError(f"invalid GRASS output name {name!r}") - child = derived_record_name(name, 0) + child = derived_record_name(name, last_record_index) if child == "MASK" or not gutils.is_clean_name(child): raise EnsembleError(f"invalid derived GRASS output name {child!r}") if drainage_output is not None: - child = derived_record_name(drainage_output, 0) + child = derived_record_name(drainage_output, last_record_index) for table_name in derived_drainage_table_names(child): if not gutils.is_clean_name(table_name): raise EnsembleError(f"invalid derived drainage table name {table_name!r}") @@ -460,15 +472,20 @@ def _build_simulation_config( raise EnsembleError(details) from error -def validate_resolved_ensemble(simulations: tuple[ResolvedSimulation, ...]) -> None: +def validate_resolved_ensemble( + simulations: tuple[ResolvedSimulation, ...], protected_inputs: tuple[Path, ...] = () +) -> None: """Reject identity and artifact collisions among resolved ensemble members.""" identities: dict[str, str] = {} artifacts: dict[str, str] = {} ensemble_sources: set[str] = set() + protected_paths = {path.resolve() for path in protected_inputs} for simulation in simulations: ensemble_sources.update(simulation.simulation_config.input_map_names.values()) if simulation.effective_mask.source is not None: ensemble_sources.add(simulation.effective_mask.source) + if simulation.simulation_config.swmm_inp is not None: + protected_paths.add(simulation.simulation_config.swmm_inp.resolve()) for simulation in simulations: existing_payload = identities.get(simulation.simulation_id) if existing_payload is not None: @@ -479,12 +496,17 @@ def validate_resolved_ensemble(simulations: tuple[ResolvedSimulation, ...]) -> N mapset = simulation.grass_params.mapset from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name + last_record_index = _last_record_index( + simulation.simulation_config.end_time - simulation.simulation_config.start_time, + simulation.simulation_config.record_step, + ) for _, name in simulation.artifacts.output_map_names: _claim_artifact(artifacts, f"{name}@{mapset}", simulation.simulation_id) _reject_output_source_alias(name, mapset, ensemble_sources) - child = derived_record_name(name, 0) - _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) - _reject_output_source_alias(child, mapset, ensemble_sources) + for index in range(last_record_index + 1): + child = derived_record_name(name, index) + _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) + _reject_output_source_alias(child, mapset, ensemble_sources) if simulation.artifacts.drainage_output is not None: drainage_name = simulation.artifacts.drainage_output _claim_artifact( @@ -493,12 +515,14 @@ def validate_resolved_ensemble(simulations: tuple[ResolvedSimulation, ...]) -> N simulation.simulation_id, ) _reject_output_source_alias(drainage_name, mapset, ensemble_sources) - child = derived_record_name(drainage_name, 0) - _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) - _reject_output_source_alias(child, mapset, ensemble_sources) - for table_name in derived_drainage_table_names(child): - _claim_artifact(artifacts, f"table:{table_name}", simulation.simulation_id) + for index in range(last_record_index + 1): + child = derived_record_name(drainage_name, index) + _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) + _reject_output_source_alias(child, mapset, ensemble_sources) + for table_name in derived_drainage_table_names(child): + _claim_artifact(artifacts, f"table:{table_name}", simulation.simulation_id) if simulation.artifacts.statistics_file is not None: + _validate_protected_file_output(simulation.artifacts.statistics_file, protected_paths) _claim_artifact( artifacts, str(simulation.artifacts.statistics_file), @@ -507,11 +531,15 @@ def validate_resolved_ensemble(simulations: tuple[ResolvedSimulation, ...]) -> N def _claim_artifact(artifacts: dict[str, str], artifact: str, simulation_id: str) -> None: - owner = artifacts.setdefault(artifact, simulation_id) - if owner != simulation_id: - raise EnsembleError( - f"output artifact {artifact!r} is shared by simulations {owner} and {simulation_id}" - ) + owner = artifacts.get(artifact) + if owner is None: + artifacts[artifact] = simulation_id + return + if owner == simulation_id: + raise EnsembleError(f"output artifact {artifact!r} is used more than once") + raise EnsembleError( + f"output artifact {artifact!r} is shared by simulations {owner} and {simulation_id}" + ) def _reject_output_source_alias(name: str, mapset: str | None, sources: set[str]) -> None: diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index ac93d4a4..f46fcce5 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path from types import SimpleNamespace import pytest @@ -10,6 +11,7 @@ from itzi.cli_parser import build_parser from itzi.itzi import ( VerbosityLevel, + _preflight_selected, _run_ensemble_batch, itzi_run, itzi_run_one, @@ -54,7 +56,12 @@ def test_run_parser_accepts_yaml_dry_run_and_member_selection(dry_option): def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): simulations = (object(), object()) - ensemble = SimpleNamespace(ensemble_id="study", simulations=simulations) + ensemble = SimpleNamespace( + ensemble_id="study", + simulations=simulations, + source=SimpleNamespace(path=Path("study.yaml")), + manifest_template=None, + ) calls = [] monkeypatch.setattr("itzi.itzi.load_batch", lambda _: ((ensemble,), ())) @@ -62,7 +69,7 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): "itzi.itzi._resolve_ensemble_in_spawn", lambda received: calls.append(received) or (), ) - monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_: None) + monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_, **__: None) _run_ensemble_batch( SimpleNamespace(config_file=["study.yaml"], resume_from=[], member=[], dry=True) @@ -71,6 +78,34 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): assert calls == [simulations] +def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): + selected_simulation = SimpleNamespace(simulation_id="sim-selected") + unselected_simulation = SimpleNamespace(simulation_id="sim-unselected") + ensemble = SimpleNamespace( + ensemble_id="study", + manifest_template=None, + source=SimpleNamespace(path=tmp_path / "study.yaml"), + ) + calls = [] + monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + "itzi.itzi._preflight_resolved_in_spawn", + lambda simulation: calls.append(simulation.simulation_id) or None, + ) + + member_failures, manifest_failures = _preflight_selected( + [(ensemble, (selected_simulation, unselected_simulation), ())], + {"study": {"sim-selected"}}, + {}, + has_selectors=True, + overwrite=False, + ) + + assert calls == ["sim-selected"] + assert member_failures == {} + assert manifest_failures == {} + + def test_run_parser_rejects_v_and_q_together(): with pytest.raises(SystemExit): build_parser().parse_args(["run", "a.ini", "-v", "-q"]) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 56a9ff35..5869878a 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -23,7 +23,12 @@ OptionSweepConfig, StatisticsOutputs, ) -from itzi.resolution import _render_artifacts, _resolve_inputs +from itzi.resolution import ( + _last_record_index, + _render_artifacts, + _resolve_inputs, + validate_resolved_ensemble, +) def write_yaml(tmp_path, content: str): @@ -313,6 +318,7 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): drainage_dataset=None, ), drainage=None, + time=SimpleNamespace(duration=timedelta(hours=1), record_step=timedelta(minutes=5)), ) monkeypatch.setattr("itzi.resolution._validate_output_names", lambda *_: None) @@ -321,6 +327,55 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" +def test_generated_output_inventory_includes_final_partial_record(): + assert _last_record_index(timedelta(seconds=60), timedelta(seconds=30)) == 2 + assert _last_record_index(timedelta(seconds=61), timedelta(seconds=30)) == 3 + + +def test_later_generated_output_cannot_alias_an_input(): + start = datetime(2026, 1, 1) # noqa: DTZ001 + config = SimpleNamespace( + start_time=start, + end_time=start + timedelta(minutes=1), + record_step=timedelta(seconds=30), + input_map_names={}, + swmm_inp=None, + ) + output = SimpleNamespace( + simulation_id="sim-output", + normalized_payload="output", + grass_params=SimpleNamespace(mapset="PERMANENT"), + effective_mask=SimpleNamespace(source=None), + simulation_config=config, + artifacts=SimpleNamespace( + output_map_names=(("water_depth", "result"),), + drainage_output=None, + statistics_file=None, + ), + ) + input_simulation = SimpleNamespace( + simulation_id="sim-input", + normalized_payload="input", + grass_params=SimpleNamespace(mapset="PERMANENT"), + effective_mask=SimpleNamespace(source=None), + simulation_config=SimpleNamespace( + start_time=start, + end_time=start + timedelta(minutes=1), + record_step=timedelta(seconds=30), + input_map_names={"ground_elevation": "result_0002@PERMANENT"}, + swmm_inp=None, + ), + artifacts=SimpleNamespace( + output_map_names=(), + drainage_output=None, + statistics_file=None, + ), + ) + + with pytest.raises(ValueError, match="result_0002@PERMANENT.*aliases"): + validate_resolved_ensemble((output, input_simulation)) + + def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): calls = [] diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 6a1aa856..b8590f87 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -3,15 +3,32 @@ from __future__ import annotations import os -from datetime import datetime +import sqlite3 +from collections.abc import Iterator +from contextlib import closing +from datetime import datetime, timedelta from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 import grass.script as gscript +import grass.temporal as tgis import pytest import yaml +from grass.pygrass import utils as gutils +from itzi_core import TemporalType from itzi.cli_parser import build_parser +from itzi.ensemble_models import EnsembleError from itzi.itzi import itzi_run +from itzi.preflight import _validate_grass_outputs +from itzi.providers.grass_interface import _init_temporal + + +@pytest.fixture +def stop_temporal_subprocesses() -> Iterator[None]: + yield + tgis.stop_subprocesses() @pytest.mark.forked @@ -127,3 +144,177 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da assert { f"{member['artifacts']['rasters']['water_depth']}_0000" for member in manifest["members"] } <= {path.name for path in mapset_path.iterdir()} + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): + gscript.mapcalc("dry_null_dem=null()") + config_path = tmp_path / "dry.yaml" + config_path.write_text( + """\ +schema_version: 1 +ensemble: + id: dry +domain: {} +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "dry_null_dem" + friction: "n" +options: {} +outputs: + rasters: + prefix: "dry_output" + variables: [water_depth] + statistics: + file: "dry-results/statistics.csv" +""", + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (tmp_path / "results" / "dry.manifest.yaml").exists() + assert not (tmp_path / "dry-results").exists() + assert not gscript.find_file(name="dry_output_water_depth_0000", element="cell").get("file") + + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(config_path)])) + + manifest = yaml.safe_load( + (tmp_path / "results" / "dry.manifest.yaml").read_text(encoding="utf-8") + ) + assert manifest["members"][0]["status"] == "validation_failed" + assert manifest["members"][0]["failure"]["phase"] == "preflight" + assert not (tmp_path / "dry-results").exists() + assert not gscript.find_file(name="dry_output_water_depth_0000", element="cell").get("file") + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_does_not_read_rainfall_cells(tmp_path): + gscript.mapcalc("dry_null_rain_0=null()") + gscript.mapcalc("dry_null_rain_1=1") + gscript.run_command( + "t.create", + output="dry_rain_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="dry_rain_series", + description="dry_rain_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="dry_rain_series", + type="raster", + maps="dry_null_rain_0,dry_null_rain_1", + start="0", + increment="30", + unit="seconds", + ) + config_path = tmp_path / "dry-rain.yaml" + config_path.write_text( + """\ +schema_version: 1 +ensemble: + id: dry-rain +domain: {} +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" + rainfall_rate: "dry_rain_series" +options: {} +outputs: + rasters: + prefix: "dry_rain_output" + variables: [water_depth] +""", + encoding="utf-8", + ) + + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + assert not (tmp_path / "results" / "dry-rain.manifest.yaml").exists() + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_checks_every_generated_output_record(test_data_temp_path): + gscript.mapcalc("existing_output_water_depth_0002=1") + config_path = Path(test_data_temp_path) / "existing-output.yaml" + config_path.write_text( + """\ +schema_version: 1 +ensemble: + id: existing-output +domain: {} +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" +options: {} +outputs: + rasters: + prefix: "existing_output" + variables: [water_depth] +""", + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (Path(test_data_temp_path) / "results" / "existing-output.manifest.yaml").exists() + assert not gscript.find_file(name="existing_output_water_depth_0000", element="cell").get( + "file" + ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5", "stop_temporal_subprocesses") +def test_preflight_checks_existing_drainage_tables(): + _init_temporal() + mapset = gutils.getenv("MAPSET") + database = ( + Path(gutils.getenv("GISDBASE")) + / gutils.getenv("LOCATION_NAME") + / mapset + / "sqlite" + / "sqlite.db" + ) + name = f"drainage_collision_{uuid4().hex[:8]}" + simulation = SimpleNamespace( + simulation_config=SimpleNamespace( + start_time=datetime(2020, 1, 1), + end_time=datetime(2020, 1, 1) + timedelta(minutes=1), + record_step=timedelta(seconds=30), + output_map_names={}, + drainage_output=name, + temporal_type=TemporalType.RELATIVE, + ) + ) + interface = SimpleNamespace( + overwrite=False, + get_current_mapset=lambda: mapset, + format_id=lambda value: f"{value}@{mapset}", + validate_output_stds_temporal_type=lambda *_: None, + ) + + database_existed = database.exists() + _validate_grass_outputs(simulation, interface) + assert database.exists() == database_existed + database.parent.mkdir(parents=True, exist_ok=True) + with closing(sqlite3.connect(database)) as connection: + connection.execute(f"CREATE TABLE {name}_0002_link (cat integer)") + connection.commit() + with pytest.raises(EnsembleError, match=f"drainage table <{name}_0002_link> already exists"): + _validate_grass_outputs(simulation, interface) diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index b6a8401f..d1489958 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -16,7 +16,7 @@ def format_id(name: str) -> str: return f"{name}@test" @staticmethod - def name_is_stds(map_id: str) -> bool: + def name_is_stds(map_id: str, *, initialize: bool = True) -> bool: return map_id == "series@test" @staticmethod @@ -27,13 +27,15 @@ def stds_temporal_sanity(self, strds_id: str) -> bool: self.temporal_checks.append(strds_id) return True - @staticmethod - def raster_list_from_strds(strds_id: str) -> list[MapData]: + def raster_list_from_strds(self, strds_id: str) -> list[MapData]: + self.stds_temporal_sanity(strds_id) start = datetime(2020, 1, 1) return [MapData(strds_id, start, start + timedelta(hours=1))] -@pytest.mark.parametrize("input_kinds", [None, {"raster": "raster", "series": "strds"}]) +@pytest.mark.parametrize( + "input_kinds", [None, {"ground_elevation": "raster", "rainfall_rate": "strds"}] +) def test_input_kinds_share_raster_and_strds_construction( input_kinds: dict[str, Literal["raster", "strds"]] | None, ) -> None: @@ -43,12 +45,12 @@ def test_input_kinds_share_raster_and_strds_construction( provider = GrassRasterInputProvider( interface, - {"raster": "raster", "series": "series"}, + {"ground_elevation": "raster", "rainfall_rate": "series"}, start, end, input_kinds, ) - assert provider.map_lists["raster"] == [MapData("raster@test", start, end)] - assert provider.map_lists["series"] == [MapData("series@test", start, end)] + assert provider.map_lists["ground_elevation"] == [MapData("raster@test", start, end)] + assert provider.map_lists["rainfall_rate"] == [MapData("series@test", start, end)] assert interface.temporal_checks == ["series@test"] From 4f42946f80f6e29687f356b9c8967b5487280be3 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 16:33:18 -0600 Subject: [PATCH 34/85] make all preflight checks in a single subprocess per ensemble --- src/itzi/grass_session.py | 34 +++++++++++++++-- src/itzi/itzi.py | 43 ++++++++++++++++----- src/itzi/providers/grass_input.py | 4 +- src/itzi/providers/grass_interface.py | 23 ++--------- src/itzi/resolution.py | 12 ------ src/itzi/simulation_runner.py | 3 +- tests/cli/test_cli.py | 53 ++++++++++++++++++++++---- tests/cli/test_ensemble_config.py | 2 - tests/grass/test_ensemble.py | 37 +++++++++++++++++- tests/grass/test_grass_input.py | 2 +- tests/grass/test_temporal_overwrite.py | 2 + tests/test_grass_session.py | 33 +++++++++++++++- 12 files changed, 189 insertions(+), 59 deletions(-) diff --git a/src/itzi/grass_session.py b/src/itzi/grass_session.py index 8f4d19f5..d0931cc3 100644 --- a/src/itzi/grass_session.py +++ b/src/itzi/grass_session.py @@ -24,6 +24,8 @@ import itzi.messenger as msgr +_initialized_temporal_session: tuple[int, str, tuple[str, str, str]] | None = None + class GrassParams(BaseModel): """Parameters for GRASS GIS session.""" @@ -92,10 +94,29 @@ def _validate_requested_context(self) -> None: f"({active_database}/{active[1]}/{active[2]})" ) - def open(self): + @staticmethod + def ensure_temporal_initialized() -> None: + """Initialize the temporal framework once for the active GRASS session.""" + global _initialized_temporal_session + + context = GrassSessionManager.current_context() + if context is None: + msgr.fatal("No active GRASS session for temporal initialization") + import grass.script as gscript + import grass.temporal as tgis + + gscript.set_raise_on_error(True) + session = (os.getpid(), os.environ["GISRC"], context) + if _initialized_temporal_session != session: + tgis.init(raise_fatal_error=True) + _initialized_temporal_session = session + tgis.set_raise_on_error(True) + + def open(self) -> None: """Open a GRASS session if needed.""" if self.current_context() is not None: self._validate_requested_context() + self.ensure_temporal_initialized() return # Check if mandatory GRASS parameters are present @@ -137,15 +158,22 @@ def open(self): session_env = getattr(self.grass_session, "env", None) if session_env and session_env.get("GISRC"): os.environ.update(session_env) - self._validate_requested_context() + try: + self._validate_requested_context() + self.ensure_temporal_initialized() + except Exception: + self.close() + raise - def close(self): + def close(self) -> None: """Stop GRASS session.""" + global _initialized_temporal_session if self.grass_session is not None and self._owns_session: try: self.grass_session.finish() except Exception as e: print(f"Warning: Error cleaning up GRASS session: {e}") + _initialized_temporal_session = None self.grass_session = None self._owns_session = False diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 9192b276..9161692a 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -161,6 +161,22 @@ def preflight_worker(simulation: ResolvedSimulation) -> str | None: return None +def preflight_ensemble_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: + """Preflight selected members in one GRASS session, retaining individual failures.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + failures: dict[str, str] = {} + with GrassSessionManager(simulations[0].grass_params): + from itzi.preflight import preflight_simulation + + for simulation in simulations: + try: + preflight_simulation(simulation) + except Exception as error: + failures[simulation.simulation_id] = f"{type(error).__name__}: {error}" + return failures + + def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: """Run a simulation in a subprocess""" worker_args = (conf_file, hotstart_file) @@ -473,6 +489,15 @@ def _preflight_resolved_in_spawn(simulation: ResolvedSimulation) -> str | None: return f"{type(error).__name__}: {error}" +def _preflight_ensemble_in_spawn(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: + try: + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(preflight_ensemble_worker, simulations).result() + except Exception as error: + detail = f"{type(error).__name__}: {error}" + return {simulation.simulation_id: detail for simulation in simulations} + + def _preflight_selected( resolved: list[ tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] @@ -507,15 +532,15 @@ def _preflight_selected( ) except EnsembleError as error: manifest_failures[ensemble.ensemble_id] = str(error) - for simulation in simulations: - if ( - simulation.simulation_id not in selected_ids - or simulation.simulation_id in failures - ): - continue - detail = _preflight_resolved_in_spawn(simulation) - if detail is not None: - failures[simulation.simulation_id] = { + to_preflight = tuple( + simulation + for simulation in simulations + if simulation.simulation_id in selected_ids + and simulation.simulation_id not in failures + ) + if to_preflight: + for simulation_id, detail in _preflight_ensemble_in_spawn(to_preflight).items(): + failures[simulation_id] = { "phase": "preflight", "detail": detail, } diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index de4b0ef8..24472e0f 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -107,12 +107,12 @@ def resolve_input_map_lists( map_id = grass_interface.format_id(map_name) kind = input_kinds.get(key) if input_kinds is not None else None if kind == "strds": - if not grass_interface.name_is_stds(map_id, initialize=False): + if not grass_interface.name_is_stds(map_id): msgr.fatal(f"STRDS input <{map_id}> is no longer available") elif kind == "raster": if not grass_interface.name_is_map(map_id): msgr.fatal(f"raster input <{map_id}> is no longer available") - elif grass_interface.name_is_stds(map_id, initialize=False): + elif grass_interface.name_is_stds(map_id): kind = "strds" elif grass_interface.name_is_map(map_id): kind = "raster" diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index d8a38441..4e5244ef 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -58,35 +58,25 @@ } -def _init_temporal() -> None: - """Initialize GRASS temporal APIs without allowing them to exit Itzi.""" - gscript.set_raise_on_error(True) - tgis.init(raise_fatal_error=True) - tgis.set_raise_on_error(True) - - # Check if color rule paths are OK for f in colors_rules_dict.values(): assert Path(f).is_file() -def file_exists(name: str, *, initialize: bool = True) -> bool: +def file_exists(name: str) -> bool: """Return True if name is an existing map or stds, False otherwise""" if not name: return False _id = GrassInterface.format_id(name) - return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds( - _id, initialize=initialize - ) + return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds(_id) def check_output_files(file_list: list[str]) -> None: """Check if the output files exist""" if gscript.overwrite() or not file_list: return - _init_temporal() for map_name in file_list: - if file_exists(map_name, initialize=False): + if file_exists(map_name): msgr.fatal(f"File {map_name} exists and will not be overwritten") @@ -222,8 +212,6 @@ def __init__( } self.mask_mode, self.mask_source = self._select_effective_mask(effective_mask) self.overwrite = gscript.overwrite() - # init temporal module - _init_temporal() # Create thread and queue for writing raster maps if self.non_blocking_write: self.raster_lock = Lock() @@ -352,13 +340,10 @@ def get_current_mapset() -> str: return gutils.getenv("MAPSET") @staticmethod - def name_is_stds(name: str, *, initialize: bool = True) -> bool: + def name_is_stds(name: str) -> bool: """return True if the name given as input is a registered strds False if not """ - # make sure temporal module is initialized - if initialize: - _init_temporal() return bool(tgis.SpaceTimeRasterDataset(name).is_in_db()) @staticmethod diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index b847ab7a..bce7f79f 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -51,7 +51,6 @@ def resolve_ensemble( actual_params = _active_grass_params(requested_params, getenv) domain = _read_domain(simulations[0].domain.region, Region, gscript) effective_mask = _resolve_effective_mask(simulations[0].domain.mask, getenv) - _initialize_temporal() input_cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] = {} swmm_cache: dict[Path, str] = {} results: list[ResolvedSimulation | ValidationFailure] = [] @@ -61,7 +60,6 @@ def resolve_ensemble( dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), Mapset, cache=input_cache, - initialize_temporal=False, ) swmm_path, swmm_digest = _resolve_swmm_input(expanded, swmm_cache) results.append( @@ -188,10 +186,7 @@ def _resolve_inputs( mapset_type: type, *, cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] | None = None, - initialize_temporal: bool = True, ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: - if initialize_temporal: - _initialize_temporal() resolved: dict[str, str] = {} kinds: dict[str, Literal["raster", "strds"]] = {} for key, identifier in inputs.items(): @@ -213,13 +208,6 @@ def _resolve_inputs( return resolved, kinds -def _initialize_temporal() -> None: - import grass.temporal as tgis - - tgis.init(raise_fatal_error=True) - tgis.set_raise_on_error(True) - - def _resolve_input_identifier( identifier: str, mapset_type: type ) -> tuple[str, Literal["raster", "strds"]]: diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 7e5937c4..880f0963 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -23,7 +23,7 @@ import itzi.messenger as msgr from itzi.ensemble_models import EffectiveMask -from itzi.grass_session import GrassParams +from itzi.grass_session import GrassParams, GrassSessionManager if TYPE_CHECKING: from itzi_core import Simulation @@ -61,6 +61,7 @@ def __init__( f"Itzi requires at least GRASS {self.grass_required_version}, " f"version {grass_version} detected." ) + GrassSessionManager.ensure_temporal_initialized() msgr.debug("GRASS session set") # return error if output files exist diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index f46fcce5..4720acc6 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2,8 +2,9 @@ import argparse import os +import sys from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace import pytest @@ -16,6 +17,7 @@ itzi_run, itzi_run_one, main, + preflight_ensemble_worker, reconcile_hotstart_commands, sim_runner_worker, ) @@ -80,6 +82,7 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): selected_simulation = SimpleNamespace(simulation_id="sim-selected") + selected_with_failure = SimpleNamespace(simulation_id="sim-failed") unselected_simulation = SimpleNamespace(simulation_id="sim-unselected") ensemble = SimpleNamespace( ensemble_id="study", @@ -89,23 +92,59 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): calls = [] monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) monkeypatch.setattr( - "itzi.itzi._preflight_resolved_in_spawn", - lambda simulation: calls.append(simulation.simulation_id) or None, + "itzi.itzi._preflight_ensemble_in_spawn", + lambda simulations: ( + calls.append(tuple(s.simulation_id for s in simulations)) + or {"sim-failed": "invalid input"} + ), ) member_failures, manifest_failures = _preflight_selected( - [(ensemble, (selected_simulation, unselected_simulation), ())], - {"study": {"sim-selected"}}, + [(ensemble, (selected_simulation, selected_with_failure, unselected_simulation), ())], + {"study": {"sim-selected", "sim-failed"}}, {}, has_selectors=True, overwrite=False, ) - assert calls == ["sim-selected"] - assert member_failures == {} + assert calls == [("sim-selected", "sim-failed")] + assert member_failures == { + "study": {"sim-failed": {"phase": "preflight", "detail": "invalid input"}} + } assert manifest_failures == {} +def test_preflight_ensemble_worker_keeps_member_failures(monkeypatch): + calls = [] + + class FakeGrassSessionManager: + def __init__(self, params): + assert params == "shared" + + def __enter__(self): + calls.append("open") + + def __exit__(self, *_): + calls.append("close") + + def preflight(simulation): + calls.append(simulation.simulation_id) + if simulation.simulation_id == "bad": + raise ValueError("invalid input") + + monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) + fake_preflight = ModuleType("itzi.preflight") + fake_preflight.preflight_simulation = preflight + monkeypatch.setitem(sys.modules, "itzi.preflight", fake_preflight) + simulations = tuple( + SimpleNamespace(simulation_id=simulation_id, grass_params="shared") + for simulation_id in ("good", "bad", "another") + ) + + assert preflight_ensemble_worker(simulations) == {"bad": "ValueError: invalid input"} + assert calls == ["open", "good", "bad", "another", "close"] + + def test_run_parser_rejects_v_and_q_together(): with pytest.raises(SystemExit): build_parser().parse_args(["run", "a.ini", "-v", "-q"]) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 5869878a..e756b546 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -390,13 +390,11 @@ def resolve(identifier, _mapset_type): {"ground_elevation": "shared", "rainfall_rate": "first"}, object, cache=cache, - initialize_temporal=False, ) resolved, _ = _resolve_inputs( {"ground_elevation": "shared", "rainfall_rate": "second"}, object, cache=cache, - initialize_temporal=False, ) assert calls == ["shared", "first", "second"] diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index b8590f87..949b6d57 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -20,9 +20,9 @@ from itzi.cli_parser import build_parser from itzi.ensemble_models import EnsembleError +from itzi.grass_session import GrassSessionManager from itzi.itzi import itzi_run from itzi.preflight import _validate_grass_outputs -from itzi.providers.grass_interface import _init_temporal @pytest.fixture @@ -146,6 +146,39 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da } <= {path.name for path in mapset_path.iterdir()} +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_preflights_multiple_members_in_one_session(tmp_path): + prefix = f"dry_multi_{uuid4().hex[:8]}" + config_path = tmp_path / "dry-multi.yaml" + config_path.write_text( + f"""\ +schema_version: 1 +ensemble: + id: dry-multi +domain: {{}} +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" +options: + cfl: [0.2, 0.3] +outputs: + rasters: + prefix: "{prefix}_{{simulation}}" + variables: [water_depth] +""", + encoding="utf-8", + ) + + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (tmp_path / "results" / "dry-multi.manifest.yaml").exists() + assert not gscript.list_strings(type="raster", pattern=f"{prefix}_*") + + @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): @@ -282,7 +315,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5", "stop_temporal_subprocesses") def test_preflight_checks_existing_drainage_tables(): - _init_temporal() + GrassSessionManager.ensure_temporal_initialized() mapset = gutils.getenv("MAPSET") database = ( Path(gutils.getenv("GISDBASE")) diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index d1489958..6c2a9ec1 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -16,7 +16,7 @@ def format_id(name: str) -> str: return f"{name}@test" @staticmethod - def name_is_stds(map_id: str, *, initialize: bool = True) -> bool: + def name_is_stds(map_id: str) -> bool: return map_id == "series@test" @staticmethod diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index 281b34e6..f88bc144 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -10,6 +10,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader +from itzi.grass_session import GrassSessionManager from itzi_core.const import TemporalType @@ -80,6 +81,7 @@ def test_temporal_fatal_errors_are_raised_as_exceptions() -> None: from itzi.providers.grass_interface import GrassInterface tgis.set_raise_on_error(False) + GrassSessionManager.ensure_temporal_initialized() with pytest.raises(FatalError, match="mapset is missing"): GrassInterface.name_is_stds("missing_strds") diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index c305227f..d9f53099 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -2,7 +2,7 @@ import sys from types import ModuleType, SimpleNamespace -from itzi.grass_session import GrassSessionManager +from itzi.grass_session import GrassParams, GrassSessionManager class FakeGrassSession: @@ -23,6 +23,7 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: monkeypatch.setattr("itzi.grass_session.importlib.util.find_spec", lambda _: None) monkeypatch.setattr("itzi.grass_session.os.access", lambda *_: True) + monkeypatch.setattr(GrassSessionManager, "ensure_temporal_initialized", lambda self: None) monkeypatch.setattr( "itzi.grass_session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" ) @@ -54,3 +55,33 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: assert os.environ["GISRC"] == "/tmp/worker-gisrc" assert session.finished + + +def test_temporal_initialization_once_per_active_session(monkeypatch) -> None: + calls: list[str] = [] + script = ModuleType("grass.script") + script.set_raise_on_error = lambda _: calls.append("script") + temporal = ModuleType("grass.temporal") + temporal.init = lambda **_: calls.append("init") + temporal.set_raise_on_error = lambda _: calls.append("temporal") + package = ModuleType("grass") + package.script = script + package.temporal = temporal + monkeypatch.setitem(sys.modules, "grass", package) + monkeypatch.setitem(sys.modules, "grass.script", script) + monkeypatch.setitem(sys.modules, "grass.temporal", temporal) + monkeypatch.setattr("itzi.grass_session._initialized_temporal_session", None) + monkeypatch.setattr( + GrassSessionManager, + "current_context", + staticmethod(lambda: ("/grassdata", "location", "mapset")), + ) + monkeypatch.setenv("GISRC", "/tmp/first-gisrc") + + GrassSessionManager(GrassParams()).open() + GrassSessionManager.ensure_temporal_initialized() + assert calls.count("init") == 1 + + monkeypatch.setenv("GISRC", "/tmp/second-gisrc") + GrassSessionManager(GrassParams()).open() + assert calls.count("init") == 2 From 7a6a7246f870fea1712b35ecb2bf2312f650751a Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 16:40:48 -0600 Subject: [PATCH 35/85] agents: immutable objects and pygrass --- AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index a2209c13..e4feb3fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,11 +7,13 @@ ## Code style - Use python type annotation. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. -- Do not quote class names in hints. Use `from __future__ import annotations` when necessary. +- Do not quote class names in hints. Use `from __futue__ import annotations` when necessary. - Do not use the `Any` type, and avoid the use of the general `object` type. Define the actual type instead. - Since the arguments types and return types are already documented by the annotations, there's no need to duplicate this information in the docstrings. - Use pydantic BaseModel when validation is needed, and dataclass for internal data exchange. - Place imports at the top of the file. Only break this rule to prevent heavy imports in a rarely used function (for example, CLI options). +- When passing data structures between fucntions, prefer immutable objects instead of equivalent mutable ones, unless immutability is desired. +- For GRASS interactions, prefer C functions calls through `pygrass` when available, instead of sub-process tool calls through `grass.script` ## General comments - The project uses `uv`. To run a command in the correct environment, use `uv run` From b7e3d5f332e84bc27e68e36492cfe36bff40dd02 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 17:01:04 -0600 Subject: [PATCH 36/85] split out one ensemble run in its own function --- src/itzi/ensemble_models.py | 10 ++ src/itzi/itzi.py | 207 ++++++++++++++++++++---------------- tests/cli/test_cli.py | 61 ++++++++++- 3 files changed, 182 insertions(+), 96 deletions(-) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 9f0a6d18..8254bb1d 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -225,6 +225,16 @@ class ValidationFailure: detail: str +@dataclass(frozen=True) +class ResolvedEnsemble: + """One expanded ensemble with its resolved members and validation results.""" + + ensemble: ExpandedEnsemble + simulations: tuple[ResolvedSimulation, ...] + failures: tuple[ValidationFailure, ...] + artifact_failure: str | None = None + + @dataclass(frozen=True) class LoadedYamlStream: """Valid ensembles and document-local failures from one YAML file.""" diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 9161692a..60a44669 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -40,8 +40,8 @@ from itzi.configreader import ConfigReader from itzi.ensemble_models import ( EnsembleError, - ExpandedEnsemble, ExpandedSimulation, + ResolvedEnsemble, ResolvedSimulation, ValidationFailure, ) @@ -348,10 +348,7 @@ def _run_ensemble_batch(cli_args) -> None: for failure in document_failures: msgr.warning(failure.format()) - resolved: list[ - tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] - ] = [] - artifact_failures: dict[str, str] = {} + resolved: list[ResolvedEnsemble] = [] for ensemble in ensembles: successful: list[ResolvedSimulation] = [] member_failures: list[ValidationFailure] = [] @@ -360,11 +357,14 @@ def _run_ensemble_batch(cli_args) -> None: successful.append(result) else: member_failures.append(result) + artifact_failure = None try: validate_resolved_ensemble(tuple(successful), (ensemble.source.path,)) except EnsembleError as error: - artifact_failures[ensemble.ensemble_id] = str(error) - resolved.append((ensemble, tuple(successful), tuple(member_failures))) + artifact_failure = str(error) + resolved.append( + ResolvedEnsemble(ensemble, tuple(successful), tuple(member_failures), artifact_failure) + ) selectors = getattr(cli_args, "member", []) has_selectors = bool(selectors) @@ -372,7 +372,6 @@ def _run_ensemble_batch(cli_args) -> None: preflight_failures, manifest_failures = _preflight_selected( resolved, selected, - artifact_failures, has_selectors=has_selectors, overwrite=bool(getattr(cli_args, "o", False)), ) @@ -380,74 +379,20 @@ def _run_ensemble_batch(cli_args) -> None: _display_dry_plan( resolved, selected, preflight_failures, manifest_failures, has_selectors=has_selectors ) - unresolved_selected = not has_selectors and any(failures for _, _, failures in resolved) + unresolved_selected = not has_selectors and any(item.failures for item in resolved) if document_failures or unresolved_selected or preflight_failures or manifest_failures: msgr.fatal("YAML batch validation failed") return failure_count = len(document_failures) - for ensemble, simulations, failures in resolved: - selected_ids = selected.get(ensemble.ensemble_id, set()) - if not selected_ids and not (not has_selectors and failures): - continue - manifest_path = _manifest_path(ensemble) - states = _initial_member_states(simulations, failures, selected_ids, has_selectors) - for simulation_id, failure in preflight_failures.get(ensemble.ensemble_id, {}).items(): - states[simulation_id]["status"] = "validation_failed" - states[simulation_id]["failure"] = failure - if ensemble.ensemble_id in manifest_failures: - msgr.warning(manifest_failures[ensemble.ensemble_id]) - failure_count += max(1, len(selected_ids)) - continue - try: - _create_manifest(manifest_path, ensemble, states, overwrite=bool(cli_args.o)) - except OSError as error: - msgr.warning(f"Cannot create manifest <{manifest_path}>: {error}") - failure_count += max(1, len(selected_ids)) - continue - - for simulation in simulations: - if simulation.simulation_id not in selected_ids: - continue - if states[simulation.simulation_id]["status"] != "planned": - continue - detail = _preflight_resolved_in_spawn(simulation) - if detail is not None: - states[simulation.simulation_id]["status"] = "validation_failed" - states[simulation.simulation_id]["failure"] = { - "phase": "preflight", - "detail": detail, - } - _update_manifest(manifest_path, ensemble, states) - continue - states[simulation.simulation_id]["status"] = "running" - try: - _update_manifest(manifest_path, ensemble, states) - except OSError as error: - msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") - failure_count += 1 - break - started = time.monotonic() - status, detail = _run_resolved_in_spawn(simulation) - state = states[simulation.simulation_id] - state["status"] = status - state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") - if detail is not None: - states[simulation.simulation_id]["failure"] = { - "phase": "execution", - "detail": detail, - } - failure_count += 1 - try: - _update_manifest(manifest_path, ensemble, states) - except OSError as error: - msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") - failure_count += 1 - break - failure_count += sum( - 1 - for state in states.values() - if state["status"] == "validation_failed" and state.get("selected", False) + for item in resolved: + failure_count += _run_one_ensemble( + item, + selected.get(item.ensemble.ensemble_id, set()), + preflight_failures.get(item.ensemble.ensemble_id, {}), + manifest_failures.get(item.ensemble.ensemble_id), + has_selectors=has_selectors, + overwrite=bool(cli_args.o), ) if failure_count: @@ -455,6 +400,82 @@ def _run_ensemble_batch(cli_args) -> None: msgr.message("Simulation(s) complete.") +def _run_one_ensemble( + resolved: ResolvedEnsemble, + selected_ids: set[str], + preflight_failures: dict[str, dict[str, str]], + manifest_failure: str | None, + *, + has_selectors: bool, + overwrite: bool, +) -> int: + """Run selected members and return the number of failures in this ensemble.""" + ensemble = resolved.ensemble + if not selected_ids and not (not has_selectors and resolved.failures): + return 0 + manifest_path = _manifest_path(ensemble) + states = _initial_member_states( + resolved.simulations, resolved.failures, selected_ids, has_selectors + ) + for simulation_id, failure in preflight_failures.items(): + states[simulation_id]["status"] = "validation_failed" + states[simulation_id]["failure"] = failure + if manifest_failure is not None: + msgr.warning(manifest_failure) + return max(1, len(selected_ids)) + try: + _create_manifest(manifest_path, ensemble, states, overwrite=overwrite) + except OSError as error: + msgr.warning(f"Cannot create manifest <{manifest_path}>: {error}") + return max(1, len(selected_ids)) + + failure_count = 0 + for simulation in resolved.simulations: + if simulation.simulation_id not in selected_ids: + continue + if states[simulation.simulation_id]["status"] != "planned": + continue + detail = _preflight_resolved_in_spawn(simulation) + if detail is not None: + states[simulation.simulation_id]["status"] = "validation_failed" + states[simulation.simulation_id]["failure"] = { + "phase": "preflight", + "detail": detail, + } + _update_manifest(manifest_path, ensemble, states) + continue + states[simulation.simulation_id]["status"] = "running" + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + started = time.monotonic() + status, detail = _run_resolved_in_spawn(simulation) + state = states[simulation.simulation_id] + state["status"] = status + state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") + if detail is not None: + states[simulation.simulation_id]["failure"] = { + "phase": "execution", + "detail": detail, + } + failure_count += 1 + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + failure_count += sum( + 1 + for state in states.values() + if state["status"] == "validation_failed" and state.get("selected", False) + ) + return failure_count + + def _resolve_ensemble_in_spawn( expanded: tuple[ExpandedSimulation, ...], ) -> tuple[ResolvedSimulation | ValidationFailure, ...]: @@ -499,11 +520,8 @@ def _preflight_ensemble_in_spawn(simulations: tuple[ResolvedSimulation, ...]) -> def _preflight_selected( - resolved: list[ - tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] - ], + resolved: list[ResolvedEnsemble], selected: dict[str, set[str]], - artifact_failures: dict[str, str], *, has_selectors: bool, overwrite: bool, @@ -511,12 +529,14 @@ def _preflight_selected( """Collect selected-member and manifest failures before creating artifacts.""" member_failures: dict[str, dict[str, dict[str, str]]] = {} manifest_failures: dict[str, str] = {} - for ensemble, simulations, _ in resolved: + for item in resolved: + ensemble = item.ensemble + simulations = item.simulations selected_ids = selected[ensemble.ensemble_id] if not selected_ids and has_selectors: continue failures = member_failures.setdefault(ensemble.ensemble_id, {}) - artifact_detail = artifact_failures.get(ensemble.ensemble_id) + artifact_detail = item.artifact_failure if artifact_detail is not None: for simulation_id in selected_ids: failures[simulation_id] = { @@ -550,28 +570,28 @@ def _preflight_selected( def _select_members( - resolved: list[ - tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] - ], + resolved: list[ResolvedEnsemble], selectors: list[str], ) -> dict[str, set[str]]: """Resolve member selectors only against successfully resolved members.""" - selected = {ensemble.ensemble_id: set() for ensemble, _, _ in resolved} + selected = {item.ensemble.ensemble_id: set() for item in resolved} all_members = { - f"{ensemble.ensemble_id}#{simulation.simulation_id}": simulation.simulation_id - for ensemble, simulations, _ in resolved - for simulation in simulations + f"{item.ensemble.ensemble_id}#{simulation.simulation_id}": simulation.simulation_id + for item in resolved + for simulation in item.simulations } if not selectors: - for ensemble, simulations, _ in resolved: - selected[ensemble.ensemble_id] = { - simulation.simulation_id for simulation in simulations + for item in resolved: + selected[item.ensemble.ensemble_id] = { + simulation.simulation_id for simulation in item.simulations } return selected if any("#" not in selector for selector in selectors) and len(resolved) != 1: msgr.fatal("Unqualified --member IDs require a batch with exactly one ensemble") for selector in selectors: - qualified = selector if "#" in selector else f"{resolved[0][0].ensemble_id}#{selector}" + qualified = ( + selector if "#" in selector else f"{resolved[0].ensemble.ensemble_id}#{selector}" + ) simulation_id = all_members.get(qualified) if simulation_id is None: msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") @@ -581,18 +601,17 @@ def _select_members( def _display_dry_plan( - resolved: list[ - tuple[ExpandedEnsemble, tuple[ResolvedSimulation, ...], tuple[ValidationFailure, ...]] - ], + resolved: list[ResolvedEnsemble], selected: dict[str, set[str]], preflight_failures: dict[str, dict[str, dict[str, str]]], manifest_failures: dict[str, str], *, has_selectors: bool, ) -> None: - for ensemble, simulations, failures in resolved: + for item in resolved: + ensemble = item.ensemble msgr.message(f"Ensemble {ensemble.ensemble_id}:") - for simulation in simulations: + for simulation in item.simulations: selection = ( "selected" if simulation.simulation_id in selected[ensemble.ensemble_id] @@ -608,7 +627,7 @@ def _display_dry_plan( msgr.message(f" drainage: {simulation.artifacts.drainage_output}") if simulation.artifacts.statistics_file is not None: msgr.message(f" statistics: {simulation.artifacts.statistics_file}") - for failure in failures: + for failure in item.failures: status = "not selected" if has_selectors else "validation failed" msgr.warning(f" {status}: {failure.detail}") if ensemble.ensemble_id in manifest_failures: diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 4720acc6..f7fc4461 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -10,10 +10,12 @@ import itzi.messenger as msgr from itzi.cli_parser import build_parser +from itzi.ensemble_models import ResolvedEnsemble from itzi.itzi import ( VerbosityLevel, _preflight_selected, _run_ensemble_batch, + _run_one_ensemble, itzi_run, itzi_run_one, main, @@ -100,9 +102,12 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): ) member_failures, manifest_failures = _preflight_selected( - [(ensemble, (selected_simulation, selected_with_failure, unselected_simulation), ())], + [ + ResolvedEnsemble( + ensemble, (selected_simulation, selected_with_failure, unselected_simulation), () + ) + ], {"study": {"sim-selected", "sim-failed"}}, - {}, has_selectors=True, overwrite=False, ) @@ -114,6 +119,58 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): assert manifest_failures == {} +def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, monkeypatch): + ensemble = SimpleNamespace( + ensemble_id="study", + source=SimpleNamespace(path=tmp_path / "study.yaml"), + manifest_template=None, + ) + artifacts = SimpleNamespace(output_map_names=(), drainage_output=None, statistics_file=None) + simulations = tuple( + SimpleNamespace(simulation_id=name, coordinates=(), artifacts=artifacts) + for name in ("good", "bad", "preflight", "unselected") + ) + runs = [] + updates = [] + monkeypatch.setattr("itzi.itzi._create_manifest", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + "itzi.itzi._update_manifest", + lambda _path, _ensemble, states: updates.append( + {key: value["status"] for key, value in states.items()} + ), + ) + monkeypatch.setattr("itzi.itzi._preflight_resolved_in_spawn", lambda _: None) + monkeypatch.setattr( + "itzi.itzi._run_resolved_in_spawn", + lambda simulation: ( + runs.append(simulation.simulation_id) + or ( + ("execution_failed", "failed") + if simulation.simulation_id == "bad" + else ("completed", None) + ) + ), + ) + + count = _run_one_ensemble( + ResolvedEnsemble(ensemble, simulations, ()), + {"good", "bad", "preflight"}, + {"preflight": {"phase": "preflight", "detail": "invalid input"}}, + None, + has_selectors=True, + overwrite=False, + ) + + assert count == 2 + assert runs == ["good", "bad"] + assert updates[-1] == { + "good": "completed", + "bad": "execution_failed", + "preflight": "validation_failed", + "unselected": "not_selected", + } + + def test_preflight_ensemble_worker_keeps_member_failures(monkeypatch): calls = [] From 5a17d93d2e6d29ca3a86c431f48b053ffde98695 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 19:26:42 -0600 Subject: [PATCH 37/85] display 2 digits for percents --- src/itzi/itzi.py | 242 ++++++++++++++++++++---------------------- src/itzi/messenger.py | 4 +- tests/cli/test_cli.py | 118 +++++++++++++++++--- 3 files changed, 222 insertions(+), 142 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 60a44669..19183506 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -28,6 +28,7 @@ import os import sys import time +from argparse import Namespace from collections.abc import Callable from concurrent.futures import ProcessPoolExecutor from datetime import timedelta @@ -340,15 +341,30 @@ def _run_legacy_batch(cli_args) -> None: msgr.fatal(f"{len(failed_files)} simulation(s) failed") -def _run_ensemble_batch(cli_args) -> None: - """Resolve and execute an ordered YAML/INI batch through spawn boundaries.""" +def _run_ensemble_batch(cli_args: Namespace) -> None: + """Resolve and execute each ensemble in order through spawn boundaries.""" if getattr(cli_args, "resume_from", []): msgr.fatal("Resume is not available for YAML ensembles.") ensembles, document_failures = load_batch(cli_args.config_file) for failure in document_failures: msgr.warning(failure.format()) - resolved: list[ResolvedEnsemble] = [] + selectors = getattr(cli_args, "member", []) + has_selectors = bool(selectors) + if has_selectors and len(ensembles) != 1 and any("#" not in s for s in selectors): + msgr.fatal("Unqualified --member IDs require a batch with exactly one ensemble") + qualified_selectors = [ + (selector, selector if "#" in selector else f"{ensembles[0].ensemble_id}#{selector}") + for selector in selectors + ] + ensemble_ids = {ensemble.ensemble_id for ensemble in ensembles} + for selector, qualified in qualified_selectors: + if qualified.partition("#")[0] not in ensemble_ids: + msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") + + dry = getattr(cli_args, "dry", False) + dry_failed = bool(document_failures) + failure_count = len(document_failures) for ensemble in ensembles: successful: list[ResolvedSimulation] = [] member_failures: list[ValidationFailure] = [] @@ -362,39 +378,43 @@ def _run_ensemble_batch(cli_args) -> None: validate_resolved_ensemble(tuple(successful), (ensemble.source.path,)) except EnsembleError as error: artifact_failure = str(error) - resolved.append( - ResolvedEnsemble(ensemble, tuple(successful), tuple(member_failures), artifact_failure) + resolved = ResolvedEnsemble( + ensemble, tuple(successful), tuple(member_failures), artifact_failure ) - - selectors = getattr(cli_args, "member", []) - has_selectors = bool(selectors) - selected = _select_members(resolved, selectors) - preflight_failures, manifest_failures = _preflight_selected( - resolved, - selected, - has_selectors=has_selectors, - overwrite=bool(getattr(cli_args, "o", False)), - ) - if getattr(cli_args, "dry", False): - _display_dry_plan( - resolved, selected, preflight_failures, manifest_failures, has_selectors=has_selectors + selected_ids = _select_members(resolved, qualified_selectors) + preflight_failures, manifest_failure = _preflight_ensemble( + resolved, + selected_ids, + has_selectors=has_selectors, + overwrite=bool(getattr(cli_args, "o", False)), ) - unresolved_selected = not has_selectors and any(item.failures for item in resolved) - if document_failures or unresolved_selected or preflight_failures or manifest_failures: - msgr.fatal("YAML batch validation failed") - return + if dry: + _display_dry_plan( + resolved, + selected_ids, + preflight_failures, + manifest_failure, + has_selectors=has_selectors, + ) + dry_failed |= bool( + (resolved.failures and not has_selectors) or preflight_failures or manifest_failure + ) + continue - failure_count = len(document_failures) - for item in resolved: failure_count += _run_one_ensemble( - item, - selected.get(item.ensemble.ensemble_id, set()), - preflight_failures.get(item.ensemble.ensemble_id, {}), - manifest_failures.get(item.ensemble.ensemble_id), + resolved, + selected_ids, + preflight_failures, + manifest_failure, has_selectors=has_selectors, overwrite=bool(cli_args.o), ) + if dry: + if dry_failed: + msgr.fatal("YAML batch validation failed") + return + if failure_count: msgr.fatal(f"{failure_count} ensemble simulation(s) failed validation or execution") msgr.message("Simulation(s) complete.") @@ -519,121 +539,89 @@ def _preflight_ensemble_in_spawn(simulations: tuple[ResolvedSimulation, ...]) -> return {simulation.simulation_id: detail for simulation in simulations} -def _preflight_selected( - resolved: list[ResolvedEnsemble], - selected: dict[str, set[str]], +def _preflight_ensemble( + resolved: ResolvedEnsemble, + selected_ids: set[str], *, has_selectors: bool, overwrite: bool, -) -> tuple[dict[str, dict[str, dict[str, str]]], dict[str, str]]: - """Collect selected-member and manifest failures before creating artifacts.""" - member_failures: dict[str, dict[str, dict[str, str]]] = {} - manifest_failures: dict[str, str] = {} - for item in resolved: - ensemble = item.ensemble - simulations = item.simulations - selected_ids = selected[ensemble.ensemble_id] - if not selected_ids and has_selectors: - continue - failures = member_failures.setdefault(ensemble.ensemble_id, {}) - artifact_detail = item.artifact_failure - if artifact_detail is not None: - for simulation_id in selected_ids: - failures[simulation_id] = { - "phase": "artifact_validation", - "detail": artifact_detail, - } - try: - _validate_manifest_destination( - _manifest_path(ensemble), - ensemble, - simulations, - overwrite=overwrite, - ) - except EnsembleError as error: - manifest_failures[ensemble.ensemble_id] = str(error) - to_preflight = tuple( - simulation - for simulation in simulations - if simulation.simulation_id in selected_ids - and simulation.simulation_id not in failures +) -> tuple[dict[str, dict[str, str]], str | None]: + """Collect one ensemble's selected-member and manifest failures.""" + if not selected_ids and has_selectors: + return {}, None + ensemble = resolved.ensemble + simulations = resolved.simulations + failures: dict[str, dict[str, str]] = {} + if resolved.artifact_failure is not None: + for simulation_id in selected_ids: + failures[simulation_id] = { + "phase": "artifact_validation", + "detail": resolved.artifact_failure, + } + manifest_failure = None + try: + _validate_manifest_destination( + _manifest_path(ensemble), ensemble, simulations, overwrite=overwrite ) - if to_preflight: - for simulation_id, detail in _preflight_ensemble_in_spawn(to_preflight).items(): - failures[simulation_id] = { - "phase": "preflight", - "detail": detail, - } - if not failures: - member_failures.pop(ensemble.ensemble_id) - return member_failures, manifest_failures + except EnsembleError as error: + manifest_failure = str(error) + to_preflight = tuple( + simulation + for simulation in simulations + if simulation.simulation_id in selected_ids and simulation.simulation_id not in failures + ) + if to_preflight: + for simulation_id, detail in _preflight_ensemble_in_spawn(to_preflight).items(): + failures[simulation_id] = {"phase": "preflight", "detail": detail} + return failures, manifest_failure def _select_members( - resolved: list[ResolvedEnsemble], - selectors: list[str], -) -> dict[str, set[str]]: - """Resolve member selectors only against successfully resolved members.""" - selected = {item.ensemble.ensemble_id: set() for item in resolved} - all_members = { - f"{item.ensemble.ensemble_id}#{simulation.simulation_id}": simulation.simulation_id - for item in resolved - for simulation in item.simulations - } + resolved: ResolvedEnsemble, + selectors: list[tuple[str, str]], +) -> set[str]: + """Select successfully resolved members of one ensemble.""" + all_members = {simulation.simulation_id for simulation in resolved.simulations} if not selectors: - for item in resolved: - selected[item.ensemble.ensemble_id] = { - simulation.simulation_id for simulation in item.simulations - } - return selected - if any("#" not in selector for selector in selectors) and len(resolved) != 1: - msgr.fatal("Unqualified --member IDs require a batch with exactly one ensemble") - for selector in selectors: - qualified = ( - selector if "#" in selector else f"{resolved[0].ensemble.ensemble_id}#{selector}" - ) - simulation_id = all_members.get(qualified) - if simulation_id is None: + return all_members + selected: set[str] = set() + for selector, qualified in selectors: + ensemble_id, _, simulation_id = qualified.partition("#") + if ensemble_id != resolved.ensemble.ensemble_id: + continue + if simulation_id not in all_members: msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") - ensemble_id, _separator, _member = qualified.partition("#") - selected[ensemble_id].add(simulation_id) + selected.add(simulation_id) return selected def _display_dry_plan( - resolved: list[ResolvedEnsemble], - selected: dict[str, set[str]], - preflight_failures: dict[str, dict[str, dict[str, str]]], - manifest_failures: dict[str, str], + resolved: ResolvedEnsemble, + selected_ids: set[str], + preflight_failures: dict[str, dict[str, str]], + manifest_failure: str | None, *, has_selectors: bool, ) -> None: - for item in resolved: - ensemble = item.ensemble - msgr.message(f"Ensemble {ensemble.ensemble_id}:") - for simulation in item.simulations: - selection = ( - "selected" - if simulation.simulation_id in selected[ensemble.ensemble_id] - else "not selected" - ) - msgr.message(f" {simulation.simulation_id} ({selection})") - if simulation.simulation_id in preflight_failures.get(ensemble.ensemble_id, {}): - failure = preflight_failures[ensemble.ensemble_id][simulation.simulation_id] - msgr.warning(f" {failure['phase']} failed: {failure['detail']}") - for variable, name in simulation.artifacts.output_map_names: - msgr.message(f" raster {variable}: {name}") - if simulation.artifacts.drainage_output is not None: - msgr.message(f" drainage: {simulation.artifacts.drainage_output}") - if simulation.artifacts.statistics_file is not None: - msgr.message(f" statistics: {simulation.artifacts.statistics_file}") - for failure in item.failures: - status = "not selected" if has_selectors else "validation failed" - msgr.warning(f" {status}: {failure.detail}") - if ensemble.ensemble_id in manifest_failures: - msgr.warning( - f" manifest validation failed: {manifest_failures[ensemble.ensemble_id]}" - ) + ensemble = resolved.ensemble + msgr.message(f"Ensemble {ensemble.ensemble_id}:") + for simulation in resolved.simulations: + selection = "selected" if simulation.simulation_id in selected_ids else "not selected" + msgr.message(f" {simulation.simulation_id} ({selection})") + if simulation.simulation_id in preflight_failures: + failure = preflight_failures[simulation.simulation_id] + msgr.warning(f" {failure['phase']} failed: {failure['detail']}") + for variable, name in simulation.artifacts.output_map_names: + msgr.message(f" raster {variable}: {name}") + if simulation.artifacts.drainage_output is not None: + msgr.message(f" drainage: {simulation.artifacts.drainage_output}") + if simulation.artifacts.statistics_file is not None: + msgr.message(f" statistics: {simulation.artifacts.statistics_file}") + for failure in resolved.failures: + status = "not selected" if has_selectors else "validation failed" + msgr.warning(f" {status}: {failure.detail}") + if manifest_failure is not None: + msgr.warning(f" manifest validation failed: {manifest_failure}") def itzi_version(cli_args): diff --git a/src/itzi/messenger.py b/src/itzi/messenger.py index 92785f20..e482cfac 100644 --- a/src/itzi/messenger.py +++ b/src/itzi/messenger.py @@ -140,7 +140,7 @@ def percent(start_time, end_time, sim_time, sim_start_time): advance_perc = sim_time_s / duration_s if verbosity() == VerbosityLevel.QUIET: - print(f"{advance_perc:.1%}", file=sys.stderr, end="\r") + print(f"{advance_perc:.2%}", file=sys.stderr, end="\r") elif verbosity() >= VerbosityLevel.MESSAGE: elapsed_s = (datetime.now() - sim_start_time).total_seconds() @@ -150,7 +150,7 @@ def percent(start_time, end_time, sim_time, sim_start_time): rate = 0 remaining = (end_time - sim_time).total_seconds() eta = timedelta(seconds=int(remaining * rate)) - txt = "Time: {sim} Advance: {perc:.1%} ETA: {eta}{pad}" + txt = "Time: {sim} Advance: {perc:.2%} ETA: {eta}{pad}" disp = txt.format( sim=sim_time.isoformat(" ").split(".")[0], perc=advance_perc, diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index f7fc4461..e112efd0 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -10,12 +10,13 @@ import itzi.messenger as msgr from itzi.cli_parser import build_parser -from itzi.ensemble_models import ResolvedEnsemble +from itzi.ensemble_models import ResolvedEnsemble, ResolvedSimulation from itzi.itzi import ( VerbosityLevel, - _preflight_selected, + _preflight_ensemble, _run_ensemble_batch, _run_one_ensemble, + _select_members, itzi_run, itzi_run_one, main, @@ -82,6 +83,83 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): assert calls == [simulations] +@pytest.mark.parametrize("dry", [False, True]) +def test_ensembles_are_checked_and_run_in_order_without_cross_ensemble_validation( + tmp_path, monkeypatch, dry +): + ensembles = tuple( + SimpleNamespace( + ensemble_id=name, + simulations=(SimpleNamespace(simulation_id=name),), + source=SimpleNamespace(path=tmp_path / f"{name}.yaml"), + manifest_template=None, + ) + for name in ("first", "second") + ) + members = { + name: ResolvedSimulation( + simulation_id=name, + coordinates=(), + grass_params=None, + domain_data=None, + effective_mask=None, + input_kinds=(), + simulation_config=SimpleNamespace( + input_map_names={"friction": "first_water_depth@PERMANENT"} + if name == "second" + else {} + ), + artifacts=SimpleNamespace( + output_map_names=(("water_depth", "first_water_depth"),) + if name == "first" + else (), + ), + normalized_payload=name, + ) + for name in ("first", "second") + } + calls = [] + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: (ensembles, ())) + + def resolve(expanded): + name = expanded[0].simulation_id + calls.append(("resolve", name)) + return (members[name],) + + def validate(simulations, _sources): + calls.append(("validate", simulations[0].simulation_id)) + assert len(simulations) == 1 + + def preflight(simulations): + calls.append(("preflight", simulations[0].simulation_id)) + return {} + + def run(resolved, selected_ids, *_args, **_kwargs): + calls.append(("run", resolved.ensemble.ensemble_id)) + assert selected_ids == {resolved.ensemble.ensemble_id} + return 0 + + def display(resolved, *_args, **_kwargs): + calls.append(("display", resolved.ensemble.ensemble_id)) + + monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_spawn", resolve) + monkeypatch.setattr("itzi.itzi.validate_resolved_ensemble", validate) + monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_, **__: None) + monkeypatch.setattr("itzi.itzi._preflight_ensemble_in_spawn", preflight) + monkeypatch.setattr("itzi.itzi._run_one_ensemble", run) + monkeypatch.setattr("itzi.itzi._display_dry_plan", display) + + _run_ensemble_batch( + SimpleNamespace(config_file=["first.yaml", "second.yaml"], member=[], dry=dry, o=True) + ) + + assert calls == [ + (step, name) + for name in ("first", "second") + for step in ("resolve", "validate", "preflight", "display" if dry else "run") + ] + + def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): selected_simulation = SimpleNamespace(simulation_id="sim-selected") selected_with_failure = SimpleNamespace(simulation_id="sim-failed") @@ -101,22 +179,36 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): ), ) - member_failures, manifest_failures = _preflight_selected( - [ - ResolvedEnsemble( - ensemble, (selected_simulation, selected_with_failure, unselected_simulation), () - ) - ], - {"study": {"sim-selected", "sim-failed"}}, + member_failures, manifest_failure = _preflight_ensemble( + ResolvedEnsemble( + ensemble, (selected_simulation, selected_with_failure, unselected_simulation), () + ), + {"sim-selected", "sim-failed"}, has_selectors=True, overwrite=False, ) assert calls == [("sim-selected", "sim-failed")] - assert member_failures == { - "study": {"sim-failed": {"phase": "preflight", "detail": "invalid input"}} - } - assert manifest_failures == {} + assert member_failures == {"sim-failed": {"phase": "preflight", "detail": "invalid input"}} + assert manifest_failure is None + + +def test_select_members_only_matches_current_ensemble(): + resolved = ResolvedEnsemble( + SimpleNamespace(ensemble_id="first"), (SimpleNamespace(simulation_id="member"),), () + ) + assert _select_members(resolved, [("second#member", "second#member")]) == set() + assert _select_members(resolved, [("first#member", "first#member")]) == {"member"} + + +def test_unqualified_selector_requires_one_ensemble(monkeypatch): + ensembles = tuple(SimpleNamespace(ensemble_id=name) for name in ("first", "second")) + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: (ensembles, ())) + + with pytest.raises(RuntimeError, match="Unqualified --member IDs"): + _run_ensemble_batch( + SimpleNamespace(config_file=["first.yaml", "second.yaml"], member=["member"]) + ) def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, monkeypatch): From 8b409b5e1442ee4e4c915a1a680502a156ccd029 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 19:29:04 -0600 Subject: [PATCH 38/85] rename functions --- src/itzi/itzi.py | 18 ++++++++++-------- src/itzi/simulation_runner.py | 2 +- tests/cli/test_cli.py | 12 ++++++------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 19183506..6e4c48ab 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -368,7 +368,7 @@ def _run_ensemble_batch(cli_args: Namespace) -> None: for ensemble in ensembles: successful: list[ResolvedSimulation] = [] member_failures: list[ValidationFailure] = [] - for result in _resolve_ensemble_in_spawn(ensemble.simulations): + for result in _resolve_ensemble_in_subprocess(ensemble.simulations): if isinstance(result, ResolvedSimulation): successful.append(result) else: @@ -455,7 +455,7 @@ def _run_one_ensemble( continue if states[simulation.simulation_id]["status"] != "planned": continue - detail = _preflight_resolved_in_spawn(simulation) + detail = _preflight_simulation_in_subprocess(simulation) if detail is not None: states[simulation.simulation_id]["status"] = "validation_failed" states[simulation.simulation_id]["failure"] = { @@ -472,7 +472,7 @@ def _run_one_ensemble( failure_count += 1 break started = time.monotonic() - status, detail = _run_resolved_in_spawn(simulation) + status, detail = _run_simulation_in_subprocess(simulation) state = states[simulation.simulation_id] state["status"] = status state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") @@ -496,7 +496,7 @@ def _run_one_ensemble( return failure_count -def _resolve_ensemble_in_spawn( +def _resolve_ensemble_in_subprocess( expanded: tuple[ExpandedSimulation, ...], ) -> tuple[ResolvedSimulation | ValidationFailure, ...]: try: @@ -514,7 +514,7 @@ def _resolve_ensemble_in_spawn( ) -def _run_resolved_in_spawn(simulation: ResolvedSimulation) -> tuple[str, str | None]: +def _run_simulation_in_subprocess(simulation: ResolvedSimulation) -> tuple[str, str | None]: try: with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: return executor.submit(resolved_sim_runner_worker, simulation).result() @@ -522,7 +522,7 @@ def _run_resolved_in_spawn(simulation: ResolvedSimulation) -> tuple[str, str | N return "execution_failed", f"{type(error).__name__}: {error}" -def _preflight_resolved_in_spawn(simulation: ResolvedSimulation) -> str | None: +def _preflight_simulation_in_subprocess(simulation: ResolvedSimulation) -> str | None: try: with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: return executor.submit(preflight_worker, simulation).result() @@ -530,7 +530,9 @@ def _preflight_resolved_in_spawn(simulation: ResolvedSimulation) -> str | None: return f"{type(error).__name__}: {error}" -def _preflight_ensemble_in_spawn(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: +def _preflight_ensemble_in_subprocess( + simulations: tuple[ResolvedSimulation, ...], +) -> dict[str, str]: try: with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: return executor.submit(preflight_ensemble_worker, simulations).result() @@ -571,7 +573,7 @@ def _preflight_ensemble( if simulation.simulation_id in selected_ids and simulation.simulation_id not in failures ) if to_preflight: - for simulation_id, detail in _preflight_ensemble_in_spawn(to_preflight).items(): + for simulation_id, detail in _preflight_ensemble_in_subprocess(to_preflight).items(): failures[simulation_id] = {"phase": "preflight", "detail": detail} return failures, manifest_failure diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 880f0963..439da544 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -143,7 +143,7 @@ def run(self): while self.sim.sim_time < self.sim.end_time: # display advance of simulation now = monotonic() - if now - last_progress_update >= 0.5: + if now - last_progress_update >= 0.2: msgr.percent( self.sim.start_time, self.sim.end_time, diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index e112efd0..032a3548 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -71,7 +71,7 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): monkeypatch.setattr("itzi.itzi.load_batch", lambda _: ((ensemble,), ())) monkeypatch.setattr( - "itzi.itzi._resolve_ensemble_in_spawn", + "itzi.itzi._resolve_ensemble_in_subprocess", lambda received: calls.append(received) or (), ) monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_, **__: None) @@ -142,10 +142,10 @@ def run(resolved, selected_ids, *_args, **_kwargs): def display(resolved, *_args, **_kwargs): calls.append(("display", resolved.ensemble.ensemble_id)) - monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_spawn", resolve) + monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_subprocess", resolve) monkeypatch.setattr("itzi.itzi.validate_resolved_ensemble", validate) monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_, **__: None) - monkeypatch.setattr("itzi.itzi._preflight_ensemble_in_spawn", preflight) + monkeypatch.setattr("itzi.itzi._preflight_ensemble_in_subprocess", preflight) monkeypatch.setattr("itzi.itzi._run_one_ensemble", run) monkeypatch.setattr("itzi.itzi._display_dry_plan", display) @@ -172,7 +172,7 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): calls = [] monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) monkeypatch.setattr( - "itzi.itzi._preflight_ensemble_in_spawn", + "itzi.itzi._preflight_ensemble_in_subprocess", lambda simulations: ( calls.append(tuple(s.simulation_id for s in simulations)) or {"sim-failed": "invalid input"} @@ -231,9 +231,9 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo {key: value["status"] for key, value in states.items()} ), ) - monkeypatch.setattr("itzi.itzi._preflight_resolved_in_spawn", lambda _: None) + monkeypatch.setattr("itzi.itzi._preflight_simulation_in_subprocess", lambda _: None) monkeypatch.setattr( - "itzi.itzi._run_resolved_in_spawn", + "itzi.itzi._run_simulation_in_subprocess", lambda simulation: ( runs.append(simulation.simulation_id) or ( From 3eebcf8cfec089281b2fc9c6441e3406857e4da1 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 21:54:40 -0600 Subject: [PATCH 39/85] add integration tests --- .github/workflows/tests.yml | 11 +- src/itzi/bmi_itzi.py | 2 +- tests/grass/conftest.py | 19 +- tests/grass/test_ensemble.py | 249 ++++++++++++++++++++++++- tests/grass/test_temporal_overwrite.py | 10 - 5 files changed, 265 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 30f6040f..aeb792f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,11 +31,12 @@ jobs: - name: Pytest run: | - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-cli-results.xml --cov-report=xml --cov-report=html tests/cli tests/test_grass_session.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-itzi-results.xml --cov-report=xml --cov-report=html tests/grass/test_itzi.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-bmi-results.xml --cov-report=xml --cov-report=html tests/grass/test_bmi.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-tutorial-results.xml --cov-report=xml --cov-report=html tests/grass/test_tutorial.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-temporal-overwrite-results.xml --cov-report=xml --cov-report=html tests/grass/test_temporal_overwrite.py + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --cov-report= --junitxml=junit/test-cli-results.xml tests/cli tests/test_grass_session.py + for test_file in tests/grass/test_*.py; do + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --cov-append --cov-report= --junitxml="junit/$(basename "$test_file" .py)-results.xml" "$test_file" + done + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} coverage xml + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} coverage html - name: Upload coverage uses: actions/upload-artifact@v7 diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 61982ced..292de5d7 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -46,7 +46,7 @@ class BmiItzi(Bmi): } ) # A list of array definition for both input and output, without overlaps - input_output_array_definition = ( + input_output_array_definition = tuple( arr_def for arr_def in ARRAY_DEFINITIONS if any(cat in arr_def.category for cat in [ArrayCategory.OUTPUT, ArrayCategory.INPUT]) diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index 57b1df68..fec00470 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -3,10 +3,11 @@ """ import os +import subprocess import sys import tempfile +from collections.abc import Iterator from pathlib import Path -import subprocess import pytest @@ -17,10 +18,26 @@ sys.path.append(grass_python_path) import grass.script as gscript # noqa: E402 +gscript.setup.setup_runtime_env() + from itzi import SimulationRunner # noqa: E402 +from itzi import grass_session # noqa: E402 from itzi.configreader import ConfigReader # noqa: E402 TESTS_ROOT = Path.cwd() +TEST_PROCESS = os.getpid() + + +@pytest.fixture(autouse=True) +def stop_temporal_subprocesses_after_test() -> Iterator[None]: + """Forked tests must stop their own GRASS RPC children before os._exit.""" + yield + if os.getpid() != TEST_PROCESS and grass_session._initialized_temporal_session is not None: + session_pid = grass_session._initialized_temporal_session[0] + if session_pid == os.getpid(): + import grass.temporal as tgis + + tgis.stop_subprocesses() @pytest.fixture(scope="session") diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 949b6d57..31b84691 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -4,7 +4,6 @@ import os import sqlite3 -from collections.abc import Iterator from contextlib import closing from datetime import datetime, timedelta from pathlib import Path @@ -12,23 +11,255 @@ from uuid import uuid4 import grass.script as gscript -import grass.temporal as tgis +import numpy as np import pytest import yaml from grass.pygrass import utils as gutils from itzi_core import TemporalType from itzi.cli_parser import build_parser -from itzi.ensemble_models import EnsembleError +from itzi.ensemble_models import EnsembleError, ResolvedSimulation, ValidationFailure from itzi.grass_session import GrassSessionManager -from itzi.itzi import itzi_run +from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, preflight_worker from itzi.preflight import _validate_grass_outputs +from itzi.providers.grass_interface import GrassInterface +from itzi.run_plan import load_batch + + +def write_study(tmp_path: Path, **changes: dict) -> Path: + """Write a minimal runnable ensemble with optional nested YAML overrides.""" + document = { + "schema_version": 1, + "ensemble": {"id": "stage-one"}, + "domain": {}, + "time": {"duration": "00:01:00", "record_step": "00:00:30"}, + "input": {"ground_elevation": "z", "friction": "n"}, + "options": {"dtmax": 0.3}, + "outputs": {}, + } + for section, fields in changes.items(): + document[section].update(fields) + path = tmp_path / "study.yaml" + path.write_text(yaml.safe_dump(document), encoding="utf-8") + return path + + +def read_manifest(tmp_path: Path) -> dict: + return yaml.safe_load( + (tmp_path / "results" / "stage-one.manifest.yaml").read_text(encoding="utf-8") + ) -@pytest.fixture -def stop_temporal_subprocesses() -> Iterator[None]: - yield - tgis.stop_subprocesses() +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( + tmp_path: Path, +) -> None: + GrassSessionManager.ensure_temporal_initialized() + mapset = gutils.getenv("MAPSET") + gscript.mapcalc("stage_rain=1") + gscript.run_command( + "t.create", + output="stage_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="stage_series", + description="stage_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="stage_series", + type="raster", + maps="stage_rain", + start="0", + increment="60", + unit="seconds", + ) + path = write_study( + tmp_path, + input={"ground_elevation": f"z@{mapset}", "rainfall_rate": "stage_series"}, + ) + expanded = load_batch([str(path)])[0][0].simulations + resolved = _resolve_ensemble_in_subprocess(expanded)[0] + assert isinstance(resolved, ResolvedSimulation) + assert resolved.simulation_config.input_map_names == { + "ground_elevation": f"z@{mapset}", + "friction": f"n@{mapset}", + "rainfall_rate": f"stage_series@{mapset}", + } + assert dict(resolved.input_kinds)["rainfall_rate"] == "strds" + assert preflight_worker(resolved) is None + + path = write_study(tmp_path, input={"rainfall_rate": f"stage_series@{mapset}"}) + qualified = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(qualified, ResolvedSimulation) + assert qualified.simulation_id == resolved.simulation_id + + gscript.mapcalc("stage_series=1") + ambiguous = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(ambiguous, ValidationFailure) + assert f"raster stage_series@{mapset}" in ambiguous.detail + assert f"STRDS stage_series@{mapset}" in ambiguous.detail + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> None: + GrassSessionManager.ensure_temporal_initialized() + gscript.mapcalc("stage_short_rain=1") + gscript.run_command( + "t.create", + output="stage_short_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="stage_short_series", + description="stage_short_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="stage_short_series", + type="raster", + maps="stage_short_rain", + start="0", + increment="30", + unit="seconds", + ) + path = write_study( + tmp_path, + input={"water_depth": "start_h", "rainfall_rate": ["stage_short_series", "rainfall"]}, + outputs={"rasters": {"prefix": "stage_{simulation}", "variables": ["water_depth"]}}, + ) + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(path)])) + + members = read_manifest(tmp_path)["members"] + assert sorted(member["status"] for member in members) == ["completed", "validation_failed"] + failed = next(member for member in members if member["status"] == "validation_failed") + completed = next(member for member in members if member["status"] == "completed") + assert failed["failure"]["phase"] == "preflight" + assert ( + "ends before simulation" in failed["failure"]["detail"] + or "inadequate temporal" in failed["failure"]["detail"] + ) + output = completed["artifacts"]["rasters"]["water_depth"] + assert "@" not in output + assert gutils.get_mapset_raster(f"{output}_0000", gutils.getenv("MAPSET")) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path) -> None: + gscript.mapcalc("stage_mask_values=if(col()==1, null(), if(col()==2, 0, 1))") + gscript.run_command("g.copy", raster="stage_mask_values,MASK") + try: + path = write_study(tmp_path, domain={"mask": "stage_mask_values"}) + expanded = load_batch([str(path)])[0][0].simulations + resolved = _resolve_ensemble_in_subprocess(expanded)[0] + assert isinstance(resolved, ResolvedSimulation) + assert resolved.effective_mask.mode == "explicit" + + config = resolved.simulation_config + interface = GrassInterface( + config.start_time, + config.end_time, + np.float32, + None, + "stage_mask_values", + ("explicit", resolved.effective_mask.source), + non_blocking_write=False, + ) + try: + explicit = interface.get_npmask() + assert explicit[:, 0].all() + assert not explicit[:, 1:].any() # zero is inside an explicit mask + assert np.isfinite(interface.read_raster_map("n")).all() # bypass active MASK + interface.mask_mode, interface.mask_source = ( + "active", + f"MASK@{gutils.getenv('MAPSET')}", + ) + active = interface.get_npmask() + assert active[:, :2].all() # zero is outside an active CELL mask + assert not active[:, 2:].any() + finally: + interface.cleanup() + + assert preflight_worker(resolved) is None + gscript.run_command("r.mask", flags="r") + assert preflight_worker(resolved) is None + + active_path = write_study(tmp_path, domain={}) + active_resolved = _resolve_ensemble_in_subprocess( + load_batch([str(active_path)])[0][0].simulations + )[0] + assert isinstance(active_resolved, ResolvedSimulation) + assert active_resolved.effective_mask.mode == "none" + gscript.run_command("g.copy", raster="stage_mask_values,MASK") + assert "effective GRASS mask changed" in (preflight_worker(active_resolved) or "") + assert gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET")) + finally: + gscript.run_command("r.mask", flags="r") + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_worker_rejects_region_and_input_changes_after_resolution(tmp_path: Path) -> None: + gscript.run_command("g.copy", raster="n,stage_worker_friction") + path = write_study(tmp_path, input={"friction": "stage_worker_friction"}) + resolved = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(resolved, ResolvedSimulation) + + gscript.run_command("g.region", n=60) + assert "GRASS domain changed" in (preflight_worker(resolved) or "") + gscript.run_command("g.region", n=50) + gscript.run_command("g.remove", flags="f", type="raster", name="stage_worker_friction") + assert "stage_worker_friction" in (preflight_worker(resolved) or "") + assert not (tmp_path / "results").exists() + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_rejects_raster_outside_domain_without_writing(tmp_path: Path) -> None: + gscript.use_temp_region() + try: + gscript.run_command("g.region", n=150, s=100, e=150, w=100, res=10) + gscript.mapcalc("stage_outside=1") + finally: + gscript.del_temp_region() + path = write_study( + tmp_path, + input={"friction": "stage_outside"}, + outputs={ + "rasters": {"prefix": "stage_spatial", "variables": ["water_depth"]}, + "statistics": {"file": "stats/results.csv"}, + }, + ) + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(path), "--dry-run"])) + assert not (tmp_path / "results").exists() + assert not (tmp_path / "stats").exists() + assert not gutils.get_mapset_raster("stage_spatial_water_depth_0000", gutils.getenv("MAPSET")) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_ensemble_output_collision_with_input_rejects_all_members(tmp_path: Path) -> None: + gscript.mapcalc("stage_alias_water_depth_0002=1") + path = write_study( + tmp_path, + input={"rainfall_rate": ["rainfall", "stage_alias_water_depth_0002"]}, + outputs={"rasters": {"prefix": "stage_alias", "variables": ["water_depth"]}}, + ) + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(path), "-o"])) + members = read_manifest(tmp_path)["members"] + assert len(members) == 2 + assert all(member["status"] == "validation_failed" for member in members) + assert all(member["failure"]["phase"] == "artifact_validation" for member in members) + assert not gutils.get_mapset_raster("stage_alias_water_depth_0000", gutils.getenv("MAPSET")) @pytest.mark.forked @@ -313,7 +544,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked -@pytest.mark.usefixtures("grass_5by5", "stop_temporal_subprocesses") +@pytest.mark.usefixtures("grass_5by5") def test_preflight_checks_existing_drainage_tables(): GrassSessionManager.ensure_temporal_initialized() mapset = gutils.getenv("MAPSET") diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index f88bc144..e09eed33 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Iterator from configparser import ConfigParser import os from uuid import uuid4 @@ -14,15 +13,6 @@ from itzi_core.const import TemporalType -@pytest.fixture(autouse=True) -def stop_temporal_subprocesses() -> Iterator[None]: - yield - - import grass.temporal as tgis - - tgis.stop_subprocesses() - - def _build_runner( test_data_temp_path: str, prefix: str, From 042fd1039c1b4fe13096e84740c543aa93b76b8a Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 22:31:26 -0600 Subject: [PATCH 40/85] add tests for python 3.14 and 3.15 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aeb792f2..bc871db3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: test: strategy: matrix: - python-version: ["3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14", "3.15"] runs-on: ubuntu-latest container: osgeo/grass-gis:releasebranch_8_4-ubuntu From 2fa8c73875f02234d0bb4c9a37b49884737827e3 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 22:31:57 -0600 Subject: [PATCH 41/85] finalize bmi object in the tests --- tests/grass/test_bmi.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/grass/test_bmi.py b/tests/grass/test_bmi.py index f194d58f..ab896d24 100644 --- a/tests/grass/test_bmi.py +++ b/tests/grass/test_bmi.py @@ -1,6 +1,7 @@ """Test the Basic Model Interface implementation.""" import os +from collections.abc import Iterator import numpy as np import pytest @@ -16,6 +17,12 @@ def bmi_object(grass_5by5, test_data_path): return itzi_bmi +@pytest.fixture(scope="class") +def finalize_bmi_object(bmi_object: BmiItzi) -> Iterator[None]: + yield + bmi_object.finalize() + + @pytest.mark.forked class TestBmiMutating: """Those functions mutate the BMI model state. @@ -53,6 +60,7 @@ def test_set_value(self, bmi_object): assert np.all(value == bmi_object.get_value_ptr(var_name)) +@pytest.mark.usefixtures("finalize_bmi_object") class TestBmi: """Test non-mutating functions. No forking needed.""" From fae5d7275c278b000c43cec2a9f7d6a4c47b1d0d Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 23:04:59 -0600 Subject: [PATCH 42/85] fix test tutorial --- tests/grass/test_tutorial.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/grass/test_tutorial.py b/tests/grass/test_tutorial.py index 7657e613..cb279fd4 100644 --- a/tests/grass/test_tutorial.py +++ b/tests/grass/test_tutorial.py @@ -5,6 +5,7 @@ import tempfile from configparser import ConfigParser from io import StringIO +from unittest.mock import patch import grass.script as gscript import numpy as np @@ -30,7 +31,13 @@ def grass_tutorial_session(test_data_temp_path): # Keep all generated files in the test_data_temp_path os.chdir(test_data_temp_path) tmpdir = tempfile.TemporaryDirectory() - gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) + # GRASS 8.4 checks the process GISRC while creating a projected project. + gisrc, _ = gscript.create_environment(tmpdir.name, "itzi_tutorial", "PERMANENT") + try: + with patch.dict(os.environ, {"GISRC": gisrc}): + gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) + finally: + pathlib.Path(gisrc).unlink() grass_session = gscript.setup.init( path=tmpdir.name, location="itzi_tutorial", mapset="PERMANENT" ) From 76ee4a86703db3a7f641b6e373b9d9f00c3b0f32 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Mon, 28 Sep 2026 23:49:51 -0600 Subject: [PATCH 43/85] fix the init of ensemble and input tests --- tests/grass/conftest.py | 2 -- tests/grass/test_ensemble.py | 19 ++++++++++++++++--- tests/grass/test_grass_input.py | 20 +++++++++++++++++--- tests/grass/test_tutorial.py | 9 +-------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index fec00470..d8ea31aa 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -18,8 +18,6 @@ sys.path.append(grass_python_path) import grass.script as gscript # noqa: E402 -gscript.setup.setup_runtime_env() - from itzi import SimulationRunner # noqa: E402 from itzi import grass_session # noqa: E402 from itzi.configreader import ConfigReader # noqa: E402 diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 31b84691..2f612742 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -14,15 +14,12 @@ import numpy as np import pytest import yaml -from grass.pygrass import utils as gutils from itzi_core import TemporalType from itzi.cli_parser import build_parser from itzi.ensemble_models import EnsembleError, ResolvedSimulation, ValidationFailure from itzi.grass_session import GrassSessionManager from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, preflight_worker -from itzi.preflight import _validate_grass_outputs -from itzi.providers.grass_interface import GrassInterface from itzi.run_plan import load_batch @@ -55,6 +52,8 @@ def read_manifest(tmp_path: Path) -> dict: def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( tmp_path: Path, ) -> None: + from grass.pygrass import utils as gutils + GrassSessionManager.ensure_temporal_initialized() mapset = gutils.getenv("MAPSET") gscript.mapcalc("stage_rain=1") @@ -107,6 +106,8 @@ def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + GrassSessionManager.ensure_temporal_initialized() gscript.mapcalc("stage_short_rain=1") gscript.run_command( @@ -153,6 +154,10 @@ def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + + from itzi.providers.grass_interface import GrassInterface + gscript.mapcalc("stage_mask_values=if(col()==1, null(), if(col()==2, 0, 1))") gscript.run_command("g.copy", raster="stage_mask_values,MASK") try: @@ -223,6 +228,8 @@ def test_worker_rejects_region_and_input_changes_after_resolution(tmp_path: Path @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_dry_run_rejects_raster_outside_domain_without_writing(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + gscript.use_temp_region() try: gscript.run_command("g.region", n=150, s=100, e=150, w=100, res=10) @@ -247,6 +254,8 @@ def test_dry_run_rejects_raster_outside_domain_without_writing(tmp_path: Path) - @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_ensemble_output_collision_with_input_rejects_all_members(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + gscript.mapcalc("stage_alias_water_depth_0002=1") path = write_study( tmp_path, @@ -546,6 +555,10 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_preflight_checks_existing_drainage_tables(): + from grass.pygrass import utils as gutils + + from itzi.preflight import _validate_grass_outputs + GrassSessionManager.ensure_temporal_initialized() mapset = gutils.getenv("MAPSET") database = ( diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index 6c2a9ec1..4567debc 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -1,10 +1,19 @@ +from __future__ import annotations + from datetime import datetime, timedelta -from typing import Literal +from typing import TYPE_CHECKING, Literal import pytest -from itzi.providers.grass_input import GrassRasterInputProvider -from itzi.providers.grass_interface import MapData +if TYPE_CHECKING: + from itzi.providers.grass_interface import MapData + + +@pytest.fixture(scope="module", autouse=True) +def grass_runtime_env() -> None: + import grass.script as gscript + + gscript.setup.setup_runtime_env() class FakeGrassInterface: @@ -28,6 +37,8 @@ def stds_temporal_sanity(self, strds_id: str) -> bool: return True def raster_list_from_strds(self, strds_id: str) -> list[MapData]: + from itzi.providers.grass_interface import MapData + self.stds_temporal_sanity(strds_id) start = datetime(2020, 1, 1) return [MapData(strds_id, start, start + timedelta(hours=1))] @@ -39,6 +50,9 @@ def raster_list_from_strds(self, strds_id: str) -> list[MapData]: def test_input_kinds_share_raster_and_strds_construction( input_kinds: dict[str, Literal["raster", "strds"]] | None, ) -> None: + from itzi.providers.grass_input import GrassRasterInputProvider + from itzi.providers.grass_interface import MapData + start = datetime(2020, 1, 1) end = start + timedelta(hours=1) interface = FakeGrassInterface() diff --git a/tests/grass/test_tutorial.py b/tests/grass/test_tutorial.py index cb279fd4..7657e613 100644 --- a/tests/grass/test_tutorial.py +++ b/tests/grass/test_tutorial.py @@ -5,7 +5,6 @@ import tempfile from configparser import ConfigParser from io import StringIO -from unittest.mock import patch import grass.script as gscript import numpy as np @@ -31,13 +30,7 @@ def grass_tutorial_session(test_data_temp_path): # Keep all generated files in the test_data_temp_path os.chdir(test_data_temp_path) tmpdir = tempfile.TemporaryDirectory() - # GRASS 8.4 checks the process GISRC while creating a projected project. - gisrc, _ = gscript.create_environment(tmpdir.name, "itzi_tutorial", "PERMANENT") - try: - with patch.dict(os.environ, {"GISRC": gisrc}): - gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) - finally: - pathlib.Path(gisrc).unlink() + gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) grass_session = gscript.setup.init( path=tmpdir.name, location="itzi_tutorial", mapset="PERMANENT" ) From 1a2706d1bc2e46f265d0309ba27ae21becc29291 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 00:08:18 -0600 Subject: [PATCH 44/85] remove 3.15 from test matrix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc871db3..0949fb9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: test: strategy: matrix: - python-version: ["3.12", "3.13", "3.14", "3.15"] + python-version: ["3.12", "3.13", "3.14"] runs-on: ubuntu-latest container: osgeo/grass-gis:releasebranch_8_4-ubuntu From 4d7d20955156010b8a0a39fb08287400ace4fe1f Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 09:41:08 -0600 Subject: [PATCH 45/85] move grass code into grass_interface --- src/itzi/preflight.py | 52 ++------ src/itzi/providers/grass_input.py | 10 +- src/itzi/providers/grass_interface.py | 138 +++++++++++++++++++-- src/itzi/resolution.py | 172 ++++++-------------------- src/itzi/simulation_runner.py | 12 +- tests/cli/test_ensemble_config.py | 9 +- tests/grass/test_ensemble.py | 30 ++--- 7 files changed, 195 insertions(+), 228 deletions(-) diff --git a/src/itzi/preflight.py b/src/itzi/preflight.py index 15e8c789..b1ff4610 100644 --- a/src/itzi/preflight.py +++ b/src/itzi/preflight.py @@ -14,9 +14,6 @@ from __future__ import annotations -import sqlite3 -from contextlib import closing -from pathlib import Path from typing import TYPE_CHECKING import numpy as np @@ -24,25 +21,20 @@ from itzi.ensemble_models import EnsembleError from itzi.manifest import validate_file_destination from itzi.providers.grass_input import resolve_input_map_lists -from itzi.providers.grass_interface import GrassInterface -from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name +from itzi.providers.grass_output import derived_record_name from itzi.resolution import _last_record_index, verify_resolved_simulation if TYPE_CHECKING: from itzi.ensemble_models import ResolvedSimulation + from itzi.providers.grass_interface import GrassInterface def preflight_simulation(simulation: ResolvedSimulation) -> None: """Validate one resolved simulation without creating user artifacts.""" - import grass.script as gscript + from itzi.providers.grass_interface import GrassInterface verify_resolved_simulation(simulation) - grass_version = gscript.parse_command("g.version", flags="g")["version"] - major, minor = (int(part) for part in grass_version.split(".")[:2]) - if (major, minor) < (8, 4): - raise EnsembleError( - f"Itzi requires at least GRASS 8.4.0, version {grass_version} detected" - ) + GrassInterface.ensure_min_version() config = simulation.simulation_config interface = GrassInterface( start_time=config.start_time, @@ -90,9 +82,6 @@ def _validate_inputs(simulation: ResolvedSimulation, interface: GrassInterface) def _validate_grass_outputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: - import grass.temporal as tgis - from grass.pygrass import utils as gutils - config = simulation.simulation_config last_index = _last_record_index( config.end_time - config.start_time, @@ -101,14 +90,6 @@ def _validate_grass_outputs(simulation: ResolvedSimulation, interface: GrassInte mapset = interface.get_current_mapset() - def map_exists(name: str, element: str) -> bool: - lookup = gutils.get_mapset_raster if element == "cell" else gutils.get_mapset_vector - return bool(lookup(name, mapset)) - - def stds_exists(name: str, stds_type: str) -> bool: - dataset = tgis.dataset_factory(stds_type, interface.format_id(name)) - return dataset is not None and bool(dataset.is_in_db()) - def reject_existing(name: str, exists: bool, kind: str) -> None: if exists and not interface.overwrite: raise EnsembleError(f"{kind} <{name}> already exists") @@ -117,14 +98,14 @@ def reject_existing(name: str, exists: bool, kind: str) -> None: if not interface.overwrite: reject_existing( name, - map_exists(name, "cell") or stds_exists(name, "strds"), + interface.raster_exists(name, mapset) or interface.stds_exists(name, "strds"), "raster output", ) interface.validate_output_stds_temporal_type(name, "strds", config.temporal_type) if not interface.overwrite: for index in range(last_index + 1): child = derived_record_name(name, index) - reject_existing(child, map_exists(child, "cell"), "raster output") + reject_existing(child, interface.raster_exists(child, mapset), "raster output") if config.drainage_output is None: return @@ -132,30 +113,13 @@ def reject_existing(name: str, exists: bool, kind: str) -> None: if not interface.overwrite: reject_existing( name, - map_exists(name, "vector") or stds_exists(name, "stvds"), + interface.vector_exists(name, mapset) or interface.stds_exists(name, "stvds"), "drainage output", ) interface.validate_output_stds_temporal_type(name, "stvds", config.temporal_type) if interface.overwrite: return - # Drainage DB links use PyGRASS Link's default mapset-local SQLite database. - database = ( - Path(gutils.getenv("GISDBASE")) - / gutils.getenv("LOCATION_NAME") - / mapset - / "sqlite" - / "sqlite.db" - ) - existing_tables: set[str] = set() - if database.is_file(): - with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as connection: - existing_tables = { - row[0] - for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") - } for index in range(last_index + 1): child = derived_record_name(name, index) - reject_existing(child, map_exists(child, "vector"), "drainage output") - for table_name in derived_drainage_table_names(child): - reject_existing(table_name, table_name in existing_tables, "drainage table") + reject_existing(child, interface.vector_exists(child, mapset), "drainage output") diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 24472e0f..482d326c 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,14 +15,14 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, NamedTuple +from typing import Literal import numpy as np from itzi_core import INPUT_ARRAY_KEYS, DomainData from itzi_core.providers import RasterInputProvider import itzi.messenger as msgr -from itzi.providers.grass_interface import MapData if TYPE_CHECKING: from datetime import datetime @@ -30,6 +30,12 @@ from itzi.providers.grass_interface import GrassInterface +class MapData(NamedTuple): + id: str + start_time: datetime + end_time: datetime + + class GrassRasterInputProvider(RasterInputProvider): def __init__( self, diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index 4e5244ef..bfff8206 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -20,7 +20,7 @@ from queue import Queue from threading import Lock, Thread from types import MappingProxyType -from typing import ClassVar, NamedTuple, Self +from typing import ClassVar, Literal, NamedTuple, Self import grass.pygrass.utils as gutils import grass.script as gscript @@ -31,7 +31,7 @@ from grass.pygrass.vector import VectorTopo from grass.pygrass.vector.geometry import Line, Point from grass.pygrass.vector.table import Link, Table -from itzi_core import TemporalType +from itzi_core import DomainData, TemporalType from itzi_core.data_containers import ( DrainageLinkAttributes, DrainageNetworkAttributes, @@ -40,6 +40,9 @@ ) import itzi.messenger as msgr +from itzi.ensemble_models import EffectiveMask, EnsembleError +from itzi.grass_session import GrassParams +from itzi.providers.grass_input import MapData # color rules _ROOT = Path(__file__).parent.parent @@ -57,6 +60,7 @@ "froude": str(RULE_FR), } +MIN_GRASS_VERSION = (8, 4) # Check if color rule paths are OK for f in colors_rules_dict.values(): @@ -117,12 +121,6 @@ def raster_writer(q, lock): q.task_done() -class MapData(NamedTuple): - id: str - start_time: datetime - end_time: datetime - - class DBLinkDescription(NamedTuple): layer: int table: Table @@ -358,6 +356,130 @@ def name_is_map(map_id: str) -> bool: name, _, mapset = map_id.partition("@") return bool(gutils.get_mapset_raster(name, mapset)) + @staticmethod + def ensure_min_version() -> None: + full = gscript.parse_command("g.version", flags="g")["version"] + major, minor = (int(part) for part in full.split(".")[:2]) + if (major, minor) < MIN_GRASS_VERSION: + required = f"{MIN_GRASS_VERSION[0]}.{MIN_GRASS_VERSION[1]}" + raise RuntimeError( + f"itzi requires at least GRASS {required}, found version {major}.{minor}" + ) + + @staticmethod + def active_grass_params(requested: GrassParams) -> GrassParams: + """Return the active session context, retaining requested region/mask/bin.""" + return GrassParams( + grassdata=str(Path(gutils.getenv("GISDBASE")).expanduser().resolve()), + location=gutils.getenv("LOCATION_NAME"), + mapset=gutils.getenv("MAPSET"), + region=requested.region, + mask=requested.mask, + grass_bin=requested.grass_bin, + ) + + @staticmethod + def read_domain(region_id: str | None) -> DomainData: + """Read the selected computational region without leaving it changed.""" + if gscript.locn_is_latlong(): + raise EnsembleError("latlong locations are not supported") + using_temp_region = region_id is not None + if using_temp_region: + gscript.use_temp_region() + gscript.run_command("g.region", region=region_id) + try: + region = Region() + if region.cols < 3 or region.rows < 3: + raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") + return DomainData( + north=region.north, + south=region.south, + east=region.east, + west=region.west, + rows=region.rows, + cols=region.cols, + crs_wkt=gscript.read_command("g.proj", flags="fw"), + ) + finally: + if using_temp_region: + gscript.del_temp_region() + + @staticmethod + def _split_identifier(identifier: str) -> tuple[str, str | None]: + name, separator, mapset = identifier.partition("@") + if separator and (not name or not mapset or "@" in mapset): + raise EnsembleError(f"invalid GRASS identifier {identifier!r}") + return name, mapset if separator else None + + @staticmethod + def resolve_input_identifier(identifier: str) -> tuple[str, Literal["raster", "strds"]]: + """Resolve both input namespaces and reject a visible ambiguity.""" + from grass.pygrass.gis import Mapset + + name, requested_mapset = GrassInterface._split_identifier(identifier) + raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") + raster_id = f"{name}@{raster_mapset}" if raster_mapset else None + visible_mapsets = [requested_mapset] if requested_mapset else Mapset().visible.read() + strds_id = None + for mapset in visible_mapsets: + candidate = f"{name}@{mapset}" + try: + is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() + except BaseException as error: + # A visible mapset may have no temporal database or may be + # read-inaccessible. It cannot contribute a STRDS candidate. + if isinstance(error, (KeyboardInterrupt, GeneratorExit)): + raise + is_strds = False + if is_strds: + strds_id = candidate + break + if raster_id is not None and strds_id is not None: + raise EnsembleError( + f"input {identifier!r} is ambiguous: raster {raster_id} " + f"and STRDS {strds_id} both exist" + ) + if raster_id is not None: + return raster_id, "raster" + if strds_id is not None: + return strds_id, "strds" + raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") + + @staticmethod + def resolve_effective_mask(mask: str | None) -> EffectiveMask: + """Select the effective mask descriptor without modifying the mapset MASK.""" + current_mapset = gutils.getenv("MAPSET") + if mask is not None: + name, requested_mapset = GrassInterface._split_identifier(mask) + mapset = gutils.get_mapset_raster(name, requested_mapset or "") + if not mapset: + raise EnsembleError(f"explicit mask {mask!r} was not found") + return EffectiveMask("explicit", f"{name}@{mapset}") + if gutils.get_mapset_raster("MASK", current_mapset): + return EffectiveMask("active", f"MASK@{current_mapset}") + return EffectiveMask("none", None) + + @staticmethod + def is_clean_name(name: str) -> bool: + """Return True if name is a valid GRASS map name.""" + return bool(gutils.is_clean_name(name)) + + @staticmethod + def raster_exists(name: str, mapset: str) -> bool: + """Return True if a raster map exists in the given mapset.""" + return bool(gutils.get_mapset_raster(name, mapset)) + + @staticmethod + def vector_exists(name: str, mapset: str) -> bool: + """Return True if a vector map exists in the given mapset.""" + return bool(gutils.get_mapset_vector(name, mapset)) + + @staticmethod + def stds_exists(name: str, stds_type: str) -> bool: + """Return True if a space-time dataset is registered in the database.""" + dataset = tgis.dataset_factory(stds_type, GrassInterface.format_id(name)) + return dataset is not None and bool(dataset.is_in_db()) + @staticmethod def set_null(map_id: str, threshold: float) -> None: """Set null values under a given threshold""" diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index bce7f79f..f90c878b 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -1,15 +1,22 @@ -"""GRASS-backed resolution of scalar ensemble members. +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. -Only functions in this module import GRASS, and only after a resolver or -execution worker has opened a session. The parent process exchanges the -immutable payloads defined in :mod:`itzi.ensemble`. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. """ from __future__ import annotations import hashlib import json -from collections.abc import Iterable +from collections.abc import Callable, Iterable from datetime import datetime, timedelta from pathlib import Path from typing import Literal @@ -43,14 +50,11 @@ def resolve_ensemble( raise EnsembleError("all simulations in an ensemble must use the same GRASS context") with GrassSessionManager(requested_params): - import grass.script as gscript - from grass.pygrass.gis import Mapset - from grass.pygrass.gis.region import Region - from grass.pygrass.utils import getenv - - actual_params = _active_grass_params(requested_params, getenv) - domain = _read_domain(simulations[0].domain.region, Region, gscript) - effective_mask = _resolve_effective_mask(simulations[0].domain.mask, getenv) + from itzi.providers.grass_interface import GrassInterface + + actual_params = GrassInterface.active_grass_params(requested_params) + domain = GrassInterface.read_domain(simulations[0].domain.region) + effective_mask = GrassInterface.resolve_effective_mask(simulations[0].domain.mask) input_cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] = {} swmm_cache: dict[Path, str] = {} results: list[ResolvedSimulation | ValidationFailure] = [] @@ -58,7 +62,6 @@ def resolve_ensemble( try: input_names, input_kinds = _resolve_inputs( dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), - Mapset, cache=input_cache, ) swmm_path, swmm_digest = _resolve_swmm_input(expanded, swmm_cache) @@ -99,17 +102,6 @@ def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams: ) -def _active_grass_params(requested_params: GrassParams, getenv) -> GrassParams: - return GrassParams( - grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), - location=getenv("LOCATION_NAME"), - mapset=getenv("MAPSET"), - region=requested_params.region, - mask=requested_params.mask, - grass_bin=requested_params.grass_bin, - ) - - def _build_resolved_simulation( expanded: ExpandedSimulation, actual_params: GrassParams, @@ -148,52 +140,24 @@ def _build_resolved_simulation( ) -def _read_domain(region_id: str | None, region_type: type, gscript) -> DomainData: - """Read the selected computational region without leaving it changed.""" - if gscript.locn_is_latlong(): - raise EnsembleError("latlong locations are not supported") - using_temp_region = region_id is not None - if using_temp_region: - gscript.use_temp_region() - gscript.run_command("g.region", region=region_id) - try: - region = region_type() - if region.cols < 3 or region.rows < 3: - raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") - return DomainData( - north=region.north, - south=region.south, - east=region.east, - west=region.west, - rows=region.rows, - cols=region.cols, - crs_wkt=gscript.read_command("g.proj", flags="fw"), - ) - finally: - if using_temp_region: - gscript.del_temp_region() - - -def _split_identifier(identifier: str) -> tuple[str, str | None]: - name, separator, mapset = identifier.partition("@") - if separator and (not name or not mapset or "@" in mapset): - raise EnsembleError(f"invalid GRASS identifier {identifier!r}") - return name, mapset if separator else None - - def _resolve_inputs( inputs: dict[str, str], - mapset_type: type, *, cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] | None = None, + resolve_one: Callable[[str], tuple[str, Literal["raster", "strds"]]] | None = None, ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: + if resolve_one is None: + from itzi.providers.grass_interface import GrassInterface + + resolve_one = GrassInterface.resolve_input_identifier + resolved: dict[str, str] = {} kinds: dict[str, Literal["raster", "strds"]] = {} for key, identifier in inputs.items(): result = cache.get(identifier) if cache is not None else None if result is None: try: - result = _resolve_input_identifier(identifier, mapset_type) + result = resolve_one(identifier) except Exception as error: if cache is not None: cache[identifier] = error @@ -208,57 +172,6 @@ def _resolve_inputs( return resolved, kinds -def _resolve_input_identifier( - identifier: str, mapset_type: type -) -> tuple[str, Literal["raster", "strds"]]: - """Resolve both input namespaces and reject a visible ambiguity.""" - from grass.pygrass import utils as gutils - import grass.temporal as tgis - - name, requested_mapset = _split_identifier(identifier) - raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") - raster_id = f"{name}@{raster_mapset}" if raster_mapset else None - visible_mapsets = [requested_mapset] if requested_mapset else mapset_type().visible.read() - strds_id = None - for mapset in visible_mapsets: - candidate = f"{name}@{mapset}" - try: - is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() - except BaseException as error: - # A visible mapset may have no temporal database or may be - # read-inaccessible. It cannot contribute a STRDS candidate. - if isinstance(error, (KeyboardInterrupt, GeneratorExit)): - raise - is_strds = False - if is_strds: - strds_id = candidate - break - if raster_id is not None and strds_id is not None: - raise EnsembleError( - f"input {identifier!r} is ambiguous: raster {raster_id} and STRDS {strds_id} both exist" - ) - if raster_id is not None: - return raster_id, "raster" - if strds_id is not None: - return strds_id, "strds" - raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") - - -def _resolve_effective_mask(mask: str | None, getenv) -> EffectiveMask: - from grass.pygrass import utils as gutils - - current_mapset = getenv("MAPSET") - if mask is not None: - name, requested_mapset = _split_identifier(mask) - mapset = gutils.get_mapset_raster(name, requested_mapset or "") - if not mapset: - raise EnsembleError(f"explicit mask {mask!r} was not found") - return EffectiveMask("explicit", f"{name}@{mapset}") - if gutils.get_mapset_raster("MASK", current_mapset): - return EffectiveMask("active", f"MASK@{current_mapset}") - return EffectiveMask("none", None) - - def _resolve_swmm_input( expanded: ExpandedSimulation, cache: dict[Path, str] | None = None ) -> tuple[Path | None, str | None]: @@ -396,8 +309,7 @@ def _last_record_index(duration: timedelta, record_step: timedelta) -> int: def _validate_output_names( output_map_names: dict[str, str], drainage_output: str | None, last_record_index: int ) -> None: - from grass.pygrass import utils as gutils - + from itzi.providers.grass_interface import GrassInterface from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name names = list(output_map_names.values()) @@ -408,15 +320,15 @@ def _validate_output_names( raise EnsembleError(f"GRASS output names must be unqualified: {name!r}") if name == "MASK": raise EnsembleError("MASK cannot be used as an output name") - if not gutils.is_clean_name(name): + if not GrassInterface.is_clean_name(name): raise EnsembleError(f"invalid GRASS output name {name!r}") child = derived_record_name(name, last_record_index) - if child == "MASK" or not gutils.is_clean_name(child): + if child == "MASK" or not GrassInterface.is_clean_name(child): raise EnsembleError(f"invalid derived GRASS output name {child!r}") if drainage_output is not None: child = derived_record_name(drainage_output, last_record_index) for table_name in derived_drainage_table_names(child): - if not gutils.is_clean_name(table_name): + if not GrassInterface.is_clean_name(table_name): raise EnsembleError(f"invalid derived drainage table name {table_name!r}") @@ -538,35 +450,23 @@ def _reject_output_source_alias(name: str, mapset: str | None, sources: set[str] def verify_resolved_simulation(simulation: ResolvedSimulation) -> None: """Re-resolve worker facts and reject a changed GRASS environment.""" - import grass.script as gscript - from grass.pygrass.gis import Mapset - from grass.pygrass.gis.region import Region - from grass.pygrass.utils import getenv - - actual = GrassParams( - grassdata=str(Path(getenv("GISDBASE")).expanduser().resolve()), - location=getenv("LOCATION_NAME"), - mapset=getenv("MAPSET"), - region=simulation.grass_params.region, - mask=simulation.grass_params.mask, - grass_bin=simulation.grass_params.grass_bin, - ) + from itzi.providers.grass_interface import GrassInterface + + actual = GrassInterface.active_grass_params(simulation.grass_params) if actual != simulation.grass_params: raise EnsembleError("GRASS context changed after resolution") - domain = _read_domain(simulation.grass_params.region, Region, gscript) + domain = GrassInterface.read_domain(simulation.grass_params.region) if domain != simulation.domain_data: raise EnsembleError("GRASS domain changed after resolution") - mask = _resolve_effective_mask(simulation.grass_params.mask, getenv) + mask = GrassInterface.resolve_effective_mask(simulation.grass_params.mask) if mask != simulation.effective_mask: raise EnsembleError("effective GRASS mask changed after resolution") - _verify_input_kinds( - simulation.simulation_config.input_map_names, dict(simulation.input_kinds), Mapset - ) + _verify_input_kinds(simulation.simulation_config.input_map_names, dict(simulation.input_kinds)) def _verify_input_kinds( - inputs: dict[str, str], expected: dict[str, Literal["raster", "strds"]], mapset_type: type + inputs: dict[str, str], expected: dict[str, Literal["raster", "strds"]] ) -> None: - resolved, kinds = _resolve_inputs(inputs, mapset_type) + resolved, kinds = _resolve_inputs(inputs) if resolved != inputs or kinds != expected: raise EnsembleError("resolved GRASS input sources changed after resolution") diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 439da544..f7a1d0d9 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -45,7 +45,6 @@ def __init__( input_kinds: Mapping[str, Literal["raster", "strds"]] | None = None, exclusive_stats: bool = False, ) -> None: - self.grass_required_version = "8.4.0" self.g_interface: GrassInterface self.sim: Simulation @@ -53,20 +52,13 @@ def __init__( msgr.display_sim_param(sim_config) # Check GRASS version - import grass.script as gscript + from itzi.providers import grass_interface - grass_version = gscript.parse_command("g.version", flags="g")["version"] - if grass_version < self.grass_required_version: - msgr.fatal( - f"Itzi requires at least GRASS {self.grass_required_version}, " - f"version {grass_version} detected." - ) + grass_interface.GrassInterface.ensure_min_version() GrassSessionManager.ensure_temporal_initialized() msgr.debug("GRASS session set") # return error if output files exist - from itzi.providers import grass_interface - output_names = list(sim_config.output_map_names.values()) if sim_config.drainage_output is not None: output_names.append(sim_config.drainage_output) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index e756b546..dd334d02 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -376,25 +376,24 @@ def test_later_generated_output_cannot_alias_an_input(): validate_resolved_ensemble((output, input_simulation)) -def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): +def test_input_resolution_cache_reuses_shared_identifiers(): calls = [] - def resolve(identifier, _mapset_type): + def resolve(identifier): calls.append(identifier) return f"{identifier}@PERMANENT", "raster" - monkeypatch.setattr("itzi.resolution._resolve_input_identifier", resolve) cache = {} _resolve_inputs( {"ground_elevation": "shared", "rainfall_rate": "first"}, - object, cache=cache, + resolve_one=resolve, ) resolved, _ = _resolve_inputs( {"ground_elevation": "shared", "rainfall_rate": "second"}, - object, cache=cache, + resolve_one=resolve, ) assert calls == ["shared", "first", "second"] diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 2f612742..81adee47 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -3,8 +3,6 @@ from __future__ import annotations import os -import sqlite3 -from contextlib import closing from datetime import datetime, timedelta from pathlib import Path from types import SimpleNamespace @@ -17,7 +15,7 @@ from itzi_core import TemporalType from itzi.cli_parser import build_parser -from itzi.ensemble_models import EnsembleError, ResolvedSimulation, ValidationFailure +from itzi.ensemble_models import ResolvedSimulation, ValidationFailure from itzi.grass_session import GrassSessionManager from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, preflight_worker from itzi.run_plan import load_batch @@ -555,19 +553,10 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") def test_preflight_checks_existing_drainage_tables(): - from grass.pygrass import utils as gutils - from itzi.preflight import _validate_grass_outputs + from itzi.providers.grass_interface import GrassInterface GrassSessionManager.ensure_temporal_initialized() - mapset = gutils.getenv("MAPSET") - database = ( - Path(gutils.getenv("GISDBASE")) - / gutils.getenv("LOCATION_NAME") - / mapset - / "sqlite" - / "sqlite.db" - ) name = f"drainage_collision_{uuid4().hex[:8]}" simulation = SimpleNamespace( simulation_config=SimpleNamespace( @@ -581,17 +570,12 @@ def test_preflight_checks_existing_drainage_tables(): ) interface = SimpleNamespace( overwrite=False, - get_current_mapset=lambda: mapset, - format_id=lambda value: f"{value}@{mapset}", + get_current_mapset=GrassInterface.get_current_mapset, + format_id=GrassInterface.format_id, validate_output_stds_temporal_type=lambda *_: None, + raster_exists=GrassInterface.raster_exists, + vector_exists=GrassInterface.vector_exists, + stds_exists=GrassInterface.stds_exists, ) - database_existed = database.exists() _validate_grass_outputs(simulation, interface) - assert database.exists() == database_existed - database.parent.mkdir(parents=True, exist_ok=True) - with closing(sqlite3.connect(database)) as connection: - connection.execute(f"CREATE TABLE {name}_0002_link (cat integer)") - connection.commit() - with pytest.raises(EnsembleError, match=f"drainage table <{name}_0002_link> already exists"): - _validate_grass_outputs(simulation, interface) From e2ad592e319ea8e7b5807c1e285cd2281f1ff4ee Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 15:17:29 -0600 Subject: [PATCH 46/85] move grass fucntions out of GrassInterface, and simplify them --- src/itzi/bmi_itzi.py | 6 +- src/itzi/grass/___init__.py | 0 src/itzi/grass/utils.py | 309 ++++++++++++++++++ src/itzi/itzi.py | 5 +- src/itzi/preflight.py | 39 ++- src/itzi/providers/grass_input.py | 25 +- src/itzi/providers/grass_interface.py | 432 ++++--------------------- src/itzi/providers/grass_output.py | 44 +-- src/itzi/resolution.py | 72 ++--- src/itzi/simulation_runner.py | 128 ++++---- tests/cli/test_ensemble_config.py | 12 +- tests/grass/conftest.py | 10 +- tests/grass/test_ensemble.py | 23 +- tests/grass/test_grass_input.py | 97 +++++- tests/grass/test_itzi.py | 7 +- tests/grass/test_temporal_overwrite.py | 8 +- 16 files changed, 663 insertions(+), 554 deletions(-) create mode 100644 src/itzi/grass/___init__.py create mode 100644 src/itzi/grass/utils.py diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 292de5d7..2108b2d4 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -71,7 +71,7 @@ def __init__(self): # Model control functions # - def initialize(self, filename=None) -> None: + def initialize(self, config_file=None) -> None: """Initialize the Itzï model. Parameters @@ -79,7 +79,7 @@ def initialize(self, filename=None) -> None: filename : str, optional Path to name of input file. """ - conf_data = ConfigReader(filename) + conf_data = ConfigReader(config_file) sim_params = conf_data.sim_config grass_params = conf_data.grass_params self.grass_session_manager = GrassSessionManager(grass_params) @@ -89,7 +89,7 @@ def initialize(self, filename=None) -> None: sim_params, grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() def update(self): """Advance model by one time step.""" diff --git a/src/itzi/grass/___init__.py b/src/itzi/grass/___init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py new file mode 100644 index 00000000..9819bda8 --- /dev/null +++ b/src/itzi/grass/utils.py @@ -0,0 +1,309 @@ +""" +Copyright (C) 2015-2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from pathlib import Path +from queue import Queue +from threading import Lock +from typing import Literal, NamedTuple + +import grass.pygrass.utils as gutils +import grass.script as gscript +import grass.temporal as tgis +import numpy as np +from grass.pygrass import raster +from grass.pygrass.gis import Mapset +from grass.pygrass.gis.region import Region +from itzi_core import DomainData + +import itzi.messenger as msgr +from itzi.ensemble_models import EffectiveMask, EnsembleError +from itzi.grass_session import GrassParams + +MIN_GRASS_VERSION = (8, 4) + +# color rules +_ROOT = Path(__file__).parent.parent +_DIR = _ROOT / "data" / "colortable" +RULE_H = _DIR / "depth.txt" +RULE_V = _DIR / "velocity.txt" +RULE_VDIR = _DIR / "vdir.txt" +RULE_FR = _DIR / "froude.txt" +colors_rules_dict = { + "water_depth": str(RULE_H), + "max_water_depth": str(RULE_H), + "flow_speed": str(RULE_V), + "max_flow_speed": str(RULE_V), + "flow_velocity_direction": str(RULE_VDIR), + "froude": str(RULE_FR), +} +for f in colors_rules_dict.values(): + assert Path(f).is_file() + + +type STDSType = Literal["strds", "stvds"] +type RasterMapType = Literal["CELL", "FCELL", "DCELL"] + + +class RasterWriteInstructions(NamedTuple): + array: np.ndarray + raster_name: str + map_type: RasterMapType + map_key: str + hmin: float + overwrite: bool + + +def file_exists(name: str) -> bool: + """Return True if an output exists in the current mapset.""" + if not name: + return False + map_id = qualify_output_id(name, get_current_mapset()) + map_name, mapset = split_identifier(map_id) + return ( + name_is_map(map_id) + or vector_exists(map_name, mapset) + or stds_exists(map_id, "strds") + or stds_exists(map_id, "stvds") + ) + + +def check_output_files(file_list: list[str]) -> None: + """Check if the output files exist""" + if gscript.overwrite() or not file_list: + return + for map_name in file_list: + if file_exists(map_name): + msgr.fatal(f"File {map_name} exists and will not be overwritten") + + +def raster_writer(q: Queue[RasterWriteInstructions | None], lock: Lock): + """Write a raster map in GRASS""" + while True: + write_instructions = q.get() + if write_instructions is None: + break + with lock: + write_raster_map_blocking(write_instructions) + q.task_done() + + +def write_raster_map_blocking(writer_instructions: RasterWriteInstructions) -> None: + raster_name = writer_instructions.raster_name + array = writer_instructions.array + map_key = writer_instructions.map_key + map_type = writer_instructions.map_type + hmin = writer_instructions.hmin + with raster.RasterRow( + raster_name, + mapset=get_current_mapset(), + mode="w", + mtype=map_type, + overwrite=writer_instructions.overwrite, + ) as newraster: + newrow = raster.Buffer((array.shape[1],), mtype=map_type) + for row in array: + newrow[:] = row[:] + newraster.put_row(newrow) + # apply color table + colors_rules = colors_rules_dict.get(map_key) + if colors_rules is not None: + gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=raster_name) + # set null values + if map_key == "water_depth" and hmin > 0: + set_null(raster_name, hmin) + + +def resolve_effective_mask(mask: str | None) -> EffectiveMask: + """Select the effective mask descriptor without modifying the mapset MASK.""" + current_mapset = gutils.getenv("MAPSET") + if mask is not None: + name, requested_mapset = split_identifier(mask) + mapset = gutils.get_mapset_raster(name, requested_mapset or "") + if not mapset: + raise EnsembleError(f"explicit mask {mask!r} was not found") + return EffectiveMask("explicit", f"{name}@{mapset}") + if has_mask(): + return EffectiveMask("active", f"MASK@{current_mapset}") + return EffectiveMask("none", None) + + +def has_mask() -> bool: + """Return True if the mapset has a mask, False otherwise.""" + return bool(gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET"))) + + +def format_id(name: str) -> str: + """Qualify a raster name using the GRASS search path if it exists.""" + if "@" in name: + return name + return f"{name}@{gutils.get_mapset_raster(name) or gutils.getenv('MAPSET')}" + + +def qualify_output_id(name: str, mapset: str) -> str: + """Give a new output its destination mapset, never a searched mapset.""" + map_name, requested_mapset = split_identifier(name) + if requested_mapset and requested_mapset != mapset: + raise EnsembleError(f"output {name!r} must be in the current mapset {mapset!r}") + return f"{map_name}@{mapset}" + + +def output_name(map_id: str) -> str: + """Return the bare name required by GRASS spatial writers.""" + name, mapset = split_identifier(map_id) + if mapset != get_current_mapset(): + raise EnsembleError(f"output {map_id!r} must be in the current mapset") + return name + + +def get_current_mapset() -> str: + return gutils.getenv("MAPSET") + + +def name_is_stds(stds_id: str) -> bool: + """Return True if the qualified ID identifies a registered STRDS.""" + return bool(tgis.SpaceTimeRasterDataset(stds_id).is_in_db()) + + +def name_is_map(map_id: str) -> bool: + """return True if the given name is a map in the grass database + False if not + """ + name, mapset = split_identifier(map_id) + return raster_exists(name, mapset) + + +def ensure_min_version() -> None: + full = gscript.parse_command("g.version", flags="g")["version"] + major, minor = (int(part) for part in full.split(".")[:2]) + if (major, minor) < MIN_GRASS_VERSION: + required = f"{MIN_GRASS_VERSION[0]}.{MIN_GRASS_VERSION[1]}" + raise RuntimeError( + f"itzi requires at least GRASS {required}, found version {major}.{minor}" + ) + + +def active_grass_params(requested: GrassParams) -> GrassParams: + """Return the active session context, retaining requested region/mask/bin.""" + return GrassParams( + grassdata=str(Path(gutils.getenv("GISDBASE")).expanduser().resolve()), + location=gutils.getenv("LOCATION_NAME"), + mapset=gutils.getenv("MAPSET"), + region=requested.region, + mask=requested.mask, + grass_bin=requested.grass_bin, + ) + + +def read_domain(region_id: str | None) -> DomainData: + """Read the selected computational region without leaving it changed.""" + if gscript.locn_is_latlong(): + raise EnsembleError("latlong locations are not supported") + using_temp_region = region_id is not None + if using_temp_region: + gscript.use_temp_region() + gscript.run_command("g.region", region=region_id) + try: + region = Region() + if region.cols < 3 or region.rows < 3: + raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") + return DomainData( + north=region.north, + south=region.south, + east=region.east, + west=region.west, + rows=region.rows, + cols=region.cols, + crs_wkt=gscript.read_command("g.proj", flags="fw"), + ) + finally: + if using_temp_region: + gscript.del_temp_region() + + +def split_identifier(identifier: str) -> tuple[str, str]: + name, separator, mapset = identifier.partition("@") + if separator and (not name or not mapset or "@" in mapset): + raise EnsembleError(f"invalid GRASS identifier {identifier!r}") + return name, mapset + + +def resolve_input_identifier(identifier: str) -> tuple[str, Literal["raster", "strds"]]: + """Resolve both input namespaces and reject a visible ambiguity.""" + + name, requested_mapset = split_identifier(identifier) + raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") + raster_id = f"{name}@{raster_mapset}" if raster_mapset else None + visible_mapsets = [requested_mapset] if requested_mapset else Mapset().visible.read() + strds_id = None + for mapset in visible_mapsets: + candidate = f"{name}@{mapset}" + try: + is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() + except BaseException as error: + # A visible mapset may have no temporal database or may be + # read-inaccessible. It cannot contribute a STRDS candidate. + if isinstance(error, (KeyboardInterrupt, GeneratorExit)): + raise + is_strds = False + if is_strds: + strds_id = candidate + break + if raster_id is not None and strds_id is not None: + raise EnsembleError( + f"input {identifier!r} is ambiguous: raster {raster_id} " + f"and STRDS {strds_id} both exist" + ) + if raster_id is not None: + return raster_id, "raster" + if strds_id is not None: + return strds_id, "strds" + raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") + + +def is_clean_name(name: str) -> bool: + """Return True if name is a valid GRASS map name.""" + return bool(gutils.is_clean_name(name)) + + +def raster_exists(name: str, mapset: str) -> bool: + return bool(gutils.get_mapset_raster(name, mapset)) + + +def vector_exists(name: str, mapset: str) -> bool: + """Return True if a vector map exists in the given mapset.""" + return bool(gutils.get_mapset_vector(name, mapset)) + + +def stds_exists(stds_id: str, stds_type: STDSType) -> bool: + """Return True if a space-time dataset is registered in the database.""" + return bool(tgis.dataset_factory(stds_type, stds_id).is_in_db()) + + +def set_null(map_id: str, threshold: float) -> None: + """Set null values under a given threshold""" + gscript.run_command("r.null", flags="f", map=map_id, setnull=f"0.0-{threshold}") + + +def replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarray: + """Normalize GRASS CELL nulls to NaN. + + FCELL/DCELL nulls are already exposed as NaN by pygrass. CELL nulls are + returned as the int32 null sentinel cast to the target dtype. + """ + if raster_type != "CELL" or not np.issubdtype(array.dtype, np.floating): + return array + + null_sentinel = array.dtype.type(np.iinfo(np.int32).min) + array[array == null_sentinel] = np.nan + return array diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 6e4c48ab..375800d8 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -98,7 +98,7 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: hotstart_path=hotstart_file, stats_file=conf_data.stats_file, ) - sim_runner.run().finalize() + sim_runner.initialize().run().finalize() except msgr.FatalError: raise SystemExit(1) from None except SystemExit as error: @@ -126,9 +126,8 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str else None, effective_mask=simulation.effective_mask, input_kinds=dict(simulation.input_kinds), - exclusive_stats=True, ) - runner.run().finalize() + runner.initialize().run().finalize() return "completed", None except Exception as error: if runner is not None: diff --git a/src/itzi/preflight.py b/src/itzi/preflight.py index b1ff4610..0c9878b5 100644 --- a/src/itzi/preflight.py +++ b/src/itzi/preflight.py @@ -31,10 +31,11 @@ def preflight_simulation(simulation: ResolvedSimulation) -> None: """Validate one resolved simulation without creating user artifacts.""" + from itzi.grass.utils import ensure_min_version from itzi.providers.grass_interface import GrassInterface verify_resolved_simulation(simulation) - GrassInterface.ensure_min_version() + ensure_min_version() config = simulation.simulation_config interface = GrassInterface( start_time=config.start_time, @@ -82,44 +83,54 @@ def _validate_inputs(simulation: ResolvedSimulation, interface: GrassInterface) def _validate_grass_outputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + from itzi.grass.utils import ( + get_current_mapset, + output_name, + raster_exists, + stds_exists, + vector_exists, + ) + config = simulation.simulation_config last_index = _last_record_index( config.end_time - config.start_time, config.record_step, ) - mapset = interface.get_current_mapset() + mapset = get_current_mapset() def reject_existing(name: str, exists: bool, kind: str) -> None: if exists and not interface.overwrite: raise EnsembleError(f"{kind} <{name}> already exists") - for name in config.output_map_names.values(): + for map_id in config.output_map_names.values(): + name = output_name(map_id) if not interface.overwrite: reject_existing( - name, - interface.raster_exists(name, mapset) or interface.stds_exists(name, "strds"), + map_id, + raster_exists(name, mapset) or stds_exists(map_id, "strds"), "raster output", ) - interface.validate_output_stds_temporal_type(name, "strds", config.temporal_type) + interface.validate_output_stds_temporal_type(map_id, "strds", config.temporal_type) if not interface.overwrite: for index in range(last_index + 1): - child = derived_record_name(name, index) - reject_existing(child, interface.raster_exists(child, mapset), "raster output") + child = derived_record_name(map_id, index) + reject_existing(child, raster_exists(output_name(child), mapset), "raster output") if config.drainage_output is None: return - name = config.drainage_output + map_id = config.drainage_output + name = output_name(map_id) if not interface.overwrite: reject_existing( - name, - interface.vector_exists(name, mapset) or interface.stds_exists(name, "stvds"), + map_id, + vector_exists(name, mapset) or stds_exists(map_id, "stvds"), "drainage output", ) - interface.validate_output_stds_temporal_type(name, "stvds", config.temporal_type) + interface.validate_output_stds_temporal_type(map_id, "stvds", config.temporal_type) if interface.overwrite: return for index in range(last_index + 1): - child = derived_record_name(name, index) - reject_existing(child, interface.vector_exists(child, mapset), "drainage output") + child = derived_record_name(map_id, index) + reject_existing(child, vector_exists(output_name(child), mapset), "drainage output") diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 482d326c..d41d2ea3 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,8 +15,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, NamedTuple -from typing import Literal +from typing import TYPE_CHECKING, Literal, NamedTuple import numpy as np from itzi_core import INPUT_ARRAY_KEYS, DomainData @@ -58,6 +57,8 @@ def __init__( def get_domain_data(self) -> DomainData: """Return a DomainData object""" + import grass.script as gscript + return DomainData( north=self.grass_interface.region.north, south=self.grass_interface.region.south, @@ -65,7 +66,7 @@ def get_domain_data(self) -> DomainData: west=self.grass_interface.region.west, rows=self.grass_interface.region.rows, cols=self.grass_interface.region.cols, - crs_wkt=self.grass_interface.get_crs_wkt(), + crs_wkt=gscript.read_command("g.proj", flags="fw"), ) def get_array( @@ -101,6 +102,7 @@ def resolve_input_map_lists( input_kinds: Mapping[str, Literal["raster", "strds"]] | None, ) -> Mapping[str, list[MapData] | None]: """Resolve provider-ready raster lists without creating an output provider.""" + from itzi.grass.utils import format_id, name_is_map, name_is_stds, resolve_input_identifier invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) if invalid_input_keys: @@ -110,20 +112,19 @@ def resolve_input_map_lists( for key, map_name in map_names.items(): if not map_name: continue - map_id = grass_interface.format_id(map_name) kind = input_kinds.get(key) if input_kinds is not None else None + if kind is None: + map_id, kind = resolve_input_identifier(map_name) + elif kind == "raster": + map_id = format_id(map_name) + else: + map_id = map_name if "@" in map_name else resolve_input_identifier(map_name)[0] if kind == "strds": - if not grass_interface.name_is_stds(map_id): + if not name_is_stds(map_id): msgr.fatal(f"STRDS input <{map_id}> is no longer available") elif kind == "raster": - if not grass_interface.name_is_map(map_id): + if not name_is_map(map_id): msgr.fatal(f"raster input <{map_id}> is no longer available") - elif grass_interface.name_is_stds(map_id): - kind = "strds" - elif grass_interface.name_is_map(map_id): - kind = "raster" - else: - msgr.fatal(f"{map_name} not found!") if kind == "strds": map_list = grass_interface.raster_list_from_strds(map_id) else: diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/providers/grass_interface.py index bfff8206..75d57e2f 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/providers/grass_interface.py @@ -16,22 +16,20 @@ import os from collections.abc import Mapping from datetime import datetime, timedelta -from pathlib import Path from queue import Queue from threading import Lock, Thread from types import MappingProxyType -from typing import ClassVar, Literal, NamedTuple, Self +from typing import ClassVar, NamedTuple import grass.pygrass.utils as gutils import grass.script as gscript import grass.temporal as tgis import numpy as np -from grass.pygrass import raster from grass.pygrass.gis.region import Region from grass.pygrass.vector import VectorTopo from grass.pygrass.vector.geometry import Line, Point from grass.pygrass.vector.table import Link, Table -from itzi_core import DomainData, TemporalType +from itzi_core import TemporalType from itzi_core.data_containers import ( DrainageLinkAttributes, DrainageNetworkAttributes, @@ -40,86 +38,21 @@ ) import itzi.messenger as msgr -from itzi.ensemble_models import EffectiveMask, EnsembleError -from itzi.grass_session import GrassParams +from itzi.grass.utils import ( + RasterMapType, + RasterWriteInstructions, + STDSType, + format_id, + has_mask, + name_is_map, + output_name, + raster_writer, + replace_cell_null_sentinel, + split_identifier, + write_raster_map_blocking, +) from itzi.providers.grass_input import MapData -# color rules -_ROOT = Path(__file__).parent.parent -_DIR = _ROOT / "data" / "colortable" -RULE_H = _DIR / "depth.txt" -RULE_V = _DIR / "velocity.txt" -RULE_VDIR = _DIR / "vdir.txt" -RULE_FR = _DIR / "froude.txt" -colors_rules_dict = { - "water_depth": str(RULE_H), - "max_water_depth": str(RULE_H), - "flow_speed": str(RULE_V), - "max_flow_speed": str(RULE_V), - "flow_velocity_direction": str(RULE_VDIR), - "froude": str(RULE_FR), -} - -MIN_GRASS_VERSION = (8, 4) - -# Check if color rule paths are OK -for f in colors_rules_dict.values(): - assert Path(f).is_file() - - -def file_exists(name: str) -> bool: - """Return True if name is an existing map or stds, False otherwise""" - if not name: - return False - _id = GrassInterface.format_id(name) - return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds(_id) - - -def check_output_files(file_list: list[str]) -> None: - """Check if the output files exist""" - if gscript.overwrite() or not file_list: - return - for map_name in file_list: - if file_exists(map_name): - msgr.fatal(f"File {map_name} exists and will not be overwritten") - - -def apply_color_table(map_name, mkey) -> None: - """Apply a color table determined by mkey to the given map""" - try: - colors_rules = colors_rules_dict[mkey] - except KeyError: - # in case no specific color table is given, use GRASS default. - pass - else: - gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=map_name) - - -def raster_writer(q, lock): - """Write a raster map in GRASS""" - while True: - # Get values from the queue - next_object = q.get() - if next_object is None: - break - arr, rast_name, mtype, mkey, hmin, overwrite = next_object - # Write raster - with lock: - with raster.RasterRow( - rast_name, mode="w", mtype=mtype, overwrite=overwrite - ) as newraster: - newrow = raster.Buffer((arr.shape[1],), mtype=mtype) - for row in arr: - newrow[:] = row[:] - newraster.put_row(newrow) - # Apply colour table - apply_color_table(rast_name, mkey) - # set null values - if mkey == "water_depth" and hmin > 0: - GrassInterface.set_null(rast_name, hmin) - # Signal end of task - q.task_done() - class DBLinkDescription(NamedTuple): layer: int @@ -213,10 +146,11 @@ def __init__( # Create thread and queue for writing raster maps if self.non_blocking_write: self.raster_lock = Lock() - self.raster_writer_queue = Queue(maxsize=15) - worker_args = (self.raster_writer_queue, self.raster_lock) + self.raster_writer_queue: Queue[RasterWriteInstructions | None] = Queue(maxsize=20) self.raster_writer_thread = Thread( - name="RasterWriter", target=raster_writer, args=worker_args + name="RasterWriter", + target=raster_writer, + args=(self.raster_writer_queue, self.raster_lock), ) self.raster_writer_thread.start() @@ -247,7 +181,7 @@ def cleanup(self) -> None: self.raster_writer_queue.put(None) self.raster_writer_thread.join() - def grass_dtype(self, dtype: str) -> str: + def grass_dtype(self, dtype: str) -> RasterMapType: if dtype in self.dtype_conv["DCELL"]: mtype = "DCELL" elif dtype in self.dtype_conv["CELL"]: @@ -276,11 +210,6 @@ def to_datetime(self, unit: str, time: int) -> datetime: """ return self.start_time + timedelta(seconds=self.to_s(unit, time)) - @staticmethod - def has_mask() -> bool: - """Return True if the mapset has a mask, False otherwise.""" - return bool(gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET"))) - def _select_effective_mask( self, effective_mask: tuple[str, str | None] | None ) -> tuple[str, str | None]: @@ -293,8 +222,8 @@ def _select_effective_mask( msgr.fatal(f"Effective {mode} mask has no source") return mode, source if self.raster_mask_id: - return "explicit", self.format_id(self.raster_mask_id) - if self.has_mask(): + return "explicit", format_id(self.raster_mask_id) + if has_mask(): return "active", f"MASK@{gutils.getenv('MAPSET')}" return "none", None @@ -310,181 +239,6 @@ def get_npmask(self) -> np.ndarray: # A mapset MASK uses CELL semantics: zero and NULL are outside. return np.isnan(grass_mask) | (grass_mask == 0) - def coor2pixel(self, coor: tuple[float, float]) -> tuple[int, int]: - """convert coordinates easting and northing to pixel row and column""" - row, col = gutils.coor2pixel(coor, self.region) - return int(row), int(col) - - def is_in_region(self, x: float, y: float) -> bool: - """For a given coordinate pair(x, y), - return True is inside raster region, False otherwise. - """ - bool_x = self.reg_bbox["w"] < x < self.reg_bbox["e"] - bool_y = self.reg_bbox["s"] < y < self.reg_bbox["n"] - return bool(bool_x and bool_y) - - @staticmethod - def format_id(name: str) -> str: - """Take a map or stds name as input - and return a fully qualified name, i.e. including mapset - """ - if "@" in name: - return name - else: - return "@".join((name, gutils.getenv("MAPSET"))) - - @staticmethod - def get_current_mapset() -> str: - return gutils.getenv("MAPSET") - - @staticmethod - def name_is_stds(name: str) -> bool: - """return True if the name given as input is a registered strds - False if not - """ - return bool(tgis.SpaceTimeRasterDataset(name).is_in_db()) - - @staticmethod - def get_crs_wkt() -> str: - return gscript.read_command("g.proj", flags="fw") - - @staticmethod - def name_is_map(map_id: str) -> bool: - """return True if the given name is a map in the grass database - False if not - """ - name, _, mapset = map_id.partition("@") - return bool(gutils.get_mapset_raster(name, mapset)) - - @staticmethod - def ensure_min_version() -> None: - full = gscript.parse_command("g.version", flags="g")["version"] - major, minor = (int(part) for part in full.split(".")[:2]) - if (major, minor) < MIN_GRASS_VERSION: - required = f"{MIN_GRASS_VERSION[0]}.{MIN_GRASS_VERSION[1]}" - raise RuntimeError( - f"itzi requires at least GRASS {required}, found version {major}.{minor}" - ) - - @staticmethod - def active_grass_params(requested: GrassParams) -> GrassParams: - """Return the active session context, retaining requested region/mask/bin.""" - return GrassParams( - grassdata=str(Path(gutils.getenv("GISDBASE")).expanduser().resolve()), - location=gutils.getenv("LOCATION_NAME"), - mapset=gutils.getenv("MAPSET"), - region=requested.region, - mask=requested.mask, - grass_bin=requested.grass_bin, - ) - - @staticmethod - def read_domain(region_id: str | None) -> DomainData: - """Read the selected computational region without leaving it changed.""" - if gscript.locn_is_latlong(): - raise EnsembleError("latlong locations are not supported") - using_temp_region = region_id is not None - if using_temp_region: - gscript.use_temp_region() - gscript.run_command("g.region", region=region_id) - try: - region = Region() - if region.cols < 3 or region.rows < 3: - raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") - return DomainData( - north=region.north, - south=region.south, - east=region.east, - west=region.west, - rows=region.rows, - cols=region.cols, - crs_wkt=gscript.read_command("g.proj", flags="fw"), - ) - finally: - if using_temp_region: - gscript.del_temp_region() - - @staticmethod - def _split_identifier(identifier: str) -> tuple[str, str | None]: - name, separator, mapset = identifier.partition("@") - if separator and (not name or not mapset or "@" in mapset): - raise EnsembleError(f"invalid GRASS identifier {identifier!r}") - return name, mapset if separator else None - - @staticmethod - def resolve_input_identifier(identifier: str) -> tuple[str, Literal["raster", "strds"]]: - """Resolve both input namespaces and reject a visible ambiguity.""" - from grass.pygrass.gis import Mapset - - name, requested_mapset = GrassInterface._split_identifier(identifier) - raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") - raster_id = f"{name}@{raster_mapset}" if raster_mapset else None - visible_mapsets = [requested_mapset] if requested_mapset else Mapset().visible.read() - strds_id = None - for mapset in visible_mapsets: - candidate = f"{name}@{mapset}" - try: - is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() - except BaseException as error: - # A visible mapset may have no temporal database or may be - # read-inaccessible. It cannot contribute a STRDS candidate. - if isinstance(error, (KeyboardInterrupt, GeneratorExit)): - raise - is_strds = False - if is_strds: - strds_id = candidate - break - if raster_id is not None and strds_id is not None: - raise EnsembleError( - f"input {identifier!r} is ambiguous: raster {raster_id} " - f"and STRDS {strds_id} both exist" - ) - if raster_id is not None: - return raster_id, "raster" - if strds_id is not None: - return strds_id, "strds" - raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") - - @staticmethod - def resolve_effective_mask(mask: str | None) -> EffectiveMask: - """Select the effective mask descriptor without modifying the mapset MASK.""" - current_mapset = gutils.getenv("MAPSET") - if mask is not None: - name, requested_mapset = GrassInterface._split_identifier(mask) - mapset = gutils.get_mapset_raster(name, requested_mapset or "") - if not mapset: - raise EnsembleError(f"explicit mask {mask!r} was not found") - return EffectiveMask("explicit", f"{name}@{mapset}") - if gutils.get_mapset_raster("MASK", current_mapset): - return EffectiveMask("active", f"MASK@{current_mapset}") - return EffectiveMask("none", None) - - @staticmethod - def is_clean_name(name: str) -> bool: - """Return True if name is a valid GRASS map name.""" - return bool(gutils.is_clean_name(name)) - - @staticmethod - def raster_exists(name: str, mapset: str) -> bool: - """Return True if a raster map exists in the given mapset.""" - return bool(gutils.get_mapset_raster(name, mapset)) - - @staticmethod - def vector_exists(name: str, mapset: str) -> bool: - """Return True if a vector map exists in the given mapset.""" - return bool(gutils.get_mapset_vector(name, mapset)) - - @staticmethod - def stds_exists(name: str, stds_type: str) -> bool: - """Return True if a space-time dataset is registered in the database.""" - dataset = tgis.dataset_factory(stds_type, GrassInterface.format_id(name)) - return dataset is not None and bool(dataset.is_in_db()) - - @staticmethod - def set_null(map_id: str, threshold: float) -> None: - """Set null values under a given threshold""" - gscript.run_command("r.null", flags="f", map=map_id, setnull=f"0.0-{threshold}") - def get_sim_extend_in_stds_unit(self, strds) -> tuple[int | datetime, int | datetime]: """Take a strds object as input Return the simulation start_time and end_time, expressed in @@ -567,7 +321,7 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: columns=",".join(MapData._fields), where=where, order="start_time" ) # check if every map exist - maps_not_found = [m[0] for m in maplist if not self.name_is_map(m[0])] + maps_not_found = [m[0] for m in maplist if not name_is_map(m[0])] if any(maps_not_found): err_msg = "STRDS <{}>: Can't find following maps: {}" str_lst = ",".join(maps_not_found) @@ -587,29 +341,27 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: def validate_output_stds_temporal_type( self, - stds_name: str | None, - stds_type: str, + stds_id: str, + stds_type: STDSType, expected_temporal_type: TemporalType, ) -> None: """Fail early when overwriting an STDS with incompatible temporal type.""" - if not stds_name or not self.overwrite: + if not self.overwrite: return - stds_id = self.format_id(stds_name) - stds = tgis.dataset_factory(stds_type, stds_id) - if stds is None or not stds.is_in_db(): + output_name(stds_id) + if not tgis.dataset_factory(stds_type, stds_id).is_in_db(): return - existing_stds = tgis.open_stds.open_old_stds(stds_id, stds_type) existing_temporal_type = TemporalType(existing_stds.get_temporal_type()) if existing_temporal_type != expected_temporal_type: msgr.fatal( - f"Output {stds_type.upper()} <{stds_name}> already exists with " + f"Output {stds_type.upper()} <{stds_id}> already exists with " f"{existing_temporal_type} temporal type and cannot be overwritten " f"by a {expected_temporal_type} simulation" ) - def read_raster_map(self, rast_name: str) -> np.ndarray: + def read_raster_map(self, raster_id: str) -> np.ndarray: """Read a raster through GRASS 8.4's no-mask row APIs. The active mapset MASK must not alter model input values. The one @@ -618,20 +370,17 @@ def read_raster_map(self, rast_name: str) -> np.ndarray: """ if self.non_blocking_write: with self.raster_lock: - return self._read_raster_map_nomask(rast_name) - return self._read_raster_map_nomask(rast_name) + return self._read_raster_map_nomask(raster_id) + return self._read_raster_map_nomask(raster_id) - def _read_raster_map_nomask(self, rast_name: str) -> np.ndarray: + def _read_raster_map_nomask(self, raster_id: str) -> np.ndarray: """Read the current computational region while bypassing GRASS MASK.""" from grass.lib import raster as libraster - name, separator, mapset = rast_name.partition("@") - if separator and (not name or not mapset or "@" in mapset): - msgr.fatal(f"Invalid raster identifier <{rast_name}>") - if not separator: - mapset = gutils.get_mapset_raster(name) + name, mapset = split_identifier(raster_id) if not mapset: - msgr.fatal(f"Raster <{rast_name}> not found") + msgr.fatal(f"Raster <{raster_id}> not found") + raster_type = libraster.Rast_map_type(name, mapset) reader_data = { libraster.CELL_TYPE: (libraster.Rast_allocate_c_buf, libraster.Rast_get_c_row_nomask), @@ -639,11 +388,11 @@ def _read_raster_map_nomask(self, rast_name: str) -> np.ndarray: libraster.DCELL_TYPE: (libraster.Rast_allocate_d_buf, libraster.Rast_get_d_row_nomask), }.get(raster_type) if reader_data is None: - msgr.fatal(f"Raster <{rast_name}> has an unsupported data type") + msgr.fatal(f"Raster <{raster_id}> has an unsupported data type") allocate, read_row = reader_data file_descriptor = libraster.Rast_open_old(name, mapset) if file_descriptor < 0: - msgr.fatal(f"Raster <{rast_name}> cannot be opened") + msgr.fatal(f"Raster <{raster_id}> cannot be opened") buffer = allocate() try: array = np.empty((self.yr, self.xr), dtype=self.dtype) @@ -652,7 +401,7 @@ def _read_raster_map_nomask(self, rast_name: str) -> np.ndarray: array[row_index] = np.ctypeslib.as_array(buffer, shape=(self.xr,)) finally: libraster.Rast_close(file_descriptor) - return self._replace_cell_null_sentinel( + return replace_cell_null_sentinel( { libraster.CELL_TYPE: "CELL", libraster.FCELL_TYPE: "FCELL", @@ -661,65 +410,21 @@ def _read_raster_map_nomask(self, rast_name: str) -> np.ndarray: array, ) - @staticmethod - def _replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarray: - """Normalize GRASS CELL nulls to NaN. - - FCELL/DCELL nulls are already exposed as NaN by pygrass. CELL nulls are - returned as the int32 null sentinel cast to the target dtype. - """ - if raster_type != "CELL" or not np.issubdtype(array.dtype, np.floating): - return array - - null_sentinel = array.dtype.type(np.iinfo(np.int32).min) - array[array == null_sentinel] = np.nan - return array - - def write_raster_map(self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float) -> Self: + def write_raster_map(self, arr: np.ndarray, raster_id: str, mkey: str, hmin: float) -> None: """Take a numpy array and write it to GRASS DB""" - assert isinstance(arr, np.ndarray), "arr not a np array!" - assert isinstance(rast_name, str), "not a string!" - assert isinstance(mkey, str), "not a string!" - if self.non_blocking_write: - self.write_raster_map_nonblocking(arr, rast_name, mkey, hmin) - else: - self.write_raster_map_blocking(arr, rast_name, mkey, hmin) - return self - - def write_raster_map_nonblocking( - self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float - ) -> Self: mtype = self.grass_dtype(str(arr.dtype)) - assert isinstance(mtype, str), "not a string!" - q_obj = ( - arr.copy(), - copy.deepcopy(rast_name), - copy.deepcopy(mtype), - copy.deepcopy(mkey), - copy.deepcopy(hmin), - self.overwrite, + write_instructions = RasterWriteInstructions( + array=arr, + raster_name=output_name(raster_id), + map_type=mtype, + map_key=mkey, + hmin=hmin, + overwrite=self.overwrite, ) - self.raster_writer_queue.put(q_obj) - return self - - def write_raster_map_blocking( - self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float - ) -> Self: - mtype = self.grass_dtype(str(arr.dtype)) - assert isinstance(mtype, str), "not a string!" - with raster.RasterRow( - rast_name, mode="w", mtype=mtype, overwrite=self.overwrite - ) as newraster: - newrow = raster.Buffer((arr.shape[1],), mtype=mtype) - for row in arr: - newrow[:] = row[:] - newraster.put_row(newrow) - # apply color table - apply_color_table(rast_name, mkey) - # set null values - if mkey == "water_depth" and hmin > 0: - GrassInterface.set_null(rast_name, hmin) - return self + if self.non_blocking_write: + self.raster_writer_queue.put(write_instructions) + else: + write_raster_map_blocking(write_instructions) def create_db_links( self, vect_map: VectorTopo, linking_elem: dict[str, DBLayerDescription] @@ -747,9 +452,10 @@ def write_vector_map( self, topology: DrainageNetworkTopology, attributes: DrainageNetworkAttributes, - map_name: str, - ) -> Self: + map_id: str, + ) -> None: """Write a vector map to GRASS GIS""" + map_name = output_name(map_id) node_attributes = {node.node_id: node for node in attributes.nodes} link_attributes = {link.link_id: link for link in attributes.links} topology_node_ids = {node.node_id for node in topology.nodes} @@ -772,7 +478,9 @@ def write_vector_map( } # set category manually cat_num = 1 - with VectorTopo(map_name, mode="w", overwrite=self.overwrite) as vector_map: + with VectorTopo( + map_name, mapset=gutils.getenv("MAPSET"), mode="w", overwrite=self.overwrite + ) as vector_map: # create db links and tables dblinks = self.create_db_links(vector_map, linking_elements) # dict to keep DB infos to write DB after geometries @@ -815,21 +523,20 @@ def write_vector_map( for attr in attrs: dbtable.insert(attr) dbtable.conn.commit() - return self def register_maps_in_stds( self, stds_title: str, - stds_name: str, + stds_id: str, map_list: list[tuple[str, datetime | timedelta]], - stds_type: str, - t_type: str, - ): + stds_type: STDSType, + t_type: TemporalType, + ) -> None: """Create a STDS, create one mapdataset for each map and register them in the temporal database """ assert isinstance(stds_title, str), "not a string!" - assert isinstance(stds_name, str), "not a string!" + output_name(stds_id) assert isinstance(t_type, str), "not a string!" # Print message in case of decreased GRASS verbosity if msgr.verbosity() <= 2: @@ -839,7 +546,6 @@ def register_maps_in_stds( if stds_type == "stvds" and grass_verbosity != "-1": os.environ["GRASS_VERBOSE"] = "-1" # create stds - stds_id = self.format_id(stds_name) stds_desc = "" stds = tgis.open_new_stds( stds_id, @@ -851,13 +557,11 @@ def register_maps_in_stds( overwrite=self.overwrite, ) - # create MapDataset objects list + dataset_type = {"strds": tgis.RasterDataset, "stvds": tgis.VectorDataset}[stds_type] map_dts_lst = [] - for map_name, map_time in map_list: - # create MapDataset - map_id = self.format_id(map_name) - map_dts_type = {"strds": tgis.RasterDataset, "stvds": tgis.VectorDataset} - map_dts = map_dts_type[stds_type](map_id) + for map_id, map_time in map_list: + output_name(map_id) + map_dts = dataset_type(map_id) # load spatial data from map map_dts.load() # set time @@ -868,9 +572,6 @@ def register_maps_in_stds( elif t_type == TemporalType.ABSOLUTE: assert isinstance(map_time, datetime) map_dts.set_absolute_time(start_time=map_time) - else: - assert False, "unknown temporal type!" - # populate the list map_dts_lst.append(map_dts) # Finally register the maps t_unit = {TemporalType.RELATIVE: "seconds", TemporalType.ABSOLUTE: ""} @@ -886,4 +587,3 @@ def register_maps_in_stds( # Restore GRASS verbosity if stds_type == "stvds" and grass_verbosity != "-1": os.environ["GRASS_VERBOSE"] = grass_verbosity - return self diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 5ce3b40e..d0483b65 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -29,16 +29,18 @@ from itzi.providers.grass_interface import GrassInterface -def derived_record_name(dataset_name: str, record_index: int) -> str: - """Return the shared raster/vector child name for one output record.""" +def derived_record_name(dataset_id: str, record_index: int) -> str: + """Return the shared raster/vector child ID for one output record.""" if record_index < 0: raise ValueError("record_index must be non-negative") - return f"{dataset_name}_{record_index:04d}" + name, separator, mapset = dataset_id.partition("@") + return f"{name}_{record_index:04d}{separator}{mapset}" -def derived_drainage_table_names(vector_name: str) -> tuple[str, str]: +def derived_drainage_table_names(vector_id: str) -> tuple[str, str]: """Return the table names created for a drainage vector child.""" - return (f"{vector_name}_node", f"{vector_name}_link") + name = vector_id.partition("@")[0] + return (f"{name}_node", f"{name}_link") class GrassRasterOutputProvider(RasterOutputProvider): @@ -52,13 +54,13 @@ def __init__( temporal_type: TemporalType, ) -> None: self.grass_interface = grass_interface - # user-selected map names. Keys are user-facing names + # Dataset IDs qualified in the current mapset. Keys are output variables. self.out_map_names = out_map_names self.hmin = hmin self.temporal_type = temporal_type - for stds_name in self.out_map_names.values(): + for stds_id in self.out_map_names.values(): self.grass_interface.validate_output_stds_temporal_type( - stds_name=stds_name, + stds_id=stds_id, stds_type="strds", expected_temporal_type=self.temporal_type, ) @@ -68,14 +70,14 @@ def __init__( def _write_array( self, array: np.ndarray, map_key: str, sim_time: datetime | timedelta ) -> None: - map_name = derived_record_name(self.out_map_names[map_key], self.record_counter[map_key]) + map_id = derived_record_name(self.out_map_names[map_key], self.record_counter[map_key]) # write the raster - self.grass_interface.write_raster_map(array, map_name, map_key, self.hmin) + self.grass_interface.write_raster_map(array, map_id, map_key, self.hmin) # Set depth values to null under the given threshold. Temporarily in gis.py # if map_key == "water_depth": # self.grass_interface.set_null(map_name, self.hmin) # add map name and time to the corresponding list - self.output_maplist[map_key].append((map_name, sim_time)) + self.output_maplist[map_key].append((map_id, sim_time)) self.record_counter[map_key] += 1 def write_arrays( @@ -92,11 +94,9 @@ def finalize(self) -> None: self.grass_interface.finalize() # register in GRASS temporal framework for map_key, lst in self.output_maplist.items(): - strds_name = self.out_map_names[map_key] - if strds_name is None: - continue + strds_id = self.out_map_names[map_key] self.grass_interface.register_maps_in_stds( - map_key, strds_name, lst, "strds", self.temporal_type + map_key, strds_id, lst, "strds", self.temporal_type ) @@ -106,14 +106,14 @@ class GrassVectorOutputProvider(VectorOutputProvider): def __init__( self, grass_interface: GrassInterface, - drainage_map_name: str, + drainage_map_id: str, temporal_type: TemporalType, ) -> None: self.grass_interface = grass_interface - self.drainage_map_name = drainage_map_name + self.drainage_map_id = drainage_map_id self.temporal_type = temporal_type self.grass_interface.validate_output_stds_temporal_type( - stds_name=self.drainage_map_name, + stds_id=self.drainage_map_id, stds_type="stvds", expected_temporal_type=self.temporal_type, ) @@ -134,9 +134,9 @@ def write_attributes( """Write drainage simulation data for current time step.""" if self.drainage_topology is None: raise RuntimeError("Drainage attributes cannot be written before topology.") - map_name = derived_record_name(self.drainage_map_name, self.record_counter) - self.grass_interface.write_vector_map(self.drainage_topology, attributes, map_name) - self.vector_drainage_maplist.append((map_name, sim_time)) + map_id = derived_record_name(self.drainage_map_id, self.record_counter) + self.grass_interface.write_vector_map(self.drainage_topology, attributes, map_id) + self.vector_drainage_maplist.append((map_id, sim_time)) self.record_counter += 1 def finalize(self) -> None: @@ -144,7 +144,7 @@ def finalize(self) -> None: if self.vector_drainage_maplist: self.grass_interface.register_maps_in_stds( stds_title="Itzï drainage results", - stds_name=self.drainage_map_name, + stds_id=self.drainage_map_id, map_list=self.vector_drainage_maplist, stds_type="stvds", t_type=self.temporal_type, diff --git a/src/itzi/resolution.py b/src/itzi/resolution.py index f90c878b..7a94824e 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/resolution.py @@ -16,7 +16,7 @@ import hashlib import json -from collections.abc import Callable, Iterable +from collections.abc import Iterable from datetime import datetime, timedelta from pathlib import Path from typing import Literal @@ -50,11 +50,11 @@ def resolve_ensemble( raise EnsembleError("all simulations in an ensemble must use the same GRASS context") with GrassSessionManager(requested_params): - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask - actual_params = GrassInterface.active_grass_params(requested_params) - domain = GrassInterface.read_domain(simulations[0].domain.region) - effective_mask = GrassInterface.resolve_effective_mask(simulations[0].domain.mask) + actual_params = active_grass_params(requested_params) + domain = read_domain(simulations[0].domain.region) + effective_mask = resolve_effective_mask(simulations[0].domain.mask) input_cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] = {} swmm_cache: dict[Path, str] = {} results: list[ResolvedSimulation | ValidationFailure] = [] @@ -112,6 +112,7 @@ def _build_resolved_simulation( swmm_path: Path | None, swmm_digest: str | None, ) -> ResolvedSimulation: + assert actual_params.mapset is not None simulation_id, normalized_payload = _simulation_identity( expanded, actual_params, @@ -120,7 +121,7 @@ def _build_resolved_simulation( input_names, swmm_digest, ) - artifacts = _render_artifacts(expanded, simulation_id) + artifacts = _render_artifacts(expanded, simulation_id, actual_params.mapset) simulation_config = _build_simulation_config( expanded, input_names, @@ -144,12 +145,8 @@ def _resolve_inputs( inputs: dict[str, str], *, cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] | None = None, - resolve_one: Callable[[str], tuple[str, Literal["raster", "strds"]]] | None = None, ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: - if resolve_one is None: - from itzi.providers.grass_interface import GrassInterface - - resolve_one = GrassInterface.resolve_input_identifier + from itzi.grass.utils import resolve_input_identifier resolved: dict[str, str] = {} kinds: dict[str, Literal["raster", "strds"]] = {} @@ -157,7 +154,7 @@ def _resolve_inputs( result = cache.get(identifier) if cache is not None else None if result is None: try: - result = resolve_one(identifier) + result = resolve_input_identifier(identifier) except Exception as error: if cache is not None: cache[identifier] = error @@ -236,6 +233,7 @@ def _simulation_identity( def _render_artifacts( expanded: ExpandedSimulation, simulation_id: str, + mapset: str, ) -> ArtifactSummary: output_map_names: dict[str, str] = {} if expanded.outputs.raster_prefix is not None: @@ -274,7 +272,11 @@ def _render_artifacts( _last_record_index(expanded.time.duration, expanded.time.record_step), ) _validate_protected_file_output(stats_file, (expanded.source.path, _drainage_path(expanded))) - return ArtifactSummary(tuple(sorted(output_map_names.items())), drainage_output, stats_file) + return ArtifactSummary( + tuple(sorted((key, f"{name}@{mapset}") for key, name in output_map_names.items())), + f"{drainage_output}@{mapset}" if drainage_output is not None else None, + stats_file, + ) def _source_relative_path(value: str, source_dir: Path) -> Path: @@ -309,7 +311,7 @@ def _last_record_index(duration: timedelta, record_step: timedelta) -> int: def _validate_output_names( output_map_names: dict[str, str], drainage_output: str | None, last_record_index: int ) -> None: - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.utils import is_clean_name from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name names = list(output_map_names.values()) @@ -320,15 +322,15 @@ def _validate_output_names( raise EnsembleError(f"GRASS output names must be unqualified: {name!r}") if name == "MASK": raise EnsembleError("MASK cannot be used as an output name") - if not GrassInterface.is_clean_name(name): + if not is_clean_name(name): raise EnsembleError(f"invalid GRASS output name {name!r}") child = derived_record_name(name, last_record_index) - if child == "MASK" or not GrassInterface.is_clean_name(child): + if child == "MASK" or not is_clean_name(child): raise EnsembleError(f"invalid derived GRASS output name {child!r}") if drainage_output is not None: child = derived_record_name(drainage_output, last_record_index) for table_name in derived_drainage_table_names(child): - if not GrassInterface.is_clean_name(table_name): + if not is_clean_name(table_name): raise EnsembleError(f"invalid derived drainage table name {table_name!r}") @@ -393,7 +395,6 @@ def validate_resolved_ensemble( raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}") raise EnsembleError(f"simulation ID digest collision {simulation.simulation_id}") identities[simulation.simulation_id] = simulation.normalized_payload - mapset = simulation.grass_params.mapset from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name last_record_index = _last_record_index( @@ -401,24 +402,20 @@ def validate_resolved_ensemble( simulation.simulation_config.record_step, ) for _, name in simulation.artifacts.output_map_names: - _claim_artifact(artifacts, f"{name}@{mapset}", simulation.simulation_id) - _reject_output_source_alias(name, mapset, ensemble_sources) + _claim_artifact(artifacts, name, simulation.simulation_id) + _reject_output_source_alias(name, ensemble_sources) for index in range(last_record_index + 1): child = derived_record_name(name, index) - _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) - _reject_output_source_alias(child, mapset, ensemble_sources) + _claim_artifact(artifacts, child, simulation.simulation_id) + _reject_output_source_alias(child, ensemble_sources) if simulation.artifacts.drainage_output is not None: drainage_name = simulation.artifacts.drainage_output - _claim_artifact( - artifacts, - f"{drainage_name}@{mapset}", - simulation.simulation_id, - ) - _reject_output_source_alias(drainage_name, mapset, ensemble_sources) + _claim_artifact(artifacts, drainage_name, simulation.simulation_id) + _reject_output_source_alias(drainage_name, ensemble_sources) for index in range(last_record_index + 1): child = derived_record_name(drainage_name, index) - _claim_artifact(artifacts, f"{child}@{mapset}", simulation.simulation_id) - _reject_output_source_alias(child, mapset, ensemble_sources) + _claim_artifact(artifacts, child, simulation.simulation_id) + _reject_output_source_alias(child, ensemble_sources) for table_name in derived_drainage_table_names(child): _claim_artifact(artifacts, f"table:{table_name}", simulation.simulation_id) if simulation.artifacts.statistics_file is not None: @@ -442,23 +439,22 @@ def _claim_artifact(artifacts: dict[str, str], artifact: str, simulation_id: str ) -def _reject_output_source_alias(name: str, mapset: str | None, sources: set[str]) -> None: - qualified = f"{name}@{mapset}" - if qualified in sources: - raise EnsembleError(f"output {qualified!r} aliases an ensemble input or mask") +def _reject_output_source_alias(map_id: str, sources: set[str]) -> None: + if map_id in sources: + raise EnsembleError(f"output {map_id!r} aliases an ensemble input or mask") def verify_resolved_simulation(simulation: ResolvedSimulation) -> None: """Re-resolve worker facts and reject a changed GRASS environment.""" - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask - actual = GrassInterface.active_grass_params(simulation.grass_params) + actual = active_grass_params(simulation.grass_params) if actual != simulation.grass_params: raise EnsembleError("GRASS context changed after resolution") - domain = GrassInterface.read_domain(simulation.grass_params.region) + domain = read_domain(simulation.grass_params.region) if domain != simulation.domain_data: raise EnsembleError("GRASS domain changed after resolution") - mask = GrassInterface.resolve_effective_mask(simulation.grass_params.mask) + mask = resolve_effective_mask(simulation.grass_params.mask) if mask != simulation.effective_mask: raise EnsembleError("effective GRASS mask changed after resolution") _verify_input_kinds(simulation.simulation_config.input_map_names, dict(simulation.input_kinds)) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index f7a1d0d9..9bac4454 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -15,15 +15,17 @@ from collections.abc import Mapping from datetime import datetime from time import monotonic -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Literal, Self import numpy as np from itzi_core import SimulationBuilder, SimulationConfig -from itzi_core.providers.csv_mass_balance_output import CSVMassBalanceOutputProvider import itzi.messenger as msgr from itzi.ensemble_models import EffectiveMask from itzi.grass_session import GrassParams, GrassSessionManager +from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider +from itzi.providers.grass_input import GrassRasterInputProvider +from itzi.providers.grass_output import GrassRasterOutputProvider, GrassVectorOutputProvider if TYPE_CHECKING: from itzi_core import Simulation @@ -43,89 +45,108 @@ def __init__( stats_file: str | None = None, effective_mask: EffectiveMask | None = None, input_kinds: Mapping[str, Literal["raster", "strds"]] | None = None, - exclusive_stats: bool = False, ) -> None: + from itzi.grass.utils import ( + check_output_files, + ensure_min_version, + get_current_mapset, + qualify_output_id, + resolve_input_identifier, + ) + + self.grass_params = grass_params + self.hotstart_path = hotstart_path + self.stats_file = stats_file + self.effective_mask = effective_mask + ensure_min_version() + GrassSessionManager.ensure_temporal_initialized() + if input_kinds is None: + resolved_inputs = { + key: resolve_input_identifier(name) + for key, name in sim_config.input_map_names.items() + } + self.input_kinds = {key: kind for key, (_, kind) in resolved_inputs.items()} + input_names = {key: map_id for key, (map_id, _) in resolved_inputs.items()} + else: + self.input_kinds = input_kinds + input_names = sim_config.input_map_names + mapset = get_current_mapset() + self.sim_config = sim_config.model_copy( + update={ + "input_map_names": input_names, + "output_map_names": { + key: qualify_output_id(name, mapset) + for key, name in sim_config.output_map_names.items() + }, + "drainage_output": qualify_output_id(sim_config.drainage_output, mapset) + if sim_config.drainage_output is not None + else None, + } + ) self.g_interface: GrassInterface self.sim: Simulation - # display parameters (if verbose) - msgr.display_sim_param(sim_config) - - # Check GRASS version - from itzi.providers import grass_interface - - grass_interface.GrassInterface.ensure_min_version() - GrassSessionManager.ensure_temporal_initialized() - msgr.debug("GRASS session set") + msgr.display_sim_param(self.sim_config) - # return error if output files exist - output_names = list(sim_config.output_map_names.values()) - if sim_config.drainage_output is not None: - output_names.append(sim_config.drainage_output) - grass_interface.check_output_files(output_names) + output_names = list(self.sim_config.output_map_names.values()) + if self.sim_config.drainage_output is not None: + output_names.append(self.sim_config.drainage_output) + check_output_files(output_names) msgr.debug("Output files OK") + def initialize(self) -> Self: + from itzi.providers.grass_interface import GrassInterface + data_type = np.float32 - # Create the grass_interface object - self.g_interface = grass_interface.GrassInterface( - start_time=sim_config.start_time, - end_time=sim_config.end_time, + self.g_interface = GrassInterface( + start_time=self.sim_config.start_time, + end_time=self.sim_config.end_time, dtype=data_type, - region_id=grass_params.region, - raster_mask_id=grass_params.mask, - effective_mask=(effective_mask.mode, effective_mask.source) - if effective_mask is not None + region_id=self.grass_params.region, + raster_mask_id=self.grass_params.mask, + effective_mask=(self.effective_mask.mode, self.effective_mask.source) + if self.effective_mask is not None else None, non_blocking_write=False, ) - # Create Simulation with GRASS backend msgr.verbose("Setting up GRASS simulation...") - from itzi.providers.grass_input import GrassRasterInputProvider - from itzi.providers.grass_output import ( - GrassRasterOutputProvider, - GrassVectorOutputProvider, - ) raster_input_provider = GrassRasterInputProvider( grass_interface=self.g_interface, - input_map_names=sim_config.input_map_names, - default_start_time=sim_config.start_time, - default_end_time=sim_config.end_time, - input_kinds=input_kinds, + input_map_names=self.sim_config.input_map_names, + default_start_time=self.sim_config.start_time, + default_end_time=self.sim_config.end_time, + input_kinds=self.input_kinds, ) raster_output_provider = GrassRasterOutputProvider( grass_interface=self.g_interface, - out_map_names=sim_config.output_map_names, - hmin=sim_config.surface_flow_parameters.hmin, - temporal_type=sim_config.temporal_type, + out_map_names=self.sim_config.output_map_names, + hmin=self.sim_config.surface_flow_parameters.hmin, + temporal_type=self.sim_config.temporal_type, ) sim_builder = ( - SimulationBuilder(sim_config, self.g_interface.get_npmask(), data_type) + SimulationBuilder(self.sim_config, self.g_interface.get_npmask(), data_type) .with_input_provider(raster_input_provider) .with_raster_output_provider(raster_output_provider) ) - if sim_config.drainage_output is not None: + if self.sim_config.drainage_output is not None: vector_output_provider = GrassVectorOutputProvider( grass_interface=self.g_interface, - temporal_type=sim_config.temporal_type, - drainage_map_name=sim_config.drainage_output, + temporal_type=self.sim_config.temporal_type, + drainage_map_id=self.sim_config.drainage_output, ) sim_builder.with_vector_output_provider(vector_output_provider) - if stats_file: - if exclusive_stats: - from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider - - stats_provider = ExclusiveCSVMassBalanceOutputProvider( - stats_file, overwrite=self.g_interface.overwrite - ) - else: - stats_provider = CSVMassBalanceOutputProvider(stats_file) + if self.stats_file: + stats_provider = ExclusiveCSVMassBalanceOutputProvider( + self.stats_file, overwrite=self.g_interface.overwrite + ) sim_builder.with_mass_balance_output_provider(stats_provider) - if hotstart_path: - sim_builder.with_hotstart(hotstart_path) + if self.hotstart_path: + sim_builder.with_hotstart(self.hotstart_path) self.sim: Simulation = sim_builder.build() # Initialize the simulation self.sim.initialize() + return self def run(self): """Run a full simulation""" @@ -143,7 +164,6 @@ def run(self): sim_start_time, ) last_progress_update = now - # step models self.step() return self diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index dd334d02..d488ac0e 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import datetime, timedelta +import sys from types import SimpleNamespace import pytest @@ -322,7 +323,7 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): ) monkeypatch.setattr("itzi.resolution._validate_output_names", lambda *_: None) - artifacts = _render_artifacts(expanded, "sim-a") + artifacts = _render_artifacts(expanded, "sim-a", "PERMANENT") assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" @@ -348,7 +349,7 @@ def test_later_generated_output_cannot_alias_an_input(): effective_mask=SimpleNamespace(source=None), simulation_config=config, artifacts=SimpleNamespace( - output_map_names=(("water_depth", "result"),), + output_map_names=(("water_depth", "result@PERMANENT"),), drainage_output=None, statistics_file=None, ), @@ -376,7 +377,7 @@ def test_later_generated_output_cannot_alias_an_input(): validate_resolved_ensemble((output, input_simulation)) -def test_input_resolution_cache_reuses_shared_identifiers(): +def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): calls = [] def resolve(identifier): @@ -384,16 +385,17 @@ def resolve(identifier): return f"{identifier}@PERMANENT", "raster" cache = {} + monkeypatch.setitem( + sys.modules, "itzi.grass.utils", SimpleNamespace(resolve_input_identifier=resolve) + ) _resolve_inputs( {"ground_elevation": "shared", "rainfall_rate": "first"}, cache=cache, - resolve_one=resolve, ) resolved, _ = _resolve_inputs( {"ground_elevation": "shared", "rainfall_rate": "second"}, cache=cache, - resolve_one=resolve, ) assert calls == ["shared", "first", "second"] diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index d8ea31aa..b8a62596 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -18,8 +18,10 @@ sys.path.append(grass_python_path) import grass.script as gscript # noqa: E402 -from itzi import SimulationRunner # noqa: E402 -from itzi import grass_session # noqa: E402 +from itzi import ( # noqa: E402 + SimulationRunner, # noqa: E402 + grass_session, # noqa: E402 +) from itzi.configreader import ConfigReader # noqa: E402 TESTS_ROOT = Path.cwd() @@ -172,14 +174,12 @@ def grass_5by5(grass_xy_session, test_data_path): @pytest.fixture(scope="class") def grass_5by5_sim(grass_5by5, test_data_path): - """ """ config_file = os.path.join(test_data_path, "5by5", "5by5.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( conf_data.sim_config, conf_data.grass_params, - stats_file=conf_data.stats_file, - ) + ).initialize() assert isinstance(sim_runner, SimulationRunner) sim_runner.run().finalize() return sim_runner diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 81adee47..c77e9711 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -145,8 +145,8 @@ def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> or "inadequate temporal" in failed["failure"]["detail"] ) output = completed["artifacts"]["rasters"]["water_depth"] - assert "@" not in output - assert gutils.get_mapset_raster(f"{output}_0000", gutils.getenv("MAPSET")) + assert output.endswith(f"@{gutils.getenv('MAPSET')}") + assert gutils.get_mapset_raster(f"{output.partition('@')[0]}_0000", gutils.getenv("MAPSET")) @pytest.mark.forked @@ -179,7 +179,9 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path explicit = interface.get_npmask() assert explicit[:, 0].all() assert not explicit[:, 1:].any() # zero is inside an explicit mask - assert np.isfinite(interface.read_raster_map("n")).all() # bypass active MASK + assert np.isfinite( + interface.read_raster_map(f"n@{gutils.getenv('MAPSET')}") + ).all() # bypass active MASK interface.mask_mode, interface.mask_source = ( "active", f"MASK@{gutils.getenv('MAPSET')}", @@ -324,6 +326,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): ] assert all( member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") + and member["artifacts"]["rasters"]["water_depth"].endswith("@5by5") for member in manifest["members"] ) @@ -380,7 +383,8 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da Path(context["GISDBASE"]) / context["LOCATION_NAME"] / context["MAPSET"] / "cellhd" ) assert { - f"{member['artifacts']['rasters']['water_depth']}_0000" for member in manifest["members"] + f"{member['artifacts']['rasters']['water_depth'].partition('@')[0]}_0000" + for member in manifest["members"] } <= {path.name for path in mapset_path.iterdir()} @@ -552,9 +556,9 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_preflight_checks_existing_drainage_tables(): +def test_preflight_accepts_unoccupied_drainage_id(): + from itzi.grass.utils import get_current_mapset from itzi.preflight import _validate_grass_outputs - from itzi.providers.grass_interface import GrassInterface GrassSessionManager.ensure_temporal_initialized() name = f"drainage_collision_{uuid4().hex[:8]}" @@ -564,18 +568,13 @@ def test_preflight_checks_existing_drainage_tables(): end_time=datetime(2020, 1, 1) + timedelta(minutes=1), record_step=timedelta(seconds=30), output_map_names={}, - drainage_output=name, + drainage_output=f"{name}@{get_current_mapset()}", temporal_type=TemporalType.RELATIVE, ) ) interface = SimpleNamespace( overwrite=False, - get_current_mapset=GrassInterface.get_current_mapset, - format_id=GrassInterface.format_id, validate_output_stds_temporal_type=lambda *_: None, - raster_exists=GrassInterface.raster_exists, - vector_exists=GrassInterface.vector_exists, - stds_exists=GrassInterface.stds_exists, ) _validate_grass_outputs(simulation, interface) diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index 4567debc..de6e70ce 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING, Literal +import numpy as np import pytest if TYPE_CHECKING: @@ -20,18 +21,6 @@ class FakeGrassInterface: def __init__(self) -> None: self.temporal_checks: list[str] = [] - @staticmethod - def format_id(name: str) -> str: - return f"{name}@test" - - @staticmethod - def name_is_stds(map_id: str) -> bool: - return map_id == "series@test" - - @staticmethod - def name_is_map(map_id: str) -> bool: - return map_id == "raster@test" - def stds_temporal_sanity(self, strds_id: str) -> bool: self.temporal_checks.append(strds_id) return True @@ -49,10 +38,21 @@ def raster_list_from_strds(self, strds_id: str) -> list[MapData]: ) def test_input_kinds_share_raster_and_strds_construction( input_kinds: dict[str, Literal["raster", "strds"]] | None, + monkeypatch: pytest.MonkeyPatch, ) -> None: + from itzi.grass import utils from itzi.providers.grass_input import GrassRasterInputProvider from itzi.providers.grass_interface import MapData + monkeypatch.setattr( + utils, + "resolve_input_identifier", + lambda name: (f"{name}@test", "strds" if name == "series" else "raster"), + ) + monkeypatch.setattr(utils, "format_id", lambda name: f"{name}@test") + monkeypatch.setattr(utils, "name_is_stds", lambda map_id: map_id == "series@test") + monkeypatch.setattr(utils, "name_is_map", lambda map_id: map_id == "raster@test") + start = datetime(2020, 1, 1) end = start + timedelta(hours=1) interface = FakeGrassInterface() @@ -68,3 +68,76 @@ def test_input_kinds_share_raster_and_strds_construction( assert provider.map_lists["ground_elevation"] == [MapData("raster@test", start, end)] assert provider.map_lists["rainfall_rate"] == [MapData("series@test", start, end)] assert interface.temporal_checks == ["series@test"] + + +@pytest.mark.forked +def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> None: + import grass.script as gscript + + from itzi.grass.utils import ( + RasterWriteInstructions, + format_id, + output_name, + qualify_output_id, + write_raster_map_blocking, + ) + from itzi.providers.grass_output import derived_record_name + + gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) + gscript.mapcalc("visible_output=1") + gscript.run_command("g.mapset", mapset="child", flags="c") + assert format_id("visible_output") == "visible_output@PERMANENT" + assert format_id("visible_output@PERMANENT") == "visible_output@PERMANENT" + + output_id = qualify_output_id("visible_output", "child") + assert output_id == "visible_output@child" + assert derived_record_name(output_id, 0) == "visible_output_0000@child" + assert output_name(output_id) == "visible_output" + with pytest.raises(ValueError, match="current mapset"): + qualify_output_id("visible_output@PERMANENT", "child") + + gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) + write_raster_map_blocking( + RasterWriteInstructions( + np.ones((3, 3), dtype=np.int32), "visible_output", "CELL", "ground_elevation", 0, False + ) + ) + assert format_id("visible_output") == "visible_output@child" + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_direct_runner_qualifies_ids_before_creating_providers(test_data_path: str) -> None: + from pathlib import Path + + from itzi.configreader import ConfigReader + from itzi.simulation_runner import SimulationRunner + + reader = ConfigReader(str(Path(test_data_path) / "5by5" / "5by5.ini")) + config = reader.sim_config.model_copy( + update={"output_map_names": {"water_depth": "direct_id_check"}} + ) + runner = SimulationRunner(config, reader.grass_params) + + assert runner.sim_config.input_map_names["ground_elevation"] == "z@5by5" + assert runner.sim_config.output_map_names["water_depth"] == "direct_id_check@5by5" + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_vector_writer_uses_bare_name_from_qualified_output_id() -> None: + from grass.pygrass.utils import get_mapset_vector + from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology + + from itzi.providers.grass_interface import GrassInterface + + start = datetime(2020, 1, 1) + with GrassInterface( + start, start + timedelta(seconds=1), np.float32, None, None, non_blocking_write=False + ) as interface: + interface.write_vector_map( + DrainageNetworkTopology(nodes=(), links=()), + DrainageNetworkAttributes(nodes=(), links=()), + "vector_id_check@5by5", + ) + assert get_mapset_vector("vector_id_check", "5by5") == "5by5" diff --git a/tests/grass/test_itzi.py b/tests/grass/test_itzi.py index ecfa1f34..32de2b32 100644 --- a/tests/grass/test_itzi.py +++ b/tests/grass/test_itzi.py @@ -93,7 +93,7 @@ def _build_timed_rain_runner( conf_data.sim_config, conf_data.grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() @pytest.mark.forked @@ -160,10 +160,9 @@ def test_region_mask(test_data_path): sim_runner = SimulationRunner( sim_params, grass_params, - stats_file=conf_data.stats_file, ) # Run simulation - sim_runner.run().finalize() + sim_runner.initialize().run().finalize() # Check temporary mask and region assert ( int(gscript.parse_command("r.univar", map="out_5by5_max_flow_speed_0002", flags="g")["n"]) @@ -219,7 +218,7 @@ def test_fails_when_region_has_no_dem_data(test_data_temp_path): conf_data.sim_config, conf_data.grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() @pytest.mark.forked diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index e09eed33..d64f4f95 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -68,13 +68,13 @@ def test_temporal_fatal_errors_are_raised_as_exceptions() -> None: import grass.temporal as tgis from grass.exceptions import FatalError - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.utils import name_is_stds tgis.set_raise_on_error(False) GrassSessionManager.ensure_temporal_initialized() with pytest.raises(FatalError, match="mapset is missing"): - GrassInterface.name_is_stds("missing_strds") + name_is_stds("missing_strds") assert gscript.get_raise_on_error() is True assert tgis.get_raise_on_error() is True @@ -99,7 +99,7 @@ def test_runner_creation_fails_when_overwriting_strds_with_different_temporal_ty prefix = f"out_overwrite_{existing_temporal_type}_{simulation_temporal_type}_{uuid4().hex[:8]}" initial_runner = _build_runner(test_data_temp_path, prefix, existing_temporal_type) - initial_runner.run().finalize() + initial_runner.initialize().run().finalize() with pytest.raises(RuntimeError, match=r"temporal type"): - _build_runner(test_data_temp_path, prefix, simulation_temporal_type) + _build_runner(test_data_temp_path, prefix, simulation_temporal_type).initialize() From 3390cf66e0740b74874ebec773282ec4ad7c43b9 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 15:25:50 -0600 Subject: [PATCH 47/85] reorganize imports --- src/itzi/ensemble_models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble_models.py index 8254bb1d..5b709e74 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble_models.py @@ -32,11 +32,13 @@ from pydantic import ( BaseModel, ConfigDict, - JsonValue as PydanticJsonValue, StrictStr, TypeAdapter, model_validator, ) +from pydantic import ( + JsonValue as PydanticJsonValue, +) from itzi.grass_session import GrassParams From f89df381af6f29e6f9a56b9f7382619fc836c8d2 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 15:26:57 -0600 Subject: [PATCH 48/85] add copyright notice --- src/itzi/run_plan.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index adf76c1c..13e02604 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -1,4 +1,16 @@ -"""Pure ordered batch loading for YAML streams and deprecated INI files.""" +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" from __future__ import annotations From bb6d04524f0cf67ffc42d32e13ab7cb4532d510d Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 16:06:32 -0600 Subject: [PATCH 49/85] update ty --- pyproject.toml | 2 +- uv.lock | 44 ++++++++++++++++++++++---------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ff6afa1..1182f1c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dev = [ "pandas>=3.0.3", "pre-commit>=4.6", "ruff>=0.15", - "ty>=0.0.55", + "ty>=0.0.84", "sphinx>=9.1.0", "sphinx-argparse>=0.5.2", "sphinx-rtd-theme>=3.1.0", diff --git a/uv.lock b/uv.lock index c33f3d53..7328cc9f 100644 --- a/uv.lock +++ b/uv.lock @@ -383,7 +383,7 @@ dev = [ { name = "sphinx", specifier = ">=9.1.0" }, { name = "sphinx-argparse", specifier = ">=0.5.2" }, { name = "sphinx-rtd-theme", specifier = ">=3.1.0" }, - { name = "ty", specifier = ">=0.0.55" }, + { name = "ty", specifier = ">=0.0.84" }, ] [[package]] @@ -1145,27 +1145,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.56" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/07/fb29aea5235b0aa8ecfc4d1cc6ddf9fba8b863d67d96c6d345694d644c43/ty-0.0.56.tar.gz", hash = "sha256:84d114dc3796361c0fc72945016eabd74d46b9ee64f198cb0e485719704681e5", size = 6050123, upload-time = "2026-07-01T16:44:56.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/48/bce79e7ca5c1cc529d3e0d37ddd1121aea4b68a4f749974ad1cc77161871/ty-0.0.56-py3-none-linux_armv6l.whl", hash = "sha256:186d4a53e15747c947e1ec3d7eec8e345d8e40a1ca10e634c585db52497e87dd", size = 11643066, upload-time = "2026-07-01T16:44:18.374Z" }, - { url = "https://files.pythonhosted.org/packages/80/d1/22555d8a1d719661f10050f3865d877bbf497da908961c75fe22217dd18a/ty-0.0.56-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aae1a980fd9535da0469b7ba2b2e1b54a907743a5e0f442dd57eee9f5bfd034c", size = 11407487, upload-time = "2026-07-01T16:44:20.956Z" }, - { url = "https://files.pythonhosted.org/packages/cf/2d/b3b7a74ce8bc59ef48843ad80179bb0d9598bbd6cfc0d11d519bdf6b1352/ty-0.0.56-py3-none-macosx_11_0_arm64.whl", hash = "sha256:afd3058c0a6c5f241e814734f133008c93ee805f61c9cf4ce7412b8822b5d9ad", size = 10962270, upload-time = "2026-07-01T16:44:22.959Z" }, - { url = "https://files.pythonhosted.org/packages/64/ac/6c2fd7de0304a8a7218a756af74f7e62a5e8540fdb175e0a869e51042345/ty-0.0.56-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:058b52f7a823ac13aae3cae30809dd6b5145794b64d8478f9ef38c75d79b4483", size = 11471406, upload-time = "2026-07-01T16:44:25.327Z" }, - { url = "https://files.pythonhosted.org/packages/50/b6/11d861156861c03c7726b74558f9a0e0092661aff83a4fda1279df28c425/ty-0.0.56-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c66e00c1522add1f2bbdd2e45828c953b35c306b7bef03ec9169c75a63699a0", size = 11445612, upload-time = "2026-07-01T16:44:27.531Z" }, - { url = "https://files.pythonhosted.org/packages/fb/ba/09df108582090f3c0770ec4bc8675affed60248f6793a78d909be16211d9/ty-0.0.56-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40903d71c669a30691b5a5d5728056c7877a1bd6be4f233a38883a8b28cf34d7", size = 12093889, upload-time = "2026-07-01T16:44:29.548Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f7/dbb4b4ccb69cd64c209ae55b1ab788ace8222c2bc1f6845be9e7cbedbf25/ty-0.0.56-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63fe3947fe0c46c69a7d950e6832ee70a9ec17321fefbff3d2e3c20baf9e5bd0", size = 12666337, upload-time = "2026-07-01T16:44:31.586Z" }, - { url = "https://files.pythonhosted.org/packages/86/e9/73f903fe4a3d9ea02f26f57c1eb07e3b1029ec92b0e8c2364718893440e3/ty-0.0.56-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71a0c1a72f9854532e710e119b6871ffe4542c8a65146f1f65dcd78fecd885b4", size = 12280247, upload-time = "2026-07-01T16:44:33.637Z" }, - { url = "https://files.pythonhosted.org/packages/d6/90/cebd222495832f1a00dcd321ba25f3cab804221a4991b992c2178bec68ee/ty-0.0.56-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70d1665596494e24d8ebd198438872b5a56ec3cae5f2bcf6c673be797acc4e3c", size = 11991107, upload-time = "2026-07-01T16:44:36.122Z" }, - { url = "https://files.pythonhosted.org/packages/b7/07/8f7337a07250f42d975cdb6decf47fc5b421e6c7da5e3e7be1e85f63a7e5/ty-0.0.56-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:778f99e51558afc1dbbe48ee38ab6aae7b31390ed8c1a1ef1499b295e9f1e82f", size = 12298970, upload-time = "2026-07-01T16:44:38.243Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b9/a52cd59034a48f5f18c6b155cc2cc36861d874b6d0af204b12c898024c3d/ty-0.0.56-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:867bc5708e0066bb4ff6c7db524bd5deea2676c62bfe71d3303138b3be850af0", size = 11425683, upload-time = "2026-07-01T16:44:40.473Z" }, - { url = "https://files.pythonhosted.org/packages/1d/2e/48e42d33357d52eefb695c0c3fcfc96879b73668a7447d1d1e0ad774fedc/ty-0.0.56-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a6012f4189c928edb330a37deb9930f982380bd4aa7c4b8e0428eec9651c7551", size = 11469258, upload-time = "2026-07-01T16:44:42.513Z" }, - { url = "https://files.pythonhosted.org/packages/d5/01/ad1b4138be1e3fa97863af3925aa2134f17a593240c35dc38c3429fb5ad1/ty-0.0.56-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee83de1a7ff4cc32837ec06134ce391d441bc5b35ecd8d3cfe053f120f3e4c1", size = 11758736, upload-time = "2026-07-01T16:44:44.567Z" }, - { url = "https://files.pythonhosted.org/packages/09/34/9d81967ff240eaa57e9249728ef7b7790747cf6d3c9a98ec86b2cfdcc8ee/ty-0.0.56-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:62619b3b0e2c6248ef30d3f0e2f2217ae9893040585be07f32324242f197cd6f", size = 12100242, upload-time = "2026-07-01T16:44:46.584Z" }, - { url = "https://files.pythonhosted.org/packages/c3/36/f51d4666d2de6cf33c1f3a1fcc4bb6b70b197dd6ceaa491eef71d78fe8e8/ty-0.0.56-py3-none-win32.whl", hash = "sha256:b30687bb5cd9729d34c889a289edf32770388d9bb05243e534e723fb45e0381b", size = 11093759, upload-time = "2026-07-01T16:44:49.171Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b4/8fb5d4acfa4afb152245b20fa263069a7547bd1f8e4bfca4eda280c897d7/ty-0.0.56-py3-none-win_amd64.whl", hash = "sha256:ad4c8c47b6f4e3f9ed3fc0b1a5d650088d229e17dd8f63c1826d6bbe94cc3235", size = 12100327, upload-time = "2026-07-01T16:44:51.26Z" }, - { url = "https://files.pythonhosted.org/packages/b8/fc/6a183e71edde90d0c35c2303f23f7a45b6891d1a2c45daf7b8f869831e19/ty-0.0.56-py3-none-win_arm64.whl", hash = "sha256:57538f273d444a5f1293fa7860e967178afe3917611fc5eff16b64e1204fe0d6", size = 11538780, upload-time = "2026-07-01T16:44:53.8Z" }, +version = "0.0.84" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/f6/34f8869c45ea87fe6a931ffa824b1fe4428167173543d92988d631add355/ty-0.0.84.tar.gz", hash = "sha256:0cefdb0cd5d399418dbe83c349ba605047b7dff815ae6f460c80e8522b40be67", size = 7409537, upload-time = "2026-09-24T13:16:24.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/0c/600648778e1c79133ce6b6a5c4c2531ae16b90c9b30f5041b38b574f61b8/ty-0.0.84-py3-none-linux_armv6l.whl", hash = "sha256:868438e2b9abfc835dd5b5b07ed693db95d8348ac40e3228252511d63ae87695", size = 13935333, upload-time = "2026-09-24T13:15:31.167Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/27f06f49945c36170765fad82076898039ae3873338e9865f876dd62f12c/ty-0.0.84-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b00f39b521f0b559cde0f6abf8dd47213b513b04f33d58efeb9b98486a16bb17", size = 13511328, upload-time = "2026-09-24T13:15:34.446Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/b12dd130e7f30141f0645614f454c925991cd389b1cc6c8284fd6bf32326/ty-0.0.84-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f78c2915567f2ee62ce0c24d86091bc76f81577677755e242d09951db588b9a", size = 13445445, upload-time = "2026-09-24T13:15:37.476Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/f012de4b3c1d9a4773326621380d768b4d48c47e620dc012775c5e01c7d7/ty-0.0.84-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61155cad0921e890d86d1d911a40b3a0e924ef4bdc3b4f3865a6c89c7e3012dc", size = 13471235, upload-time = "2026-09-24T13:15:40.64Z" }, + { url = "https://files.pythonhosted.org/packages/05/32/dca630f5f5353fa7bf67eebccf1549df99b16656d21071d0df6677c826a5/ty-0.0.84-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:417c3bf14947ed16b0b92976ab333c27915590354e96d4f581709068d3811355", size = 13680704, upload-time = "2026-09-24T13:15:43.817Z" }, + { url = "https://files.pythonhosted.org/packages/44/8d/33702c8f62f05ad45b33644fb914d483de6a0a7bc8e51884e9b8773e6d69/ty-0.0.84-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e06ef117759f654f5379efe9ae79bda8a186016c016f1023d05c34f8e15c1546", size = 14518923, upload-time = "2026-09-24T13:15:46.802Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a1/da3246e6ef2ae0e842bca4058b25c54bee55ab058ffff4ee54e72517ec01/ty-0.0.84-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a05d1d53db8b64410ab38c6e32c5898510584869d1d5dd9c0605db81c9aef7df", size = 15027731, upload-time = "2026-09-24T13:15:49.755Z" }, + { url = "https://files.pythonhosted.org/packages/20/17/b0099098a560aeb4f9fde6c912f9a81622c3e44e6555e1fbe322b368ec31/ty-0.0.84-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b92400494ab855cfb1204ac1101e3f53b2d09a8e21d171758b2cf9acdab1809e", size = 14748260, upload-time = "2026-09-24T13:15:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/60/d5/32cd67aedb271f006e716ad093806c96c5d7e2af86be8255469dbd3d76c8/ty-0.0.84-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af17eb391ae3027fed9be1780bea555c2b8a25ba68e184c627b4981b2c9eaab7", size = 14205858, upload-time = "2026-09-24T13:15:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/638ac62ca2e8eca1b92e3e16273085744fada728b993500819b5c258ca15/ty-0.0.84-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:44cf06af0e7ec78ea6d8ff33a3450fc19627756a4fba681f7410d4e3ef63047a", size = 14573196, upload-time = "2026-09-24T13:15:59.165Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f6/9ee81c5d8921e5fd5f10889ae563c95cd3272b31df2ada528919ca1cef6b/ty-0.0.84-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c6e56b443ff9ba462e34bfbc1e582004c2719ed35f8f93a433744b0ba0afb2d9", size = 13480760, upload-time = "2026-09-24T13:16:02.313Z" }, + { url = "https://files.pythonhosted.org/packages/f6/83/fbe5d1177667ae16d84c3242846b143809c9cc09568b333e5a9b4d5908f6/ty-0.0.84-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8b6562fcc48820c1030e2a4509ef2c14ef93043ca4860280074314ac3a825b95", size = 13696346, upload-time = "2026-09-24T13:16:05.391Z" }, + { url = "https://files.pythonhosted.org/packages/86/9a/1b4ac9ddc34cc0ab4624bc219825d66af52f1ad87f4778ad08ad03fa06d5/ty-0.0.84-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dee7421451efcbd70f03e95a8f2c8d9e5ec3d35edcd27b4095ee00ee062abfc4", size = 13959410, upload-time = "2026-09-24T13:16:08.661Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/70e289437a67dfd686056ab4fd91880261a8233033a6a95d29393741fb97/ty-0.0.84-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:31e96307cf621babe969307dd657783d672e2d8bdbd06493710da635bf63fa07", size = 14350294, upload-time = "2026-09-24T13:16:11.865Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f4/ae7dbd118425c850981d23efb850f0d9d7f6b8d2b852bde2767434f668b4/ty-0.0.84-py3-none-win32.whl", hash = "sha256:4bdc85f91cbaae35825f2b17bb4ac5de0563367875cb0f42b2bf07a1bf7ff2ba", size = 13176439, upload-time = "2026-09-24T13:16:15.049Z" }, + { url = "https://files.pythonhosted.org/packages/29/ec/994d87252f065dda5a500bc8afc726d18f6582d81d6f4ff1feb8a267c800/ty-0.0.84-py3-none-win_amd64.whl", hash = "sha256:71cc9aa7a7c89d4e12c6a814e9c0c3310ad5bee44efedbd6f5a42d91e9ac0f0d", size = 13962879, upload-time = "2026-09-24T13:16:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, ] [[package]] From 6e79a6bb3669dbc208cf4e3be7a8db01618b2233 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 16:45:12 -0600 Subject: [PATCH 50/85] refactor: move grass and ensemble into their own submodules --- src/itzi/bmi_itzi.py | 2 +- src/itzi/configreader.py | 2 +- src/itzi/ensemble/__init__.py | 11 +++ .../models.py} | 2 +- src/itzi/{ => ensemble}/preflight.py | 15 ++-- src/itzi/{ => ensemble}/resolution.py | 7 +- .../schema.py} | 2 +- src/itzi/{ensemble.py => ensemble/yaml.py} | 4 +- src/itzi/grass/{___init__.py => __init__.py} | 0 .../grass_interface.py => grass/interface.py} | 16 ++++- src/itzi/grass/names.py | 27 ++++++++ .../{grass_session.py => grass/session.py} | 0 src/itzi/grass/utils.py | 65 ++++++++++++++++-- src/itzi/itzi.py | 10 +-- src/itzi/manifest.py | 2 +- src/itzi/providers/grass_input.py | 68 ++----------------- src/itzi/providers/grass_output.py | 18 +---- src/itzi/run_plan.py | 2 +- src/itzi/simulation_runner.py | 8 +-- tests/cli/test_cli.py | 9 ++- tests/cli/test_ensemble_config.py | 8 +-- tests/grass/conftest.py | 6 +- tests/grass/test_ensemble.py | 8 +-- tests/grass/test_grass_input.py | 8 +-- tests/grass/test_temporal_overwrite.py | 2 +- tests/test_data/5by5/5by5.ini | 3 - tests/test_data/5by5/5by5_mask.ini | 3 - tests/test_grass_session.py | 10 +-- 28 files changed, 175 insertions(+), 143 deletions(-) create mode 100644 src/itzi/ensemble/__init__.py rename src/itzi/{ensemble_models.py => ensemble/models.py} (99%) rename src/itzi/{ => ensemble}/preflight.py (91%) rename src/itzi/{ => ensemble}/resolution.py (98%) rename src/itzi/{ensemble_schema.py => ensemble/schema.py} (99%) rename src/itzi/{ensemble.py => ensemble/yaml.py} (99%) rename src/itzi/grass/{___init__.py => __init__.py} (100%) rename src/itzi/{providers/grass_interface.py => grass/interface.py} (98%) create mode 100644 src/itzi/grass/names.py rename src/itzi/{grass_session.py => grass/session.py} (100%) diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 2108b2d4..2059972c 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -22,7 +22,7 @@ from itzi_core import ARRAY_DEFINITIONS, ArrayCategory from itzi.configreader import ConfigReader -from itzi.grass_session import GrassSessionManager +from itzi.grass.session import GrassSessionManager from itzi.simulation_runner import SimulationRunner diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index 2354ae14..e3475aee 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -32,7 +32,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError import itzi.messenger as msgr -from itzi.grass_session import GrassParams +from itzi.grass.session import GrassParams DEPRECATED_INPUT_ALIASES: list[tuple[str, str]] = [ # (old, new) diff --git a/src/itzi/ensemble/__init__.py b/src/itzi/ensemble/__init__.py new file mode 100644 index 00000000..25c82a2c --- /dev/null +++ b/src/itzi/ensemble/__init__.py @@ -0,0 +1,11 @@ +"""Configuration and execution plans for YAML ensembles.""" + +from itzi.ensemble.yaml import ( + expand_yaml_document as expand_yaml_document, +) +from itzi.ensemble.yaml import ( + load_yaml_stream as load_yaml_stream, +) +from itzi.ensemble.yaml import ( + normalize_time as normalize_time, +) diff --git a/src/itzi/ensemble_models.py b/src/itzi/ensemble/models.py similarity index 99% rename from src/itzi/ensemble_models.py rename to src/itzi/ensemble/models.py index 5b709e74..06800b2a 100644 --- a/src/itzi/ensemble_models.py +++ b/src/itzi/ensemble/models.py @@ -40,7 +40,7 @@ JsonValue as PydanticJsonValue, ) -from itzi.grass_session import GrassParams +from itzi.grass.session import GrassParams GREEN_AMPT_KEYS = ( "effective_porosity", diff --git a/src/itzi/preflight.py b/src/itzi/ensemble/preflight.py similarity index 91% rename from src/itzi/preflight.py rename to src/itzi/ensemble/preflight.py index 0c9878b5..0b708ea8 100644 --- a/src/itzi/preflight.py +++ b/src/itzi/ensemble/preflight.py @@ -18,21 +18,20 @@ import numpy as np -from itzi.ensemble_models import EnsembleError +from itzi.ensemble.models import EnsembleError from itzi.manifest import validate_file_destination -from itzi.providers.grass_input import resolve_input_map_lists -from itzi.providers.grass_output import derived_record_name -from itzi.resolution import _last_record_index, verify_resolved_simulation +from itzi.grass.names import derived_record_name +from itzi.ensemble.resolution import _last_record_index, verify_resolved_simulation if TYPE_CHECKING: - from itzi.ensemble_models import ResolvedSimulation - from itzi.providers.grass_interface import GrassInterface + from itzi.ensemble.models import ResolvedSimulation + from itzi.grass.interface import GrassInterface def preflight_simulation(simulation: ResolvedSimulation) -> None: """Validate one resolved simulation without creating user artifacts.""" from itzi.grass.utils import ensure_min_version - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.interface import GrassInterface verify_resolved_simulation(simulation) ensure_min_version() @@ -60,6 +59,8 @@ def preflight_simulation(simulation: ResolvedSimulation) -> None: def _validate_inputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + from itzi.grass.utils import resolve_input_map_lists + config = simulation.simulation_config mask = interface.get_npmask() if np.all(mask): diff --git a/src/itzi/resolution.py b/src/itzi/ensemble/resolution.py similarity index 98% rename from src/itzi/resolution.py rename to src/itzi/ensemble/resolution.py index 7a94824e..79692fb1 100644 --- a/src/itzi/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -24,7 +24,7 @@ from itzi_core import DomainData, SimulationConfig, SurfaceFlowParameters from pydantic import ValidationError -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( ArtifactSummary, EffectiveMask, EnsembleError, @@ -34,7 +34,8 @@ format_duration, render_template, ) -from itzi.grass_session import GrassParams, GrassSessionManager +from itzi.grass.session import GrassParams, GrassSessionManager +from itzi.grass.names import derived_drainage_table_names, derived_record_name def resolve_ensemble( @@ -312,7 +313,6 @@ def _validate_output_names( output_map_names: dict[str, str], drainage_output: str | None, last_record_index: int ) -> None: from itzi.grass.utils import is_clean_name - from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name names = list(output_map_names.values()) if drainage_output is not None: @@ -395,7 +395,6 @@ def validate_resolved_ensemble( raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}") raise EnsembleError(f"simulation ID digest collision {simulation.simulation_id}") identities[simulation.simulation_id] = simulation.normalized_payload - from itzi.providers.grass_output import derived_drainage_table_names, derived_record_name last_record_index = _last_record_index( simulation.simulation_config.end_time - simulation.simulation_config.start_time, diff --git a/src/itzi/ensemble_schema.py b/src/itzi/ensemble/schema.py similarity index 99% rename from src/itzi/ensemble_schema.py rename to src/itzi/ensemble/schema.py index 2b53c692..e3e94be1 100644 --- a/src/itzi/ensemble_schema.py +++ b/src/itzi/ensemble/schema.py @@ -26,7 +26,7 @@ ) from pydantic import JsonValue as PydanticJsonValue -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( DomainConfig, StrictModel, _canonical_json, diff --git a/src/itzi/ensemble.py b/src/itzi/ensemble/yaml.py similarity index 99% rename from src/itzi/ensemble.py rename to src/itzi/ensemble/yaml.py index a3f96617..93510b2a 100644 --- a/src/itzi/ensemble.py +++ b/src/itzi/ensemble/yaml.py @@ -30,7 +30,7 @@ from yaml.resolver import BaseResolver import itzi.messenger as msgr -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( DIRECT_INPUT_KEYS, MAX_ENSEMBLE_MEMBERS, SURFACE_FLOW_DEFAULTS, @@ -48,7 +48,7 @@ parse_duration, render_template, ) -from itzi.ensemble_schema import ( +from itzi.ensemble.schema import ( ConstantInfiltration, DrainageSweepConfig, GreenAmptInfiltration, diff --git a/src/itzi/grass/___init__.py b/src/itzi/grass/__init__.py similarity index 100% rename from src/itzi/grass/___init__.py rename to src/itzi/grass/__init__.py diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/grass/interface.py similarity index 98% rename from src/itzi/providers/grass_interface.py rename to src/itzi/grass/interface.py index 75d57e2f..69dd040e 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/grass/interface.py @@ -29,7 +29,7 @@ from grass.pygrass.vector import VectorTopo from grass.pygrass.vector.geometry import Line, Point from grass.pygrass.vector.table import Link, Table -from itzi_core import TemporalType +from itzi_core import DomainData, TemporalType from itzi_core.data_containers import ( DrainageLinkAttributes, DrainageNetworkAttributes, @@ -39,10 +39,12 @@ import itzi.messenger as msgr from itzi.grass.utils import ( + MapData, RasterMapType, RasterWriteInstructions, STDSType, format_id, + get_crs_wkt, has_mask, name_is_map, output_name, @@ -51,7 +53,6 @@ split_identifier, write_raster_map_blocking, ) -from itzi.providers.grass_input import MapData class DBLinkDescription(NamedTuple): @@ -227,6 +228,17 @@ def _select_effective_mask( return "active", f"MASK@{gutils.getenv('MAPSET')}" return "none", None + def get_domain_data(self) -> DomainData: + return DomainData( + north=self.region.north, + south=self.region.south, + east=self.region.east, + west=self.region.west, + rows=self.region.rows, + cols=self.region.cols, + crs_wkt=get_crs_wkt(), + ) + def get_npmask(self) -> np.ndarray: """Return a boolean numpy ndarray where True is outside the domain.""" if self.mask_mode == "none": diff --git a/src/itzi/grass/names.py b/src/itzi/grass/names.py new file mode 100644 index 00000000..21793d88 --- /dev/null +++ b/src/itzi/grass/names.py @@ -0,0 +1,27 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + + +def derived_record_name(dataset_id: str, record_index: int) -> str: + """Return the shared raster/vector child ID for one output record.""" + if record_index < 0: + raise ValueError("record_index must be non-negative") + name, separator, mapset = dataset_id.partition("@") + return f"{name}_{record_index:04d}{separator}{mapset}" + + +def derived_drainage_table_names(vector_id: str) -> tuple[str, str]: + """Return the table names created for a drainage vector child.""" + name = vector_id.partition("@")[0] + return (f"{name}_node", f"{name}_link") diff --git a/src/itzi/grass_session.py b/src/itzi/grass/session.py similarity index 100% rename from src/itzi/grass_session.py rename to src/itzi/grass/session.py diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index 9819bda8..e911b70a 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -1,5 +1,5 @@ """ -Copyright (C) 2015-2026 Laurent Courty +Copyright (C) 2026 Laurent Courty This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -12,10 +12,14 @@ GNU General Public License for more details. """ +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime from pathlib import Path from queue import Queue from threading import Lock -from typing import Literal, NamedTuple +from typing import TYPE_CHECKING, Literal, NamedTuple import grass.pygrass.utils as gutils import grass.script as gscript @@ -24,11 +28,14 @@ from grass.pygrass import raster from grass.pygrass.gis import Mapset from grass.pygrass.gis.region import Region -from itzi_core import DomainData +from itzi_core import INPUT_ARRAY_KEYS, DomainData import itzi.messenger as msgr -from itzi.ensemble_models import EffectiveMask, EnsembleError -from itzi.grass_session import GrassParams +from itzi.ensemble.models import EffectiveMask, EnsembleError +from itzi.grass.session import GrassParams + +if TYPE_CHECKING: + from itzi.grass.interface import GrassInterface MIN_GRASS_VERSION = (8, 4) @@ -64,6 +71,12 @@ class RasterWriteInstructions(NamedTuple): overwrite: bool +class MapData(NamedTuple): + id: str + start_time: datetime + end_time: datetime + + def file_exists(name: str) -> bool: """Return True if an output exists in the current mapset.""" if not name: @@ -175,6 +188,10 @@ def name_is_stds(stds_id: str) -> bool: return bool(tgis.SpaceTimeRasterDataset(stds_id).is_in_db()) +def get_crs_wkt() -> str: + return gscript.read_command("g.proj", flags="fw") + + def name_is_map(map_id: str) -> bool: """return True if the given name is a map in the grass database False if not @@ -224,7 +241,7 @@ def read_domain(region_id: str | None) -> DomainData: west=region.west, rows=region.rows, cols=region.cols, - crs_wkt=gscript.read_command("g.proj", flags="fw"), + crs_wkt=get_crs_wkt(), ) finally: if using_temp_region: @@ -307,3 +324,39 @@ def replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarra null_sentinel = array.dtype.type(np.iinfo(np.int32).min) array[array == null_sentinel] = np.nan return array + + +def resolve_input_map_lists( + grass_interface: GrassInterface, + map_names: Mapping[str, str | None], + start_time: datetime, + end_time: datetime, + input_kinds: Mapping[str, Literal["raster", "strds"]] | None, +) -> dict[str, list[MapData] | None]: + """Resolve provider-ready raster lists without creating an output provider.""" + invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) + if invalid_input_keys: + raise ValueError(f"Invalid input keys found: {', '.join(invalid_input_keys)}") + + map_lists: dict[str, list[MapData] | None] = {arr_key: None for arr_key in INPUT_ARRAY_KEYS} + for key, map_name in map_names.items(): + if not map_name: + continue + kind = input_kinds.get(key) if input_kinds is not None else None + if kind is None: + map_id, kind = resolve_input_identifier(map_name) + elif kind == "raster": + map_id = format_id(map_name) + else: + map_id = map_name if "@" in map_name else resolve_input_identifier(map_name)[0] + if kind == "strds": + if not name_is_stds(map_id): + msgr.fatal(f"STRDS input <{map_id}> is no longer available") + elif kind == "raster" and not name_is_map(map_id): + msgr.fatal(f"raster input <{map_id}> is no longer available") + if kind == "strds": + map_list = grass_interface.raster_list_from_strds(map_id) + else: + map_list = [MapData(id=map_id, start_time=start_time, end_time=end_time)] + map_lists[key] = map_list + return map_lists diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 375800d8..48800ba8 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -39,14 +39,14 @@ import itzi.messenger as msgr from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( EnsembleError, ExpandedSimulation, ResolvedEnsemble, ResolvedSimulation, ValidationFailure, ) -from itzi.grass_session import GrassSessionManager +from itzi.grass.session import GrassSessionManager from itzi.manifest import ( _create_manifest, _initial_member_states, @@ -55,7 +55,7 @@ _validate_manifest_destination, ) from itzi.messenger import VerbosityLevel -from itzi.resolution import ( +from itzi.ensemble.resolution import ( resolve_ensemble, validate_resolved_ensemble, verify_resolved_simulation, @@ -153,7 +153,7 @@ def preflight_worker(simulation: ResolvedSimulation) -> str | None: msgr._itzi_logger.set_verbosity(msgr.verbosity()) try: with GrassSessionManager(simulation.grass_params): - from itzi.preflight import preflight_simulation + from itzi.ensemble.preflight import preflight_simulation preflight_simulation(simulation) except Exception as error: @@ -167,7 +167,7 @@ def preflight_ensemble_worker(simulations: tuple[ResolvedSimulation, ...]) -> di msgr._itzi_logger.set_verbosity(msgr.verbosity()) failures: dict[str, str] = {} with GrassSessionManager(simulations[0].grass_params): - from itzi.preflight import preflight_simulation + from itzi.ensemble.preflight import preflight_simulation for simulation in simulations: try: diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py index bf76d153..7fdc7d88 100644 --- a/src/itzi/manifest.py +++ b/src/itzi/manifest.py @@ -22,7 +22,7 @@ import yaml -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( EnsembleError, ExpandedEnsemble, JsonValue, diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index d41d2ea3..97c29c12 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,24 +15,16 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Literal, NamedTuple +from typing import TYPE_CHECKING, Literal import numpy as np -from itzi_core import INPUT_ARRAY_KEYS, DomainData +from itzi_core import DomainData from itzi_core.providers import RasterInputProvider -import itzi.messenger as msgr - if TYPE_CHECKING: from datetime import datetime - from itzi.providers.grass_interface import GrassInterface - - -class MapData(NamedTuple): - id: str - start_time: datetime - end_time: datetime + from itzi.grass.interface import GrassInterface class GrassRasterInputProvider(RasterInputProvider): @@ -44,6 +36,8 @@ def __init__( default_end_time: datetime, input_kinds: Mapping[str, Literal["raster", "strds"]] | None, ) -> None: + from itzi.grass.utils import resolve_input_map_lists + self.grass_interface = grass_interface self.start_time = default_start_time self.end_time = default_end_time @@ -56,18 +50,7 @@ def __init__( ) def get_domain_data(self) -> DomainData: - """Return a DomainData object""" - import grass.script as gscript - - return DomainData( - north=self.grass_interface.region.north, - south=self.grass_interface.region.south, - east=self.grass_interface.region.east, - west=self.grass_interface.region.west, - rows=self.grass_interface.region.rows, - cols=self.grass_interface.region.cols, - crs_wkt=gscript.read_command("g.proj", flags="fw"), - ) + return self.grass_interface.get_domain_data() def get_array( self, map_key: str, current_time: datetime @@ -92,42 +75,3 @@ def get_array( # No gap is expected here: GRASS temporal sanity is checked at init. # An in-range lookup should always hit one map. raise ValueError(f"No map found for {map_key} at time {current_time}") - - -def resolve_input_map_lists( - grass_interface: GrassInterface, - map_names: Mapping[str, str | None], - start_time: datetime, - end_time: datetime, - input_kinds: Mapping[str, Literal["raster", "strds"]] | None, -) -> Mapping[str, list[MapData] | None]: - """Resolve provider-ready raster lists without creating an output provider.""" - from itzi.grass.utils import format_id, name_is_map, name_is_stds, resolve_input_identifier - - invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) - if invalid_input_keys: - raise ValueError(f"Invalid input keys found: {', '.join(invalid_input_keys)}") - - map_lists: dict[str, list[MapData] | None] = {arr_key: None for arr_key in INPUT_ARRAY_KEYS} - for key, map_name in map_names.items(): - if not map_name: - continue - kind = input_kinds.get(key) if input_kinds is not None else None - if kind is None: - map_id, kind = resolve_input_identifier(map_name) - elif kind == "raster": - map_id = format_id(map_name) - else: - map_id = map_name if "@" in map_name else resolve_input_identifier(map_name)[0] - if kind == "strds": - if not name_is_stds(map_id): - msgr.fatal(f"STRDS input <{map_id}> is no longer available") - elif kind == "raster": - if not name_is_map(map_id): - msgr.fatal(f"raster input <{map_id}> is no longer available") - if kind == "strds": - map_list = grass_interface.raster_list_from_strds(map_id) - else: - map_list = [MapData(id=map_id, start_time=start_time, end_time=end_time)] - map_lists[key] = map_list - return map_lists diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index d0483b65..dc8ac9b4 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -20,27 +20,15 @@ import numpy as np from itzi_core.providers import RasterOutputProvider, VectorOutputProvider +from itzi.grass.names import derived_record_name + if TYPE_CHECKING: from datetime import datetime, timedelta from itzi_core import TemporalType from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology - from itzi.providers.grass_interface import GrassInterface - - -def derived_record_name(dataset_id: str, record_index: int) -> str: - """Return the shared raster/vector child ID for one output record.""" - if record_index < 0: - raise ValueError("record_index must be non-negative") - name, separator, mapset = dataset_id.partition("@") - return f"{name}_{record_index:04d}{separator}{mapset}" - - -def derived_drainage_table_names(vector_id: str) -> tuple[str, str]: - """Return the table names created for a drainage vector child.""" - name = vector_id.partition("@")[0] - return (f"{name}_node", f"{name}_link") + from itzi.grass.interface import GrassInterface class GrassRasterOutputProvider(RasterOutputProvider): diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py index 13e02604..24971bc5 100644 --- a/src/itzi/run_plan.py +++ b/src/itzi/run_plan.py @@ -22,7 +22,7 @@ import itzi.messenger as msgr from itzi.configreader import ConfigReader from itzi.ensemble import load_yaml_stream -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( DIRECT_INPUT_KEYS, GREEN_AMPT_KEYS, DocumentFailure, diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 9bac4454..b5a92483 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -21,8 +21,8 @@ from itzi_core import SimulationBuilder, SimulationConfig import itzi.messenger as msgr -from itzi.ensemble_models import EffectiveMask -from itzi.grass_session import GrassParams, GrassSessionManager +from itzi.ensemble.models import EffectiveMask +from itzi.grass.session import GrassParams, GrassSessionManager from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider from itzi.providers.grass_input import GrassRasterInputProvider from itzi.providers.grass_output import GrassRasterOutputProvider, GrassVectorOutputProvider @@ -30,7 +30,7 @@ if TYPE_CHECKING: from itzi_core import Simulation - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.interface import GrassInterface class SimulationRunner: @@ -95,7 +95,7 @@ def __init__( msgr.debug("Output files OK") def initialize(self) -> Self: - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.interface import GrassInterface data_type = np.float32 self.g_interface = GrassInterface( diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 032a3548..f59808f6 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -10,7 +10,7 @@ import itzi.messenger as msgr from itzi.cli_parser import build_parser -from itzi.ensemble_models import ResolvedEnsemble, ResolvedSimulation +from itzi.ensemble.models import ResolvedEnsemble, ResolvedSimulation from itzi.itzi import ( VerbosityLevel, _preflight_ensemble, @@ -282,9 +282,9 @@ def preflight(simulation): raise ValueError("invalid input") monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) - fake_preflight = ModuleType("itzi.preflight") + fake_preflight = ModuleType("itzi.ensemble.preflight") fake_preflight.preflight_simulation = preflight - monkeypatch.setitem(sys.modules, "itzi.preflight", fake_preflight) + monkeypatch.setitem(sys.modules, "itzi.ensemble.preflight", fake_preflight) simulations = tuple( SimpleNamespace(simulation_id=simulation_id, grass_params="shared") for simulation_id in ("good", "bad", "another") @@ -415,6 +415,9 @@ def __init__(self, *args, **kwargs): runner_arguments["args"] = args runner_arguments["kwargs"] = kwargs + def initialize(self): + return self + def run(self): return self diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index d488ac0e..eb0f32f9 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -11,20 +11,20 @@ from pydantic import ValidationError from itzi.ensemble import load_yaml_stream -from itzi.ensemble_models import ( +from itzi.ensemble.models import ( MAX_ENSEMBLE_MEMBERS, format_duration, parse_duration, render_template, ) -from itzi.ensemble_schema import ( +from itzi.ensemble.schema import ( EnsembleMetadata, InputSweepConfig, ManifestOutputs, OptionSweepConfig, StatisticsOutputs, ) -from itzi.resolution import ( +from itzi.ensemble.resolution import ( _last_record_index, _render_artifacts, _resolve_inputs, @@ -321,7 +321,7 @@ def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): drainage=None, time=SimpleNamespace(duration=timedelta(hours=1), record_step=timedelta(minutes=5)), ) - monkeypatch.setattr("itzi.resolution._validate_output_names", lambda *_: None) + monkeypatch.setattr("itzi.ensemble.resolution._validate_output_names", lambda *_: None) artifacts = _render_artifacts(expanded, "sim-a", "PERMANENT") diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index b8a62596..f675144a 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -20,9 +20,9 @@ from itzi import ( # noqa: E402 SimulationRunner, # noqa: E402 - grass_session, # noqa: E402 ) from itzi.configreader import ConfigReader # noqa: E402 +from itzi.grass import session # noqa: E402 TESTS_ROOT = Path.cwd() TEST_PROCESS = os.getpid() @@ -32,8 +32,8 @@ def stop_temporal_subprocesses_after_test() -> Iterator[None]: """Forked tests must stop their own GRASS RPC children before os._exit.""" yield - if os.getpid() != TEST_PROCESS and grass_session._initialized_temporal_session is not None: - session_pid = grass_session._initialized_temporal_session[0] + if os.getpid() != TEST_PROCESS and session._initialized_temporal_session is not None: + session_pid = session._initialized_temporal_session[0] if session_pid == os.getpid(): import grass.temporal as tgis diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index c77e9711..cdc570d6 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -15,8 +15,8 @@ from itzi_core import TemporalType from itzi.cli_parser import build_parser -from itzi.ensemble_models import ResolvedSimulation, ValidationFailure -from itzi.grass_session import GrassSessionManager +from itzi.ensemble.models import ResolvedSimulation, ValidationFailure +from itzi.grass.session import GrassSessionManager from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, preflight_worker from itzi.run_plan import load_batch @@ -154,7 +154,7 @@ def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path) -> None: from grass.pygrass import utils as gutils - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.interface import GrassInterface gscript.mapcalc("stage_mask_values=if(col()==1, null(), if(col()==2, 0, 1))") gscript.run_command("g.copy", raster="stage_mask_values,MASK") @@ -558,7 +558,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.usefixtures("grass_5by5") def test_preflight_accepts_unoccupied_drainage_id(): from itzi.grass.utils import get_current_mapset - from itzi.preflight import _validate_grass_outputs + from itzi.ensemble.preflight import _validate_grass_outputs GrassSessionManager.ensure_temporal_initialized() name = f"drainage_collision_{uuid4().hex[:8]}" diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index de6e70ce..cfde66fb 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -7,7 +7,7 @@ import pytest if TYPE_CHECKING: - from itzi.providers.grass_interface import MapData + from itzi.grass.utils import MapData @pytest.fixture(scope="module", autouse=True) @@ -26,7 +26,7 @@ def stds_temporal_sanity(self, strds_id: str) -> bool: return True def raster_list_from_strds(self, strds_id: str) -> list[MapData]: - from itzi.providers.grass_interface import MapData + from itzi.grass.utils import MapData self.stds_temporal_sanity(strds_id) start = datetime(2020, 1, 1) @@ -42,7 +42,7 @@ def test_input_kinds_share_raster_and_strds_construction( ) -> None: from itzi.grass import utils from itzi.providers.grass_input import GrassRasterInputProvider - from itzi.providers.grass_interface import MapData + from itzi.grass.utils import MapData monkeypatch.setattr( utils, @@ -129,7 +129,7 @@ def test_vector_writer_uses_bare_name_from_qualified_output_id() -> None: from grass.pygrass.utils import get_mapset_vector from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.interface import GrassInterface start = datetime(2020, 1, 1) with GrassInterface( diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index d64f4f95..97b47e53 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -9,7 +9,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader -from itzi.grass_session import GrassSessionManager +from itzi.grass.session import GrassSessionManager from itzi_core.const import TemporalType diff --git a/tests/test_data/5by5/5by5.ini b/tests/test_data/5by5/5by5.ini index a8b49768..f7b1c622 100644 --- a/tests/test_data/5by5/5by5.ini +++ b/tests/test_data/5by5/5by5.ini @@ -11,9 +11,6 @@ record_step = 00:00:30 prefix = out_5by5 values = water_depth, max_water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, froude, flow_rate_x, flow_rate_y, created_volume -[statistics] -stats_file = 5by5.csv - [options] hmin = 0.0001 dtmax = 0.3 diff --git a/tests/test_data/5by5/5by5_mask.ini b/tests/test_data/5by5/5by5_mask.ini index 4feaac48..c29e28a9 100644 --- a/tests/test_data/5by5/5by5_mask.ini +++ b/tests/test_data/5by5/5by5_mask.ini @@ -11,9 +11,6 @@ record_step = 00:00:30 prefix = out_5by5 values = water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, flow_rate_x, flow_rate_y -[statistics] -stats_file = 5by5.csv - [options] hmin = 0.001 dtmax = 0.3 diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index d9f53099..e5200c63 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -2,7 +2,7 @@ import sys from types import ModuleType, SimpleNamespace -from itzi.grass_session import GrassParams, GrassSessionManager +from itzi.grass.session import GrassParams, GrassSessionManager class FakeGrassSession: @@ -21,11 +21,11 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: grass_package = ModuleType("grass") grass_package.script = grass_script - monkeypatch.setattr("itzi.grass_session.importlib.util.find_spec", lambda _: None) - monkeypatch.setattr("itzi.grass_session.os.access", lambda *_: True) + monkeypatch.setattr("itzi.grass.session.importlib.util.find_spec", lambda _: None) + monkeypatch.setattr("itzi.grass.session.os.access", lambda *_: True) monkeypatch.setattr(GrassSessionManager, "ensure_temporal_initialized", lambda self: None) monkeypatch.setattr( - "itzi.grass_session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" + "itzi.grass.session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" ) monkeypatch.delenv("GISRC", raising=False) monkeypatch.setattr( @@ -70,7 +70,7 @@ def test_temporal_initialization_once_per_active_session(monkeypatch) -> None: monkeypatch.setitem(sys.modules, "grass", package) monkeypatch.setitem(sys.modules, "grass.script", script) monkeypatch.setitem(sys.modules, "grass.temporal", temporal) - monkeypatch.setattr("itzi.grass_session._initialized_temporal_session", None) + monkeypatch.setattr("itzi.grass.session._initialized_temporal_session", None) monkeypatch.setattr( GrassSessionManager, "current_context", From aad3a73fd7ae9f8444181b5a4217288b6f6d858f Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 16:49:15 -0600 Subject: [PATCH 51/85] move colortable data in the grass submodule --- pyproject.toml | 2 +- src/itzi/{data => grass}/colortable/depth.txt | 0 src/itzi/{data => grass}/colortable/froude.txt | 0 src/itzi/{data => grass}/colortable/vdir.txt | 0 src/itzi/{data => grass}/colortable/velocity.txt | 0 src/itzi/grass/utils.py | 3 +-- 6 files changed, 2 insertions(+), 3 deletions(-) rename src/itzi/{data => grass}/colortable/depth.txt (100%) rename src/itzi/{data => grass}/colortable/froude.txt (100%) rename src/itzi/{data => grass}/colortable/vdir.txt (100%) rename src/itzi/{data => grass}/colortable/velocity.txt (100%) diff --git a/pyproject.toml b/pyproject.toml index 1182f1c1..1dc0d865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -"itzi.data.colortable" = ["*.txt"] +"itzi.grass" = ["colortable/*.txt"] [tool.pytest.ini_options] pythonpath = ["src", "."] diff --git a/src/itzi/data/colortable/depth.txt b/src/itzi/grass/colortable/depth.txt similarity index 100% rename from src/itzi/data/colortable/depth.txt rename to src/itzi/grass/colortable/depth.txt diff --git a/src/itzi/data/colortable/froude.txt b/src/itzi/grass/colortable/froude.txt similarity index 100% rename from src/itzi/data/colortable/froude.txt rename to src/itzi/grass/colortable/froude.txt diff --git a/src/itzi/data/colortable/vdir.txt b/src/itzi/grass/colortable/vdir.txt similarity index 100% rename from src/itzi/data/colortable/vdir.txt rename to src/itzi/grass/colortable/vdir.txt diff --git a/src/itzi/data/colortable/velocity.txt b/src/itzi/grass/colortable/velocity.txt similarity index 100% rename from src/itzi/data/colortable/velocity.txt rename to src/itzi/grass/colortable/velocity.txt diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index e911b70a..14973f7b 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -40,8 +40,7 @@ MIN_GRASS_VERSION = (8, 4) # color rules -_ROOT = Path(__file__).parent.parent -_DIR = _ROOT / "data" / "colortable" +_DIR = Path(__file__).parent / "colortable" RULE_H = _DIR / "depth.txt" RULE_V = _DIR / "velocity.txt" RULE_VDIR = _DIR / "vdir.txt" From 162e6d1de699a18c87b333c66e7fd138c1519c6e Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 17:04:42 -0600 Subject: [PATCH 52/85] remove nullable dict entries --- src/itzi/grass/utils.py | 2 +- src/itzi/providers/grass_input.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index 14973f7b..ab87c2a8 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -327,7 +327,7 @@ def replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarra def resolve_input_map_lists( grass_interface: GrassInterface, - map_names: Mapping[str, str | None], + map_names: Mapping[str, str], start_time: datetime, end_time: datetime, input_kinds: Mapping[str, Literal["raster", "strds"]] | None, diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 97c29c12..0a7adb42 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -31,7 +31,7 @@ class GrassRasterInputProvider(RasterInputProvider): def __init__( self, grass_interface: GrassInterface, - input_map_names: Mapping[str, str | None], + input_map_names: Mapping[str, str], default_start_time: datetime, default_end_time: datetime, input_kinds: Mapping[str, Literal["raster", "strds"]] | None, From bea705526778115d89c990cd0c4c2d046806a880 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 17:08:58 -0600 Subject: [PATCH 53/85] stronger typing in GrassSession --- src/itzi/grass/session.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/itzi/grass/session.py b/src/itzi/grass/session.py index d0931cc3..5834f30e 100644 --- a/src/itzi/grass/session.py +++ b/src/itzi/grass/session.py @@ -73,18 +73,18 @@ def _validate_requested_context(self) -> None: active = self.current_context() if active is None: msgr.fatal("Unable to determine the active GRASS context") - requested = ( + gisdb, location, mapset = ( self.grass_params.grassdata, self.grass_params.location, self.grass_params.mapset, ) - if not any(requested): + if not any((gisdb, location, mapset)): return - if not all(requested): + if not gisdb or not location or not mapset: msgr.fatal("GRASS database, location, and mapset must be supplied together") - requested_database = str(Path(requested[0]).expanduser().resolve()) + requested_database = str(Path(gisdb).expanduser().resolve()) active_database = str(Path(active[0]).expanduser().resolve()) - if (requested_database, requested[1], requested[2]) != ( + if (requested_database, location, mapset) != ( active_database, active[1], active[2], @@ -119,15 +119,12 @@ def open(self) -> None: self.ensure_temporal_initialized() return - # Check if mandatory GRASS parameters are present - if not all( - [self.grass_params.grassdata, self.grass_params.location, self.grass_params.mapset] - ): - msgr.fatal("No GRASS parameters to create a session.") - gisdb = self.grass_params.grassdata location = self.grass_params.location mapset = self.grass_params.mapset + # Check if mandatory GRASS parameters are present + if not gisdb or not location or not mapset: + msgr.fatal("No GRASS parameters to create a session.") # Check if the given parameters exist and can be accessed error_msg = "'{}' does not exist or does not have adequate permissions" From a0b9214d894e3d188991943aff38a0e9c32b48cc Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 17:13:01 -0600 Subject: [PATCH 54/85] organize imports --- src/itzi/ensemble/preflight.py | 6 +++--- src/itzi/itzi.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/itzi/ensemble/preflight.py b/src/itzi/ensemble/preflight.py index 0b708ea8..a65d16ac 100644 --- a/src/itzi/ensemble/preflight.py +++ b/src/itzi/ensemble/preflight.py @@ -19,9 +19,9 @@ import numpy as np from itzi.ensemble.models import EnsembleError -from itzi.manifest import validate_file_destination -from itzi.grass.names import derived_record_name from itzi.ensemble.resolution import _last_record_index, verify_resolved_simulation +from itzi.grass.names import derived_record_name +from itzi.manifest import validate_file_destination if TYPE_CHECKING: from itzi.ensemble.models import ResolvedSimulation @@ -30,8 +30,8 @@ def preflight_simulation(simulation: ResolvedSimulation) -> None: """Validate one resolved simulation without creating user artifacts.""" - from itzi.grass.utils import ensure_min_version from itzi.grass.interface import GrassInterface + from itzi.grass.utils import ensure_min_version verify_resolved_simulation(simulation) ensure_min_version() diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 48800ba8..d7a120b5 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -46,6 +46,11 @@ ResolvedSimulation, ValidationFailure, ) +from itzi.ensemble.resolution import ( + resolve_ensemble, + validate_resolved_ensemble, + verify_resolved_simulation, +) from itzi.grass.session import GrassSessionManager from itzi.manifest import ( _create_manifest, @@ -55,11 +60,6 @@ _validate_manifest_destination, ) from itzi.messenger import VerbosityLevel -from itzi.ensemble.resolution import ( - resolve_ensemble, - validate_resolved_ensemble, - verify_resolved_simulation, -) from itzi.run_plan import load_batch from itzi.simulation_runner import SimulationRunner From 6518323e164512ff79c1958fda2cbd0b383e4356 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 17:45:03 -0600 Subject: [PATCH 55/85] various simplifications --- src/itzi/ensemble/models.py | 20 ++++---------------- src/itzi/ensemble/schema.py | 15 +++------------ src/itzi/ensemble/yaml.py | 17 ++--------------- src/itzi/itzi.py | 28 +++++++++++++--------------- src/itzi/simulation_runner.py | 3 +-- tests/cli/test_cli.py | 9 +++++++-- tests/cli/test_ensemble_config.py | 3 +++ 7 files changed, 33 insertions(+), 62 deletions(-) diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index 06800b2a..31b4acac 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -29,16 +29,8 @@ TemporalType, ) from itzi_core.const import DefaultValues -from pydantic import ( - BaseModel, - ConfigDict, - StrictStr, - TypeAdapter, - model_validator, -) -from pydantic import ( - JsonValue as PydanticJsonValue, -) +from pydantic import BaseModel, ConfigDict, StrictStr, model_validator +from pydantic import JsonValue as PydanticJsonValue from itzi.grass.session import GrassParams @@ -68,8 +60,6 @@ None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] ) -_JSON_VALUE_ADAPTER = TypeAdapter(PydanticJsonValue) - class EnsembleError(ValueError): """Configuration error associated with a single YAML document.""" @@ -275,10 +265,8 @@ def format_duration(value: timedelta) -> str: return rendered -def _canonical_json(value: PydanticJsonValue | BaseModel) -> str: - """Serialize a JSON-compatible value or model in a stable form.""" - if isinstance(value, BaseModel): - value = _JSON_VALUE_ADAPTER.validate_json(value.model_dump_json()) +def _canonical_json(value: PydanticJsonValue) -> str: + """Serialize a JSON-compatible value in a stable form.""" try: return json.dumps( value, diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index e3e94be1..080602d6 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Annotated, Literal, cast +from typing import Annotated, Literal from itzi_core import OUTPUT_ARRAY_KEYS from pydantic import ( @@ -24,13 +24,7 @@ field_validator, model_validator, ) -from pydantic import JsonValue as PydanticJsonValue - -from itzi.ensemble.models import ( - DomainConfig, - StrictModel, - _canonical_json, -) +from itzi.ensemble.models import DomainConfig, StrictModel type SweepString = StrictStr | Annotated[list[StrictStr], Field(min_length=1)] type FiniteFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] @@ -221,9 +215,6 @@ def validate_drainage_output(self) -> YamlEnsembleDocumentV1: def _validate_sweep[T](value: T) -> T: """Validate one scalar-or-list sweep field after Pydantic's type validation.""" values = value if isinstance(value, list) else [value] - semantic_values = [ - _canonical_json(cast(PydanticJsonValue | InfiltrationAlternative, item)) for item in values - ] - if len(semantic_values) != len(set(semantic_values)): + if len(values) != len(set(values)): raise ValueError("contains duplicate semantic values") return value diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py index 93510b2a..cf1f38d9 100644 --- a/src/itzi/ensemble/yaml.py +++ b/src/itzi/ensemble/yaml.py @@ -487,19 +487,6 @@ def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: def _freeze_json(value: SweepScalar) -> JsonValue: - if isinstance(value, (str, float)): + if isinstance(value, (str, int, float)): return value - if isinstance(value, NoInfiltration): - return (("type", value.type),) - if isinstance(value, ConstantInfiltration): - return (("rate", value.rate), ("type", value.type)) - assert isinstance(value, GreenAmptInfiltration) - values: list[tuple[str, JsonValue]] = [ - ("capillary_pressure", value.capillary_pressure), - ("effective_porosity", value.effective_porosity), - ("hydraulic_conductivity", value.hydraulic_conductivity), - ("type", value.type), - ] - if value.soil_water_content is not None: - values.insert(3, ("soil_water_content", value.soil_water_content)) - return tuple(values) + return tuple(sorted(value.model_dump(exclude_none=True).items())) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index d7a120b5..104c6247 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -46,11 +46,7 @@ ResolvedSimulation, ValidationFailure, ) -from itzi.ensemble.resolution import ( - resolve_ensemble, - validate_resolved_ensemble, - verify_resolved_simulation, -) +from itzi.ensemble.resolution import resolve_ensemble, validate_resolved_ensemble from itzi.grass.session import GrassSessionManager from itzi.manifest import ( _create_manifest, @@ -117,7 +113,6 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str runner: SimulationRunner | None = None try: with GrassSessionManager(simulation.grass_params): - verify_resolved_simulation(simulation) runner = SimulationRunner( simulation.simulation_config, simulation.grass_params, @@ -449,20 +444,22 @@ def _run_one_ensemble( return max(1, len(selected_ids)) failure_count = 0 + member_started = False for simulation in resolved.simulations: if simulation.simulation_id not in selected_ids: continue if states[simulation.simulation_id]["status"] != "planned": continue - detail = _preflight_simulation_in_subprocess(simulation) - if detail is not None: - states[simulation.simulation_id]["status"] = "validation_failed" - states[simulation.simulation_id]["failure"] = { - "phase": "preflight", - "detail": detail, - } - _update_manifest(manifest_path, ensemble, states) - continue + if member_started: + detail = _preflight_simulation_in_subprocess(simulation) + if detail is not None: + states[simulation.simulation_id]["status"] = "validation_failed" + states[simulation.simulation_id]["failure"] = { + "phase": "preflight", + "detail": detail, + } + _update_manifest(manifest_path, ensemble, states) + continue states[simulation.simulation_id]["status"] = "running" try: _update_manifest(manifest_path, ensemble, states) @@ -471,6 +468,7 @@ def _run_one_ensemble( failure_count += 1 break started = time.monotonic() + member_started = True status, detail = _run_simulation_in_subprocess(simulation) state = states[simulation.simulation_id] state["status"] = status diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index b5a92483..06307c4b 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -22,7 +22,7 @@ import itzi.messenger as msgr from itzi.ensemble.models import EffectiveMask -from itzi.grass.session import GrassParams, GrassSessionManager +from itzi.grass.session import GrassParams from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider from itzi.providers.grass_input import GrassRasterInputProvider from itzi.providers.grass_output import GrassRasterOutputProvider, GrassVectorOutputProvider @@ -59,7 +59,6 @@ def __init__( self.stats_file = stats_file self.effective_mask = effective_mask ensure_min_version() - GrassSessionManager.ensure_temporal_initialized() if input_kinds is None: resolved_inputs = { key: resolve_input_identifier(name) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index f59808f6..da61ecd8 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -220,10 +220,11 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo artifacts = SimpleNamespace(output_map_names=(), drainage_output=None, statistics_file=None) simulations = tuple( SimpleNamespace(simulation_id=name, coordinates=(), artifacts=artifacts) - for name in ("good", "bad", "preflight", "unselected") + for name in ("preflight", "unselected", "good", "bad") ) runs = [] updates = [] + preflights = [] monkeypatch.setattr("itzi.itzi._create_manifest", lambda *_args, **_kwargs: None) monkeypatch.setattr( "itzi.itzi._update_manifest", @@ -231,7 +232,10 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo {key: value["status"] for key, value in states.items()} ), ) - monkeypatch.setattr("itzi.itzi._preflight_simulation_in_subprocess", lambda _: None) + monkeypatch.setattr( + "itzi.itzi._preflight_simulation_in_subprocess", + lambda simulation: preflights.append(simulation.simulation_id) or None, + ) monkeypatch.setattr( "itzi.itzi._run_simulation_in_subprocess", lambda simulation: ( @@ -255,6 +259,7 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo assert count == 2 assert runs == ["good", "bad"] + assert preflights == ["bad"] assert updates[-1] == { "good": "completed", "bad": "execution_failed", diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index eb0f32f9..a939ae6b 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -231,6 +231,9 @@ def test_infiltration_alternatives_are_explicit(tmp_path): labelled_stream = load_yaml_stream(write_yaml(tmp_path, labelled)) assert labelled_stream.failures[0].phase == "schema" + duplicate = content.replace(" - type: none\n", " - type: none\n - type: none\n") + assert load_yaml_stream(write_yaml(tmp_path, duplicate)).failures[0].phase == "schema" + def test_yaml_rejects_unsupported_hotstart(tmp_path): content = ( From b59d004a05b17eb6b006b51d85bebac1a2246a94 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 17:54:03 -0600 Subject: [PATCH 56/85] check map keys at input and output --- src/itzi/providers/grass_input.py | 8 +++++++- src/itzi/providers/grass_output.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 0a7adb42..6fae429d 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Literal import numpy as np -from itzi_core import DomainData +from itzi_core import INPUT_ARRAY_KEYS, DomainData from itzi_core.providers import RasterInputProvider if TYPE_CHECKING: @@ -38,6 +38,12 @@ def __init__( ) -> None: from itzi.grass.utils import resolve_input_map_lists + invalid_input_keys = sorted(set(input_map_names) - INPUT_ARRAY_KEYS) + if invalid_input_keys: + raise ValueError( + f"input_map_names contains invalid input keys: {', '.join(invalid_input_keys)}" + ) + self.grass_interface = grass_interface self.start_time = default_start_time self.end_time = default_end_time diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index dc8ac9b4..65d5e353 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from datetime import datetime, timedelta - from itzi_core import TemporalType + from itzi_core import OUTPUT_ARRAY_KEYS, TemporalType from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology from itzi.grass.interface import GrassInterface @@ -41,6 +41,12 @@ def __init__( hmin: float, temporal_type: TemporalType, ) -> None: + invalid_output_keys = sorted(set(out_map_names) - OUTPUT_ARRAY_KEYS) + if invalid_output_keys: + raise ValueError( + f"out_map_names contains invalid input keys: {', '.join(invalid_output_keys)}" + ) + self.grass_interface = grass_interface # Dataset IDs qualified in the current mapset. Keys are output variables. self.out_map_names = out_map_names From bb6d2e14bc915f709fc760210432322a2313935d Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 18:04:33 -0600 Subject: [PATCH 57/85] Simplify validation --- src/itzi/ensemble/models.py | 11 ------ src/itzi/ensemble/resolution.py | 4 -- src/itzi/ensemble/schema.py | 66 ++++++++++--------------------- src/itzi/ensemble/yaml.py | 5 +-- src/itzi/itzi.py | 8 +--- tests/cli/test_ensemble_config.py | 6 ++- 6 files changed, 28 insertions(+), 72 deletions(-) diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index 31b4acac..2cc5bc96 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -28,7 +28,6 @@ SimulationConfig, TemporalType, ) -from itzi_core.const import DefaultValues from pydantic import BaseModel, ConfigDict, StrictStr, model_validator from pydantic import JsonValue as PydanticJsonValue @@ -43,16 +42,6 @@ DIRECT_INPUT_KEYS = tuple( key for key in INPUT_ARRAY_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} ) -SURFACE_FLOW_DEFAULTS = { - "hmin": DefaultValues.HFMIN, - "cfl": DefaultValues.CFL, - "theta": DefaultValues.THETA, - "g": DefaultValues.G, - "dtmax": DefaultValues.DTMAX, - "slope_threshold": DefaultValues.SLOPE_THRESHOLD, - "max_slope": DefaultValues.MAX_SLOPE, - "max_error": DefaultValues.MAX_ERROR, -} MAX_ENSEMBLE_MEMBERS = 100 MAX_BATCH_MEMBERS = 200 diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index 79692fb1..e67611e8 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -45,10 +45,6 @@ def resolve_ensemble( if not simulations: return () requested_params = _requested_grass_params(simulations[0]) - if any( - _requested_grass_params(simulation) != requested_params for simulation in simulations[1:] - ): - raise EnsembleError("all simulations in an ensemble must use the same GRASS context") with GrassSessionManager(requested_params): from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index 080602d6..113e23c1 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -18,6 +18,7 @@ from itzi_core import OUTPUT_ARRAY_KEYS from pydantic import ( + AfterValidator, Field, StrictFloat, StrictStr, @@ -26,9 +27,22 @@ ) from itzi.ensemble.models import DomainConfig, StrictModel -type SweepString = StrictStr | Annotated[list[StrictStr], Field(min_length=1)] + +def _validate_sweep[T](values: list[T]) -> list[T]: + """Reject duplicate values in a sweep after type validation.""" + if len(values) != len(set(values)): + raise ValueError("contains duplicate semantic values") + return values + + +type SweepString = ( + StrictStr | Annotated[list[StrictStr], Field(min_length=1), AfterValidator(_validate_sweep)] +) type FiniteFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] -type SweepFloat = FiniteFloat | Annotated[list[FiniteFloat], Field(min_length=1)] +type SweepFloat = ( + FiniteFloat + | Annotated[list[FiniteFloat], Field(min_length=1), AfterValidator(_validate_sweep)] +) class EnsembleMetadata(StrictModel): @@ -82,7 +96,10 @@ class GreenAmptInfiltration(StrictModel): Field(discriminator="type"), ] type SweepInfiltration = ( - InfiltrationAlternative | Annotated[list[InfiltrationAlternative], Field(min_length=1)] + InfiltrationAlternative + | Annotated[ + list[InfiltrationAlternative], Field(min_length=1), AfterValidator(_validate_sweep) + ] ) @@ -98,26 +115,6 @@ class InputSweepConfig(StrictModel): boundary_type: SweepString | None = None infiltration: SweepInfiltration = NoInfiltration(type="none") - @field_validator( - "ground_elevation", - "friction", - "water_depth", - "water_surface_elevation", - "losses", - "rainfall_rate", - "inflow", - "boundary_value", - "boundary_type", - ) - @classmethod - def validate_string_sweep(cls, value: SweepString | None) -> SweepString | None: - return _validate_sweep(value) - - @field_validator("infiltration") - @classmethod - def validate_infiltration_sweep(cls, value: SweepInfiltration) -> SweepInfiltration: - return _validate_sweep(value) - @model_validator(mode="after") def validate_initial_conditions(self) -> InputSweepConfig: if self.water_depth is not None and self.water_surface_elevation is not None: @@ -136,11 +133,6 @@ class OptionSweepConfig(StrictModel): max_error: SweepFloat | None = None dtinf: SweepFloat | None = None - @field_validator("*") - @classmethod - def validate_numeric_sweep(cls, value: SweepFloat | None) -> SweepFloat | None: - return _validate_sweep(value) - class DrainageSweepConfig(StrictModel): swmm_input: SweepString @@ -148,16 +140,6 @@ class DrainageSweepConfig(StrictModel): free_weir_coeff: SweepFloat submerged_weir_coeff: SweepFloat - @field_validator("swmm_input") - @classmethod - def validate_input_sweep(cls, value: SweepString) -> SweepString: - return _validate_sweep(value) - - @field_validator("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") - @classmethod - def validate_coeff_sweep(cls, value: SweepFloat) -> SweepFloat: - return _validate_sweep(value) - class RasterOutputs(StrictModel): prefix: StrictStr @@ -210,11 +192,3 @@ def validate_drainage_output(self) -> YamlEnsembleDocumentV1: if self.outputs.drainage is not None and self.drainage is None: raise ValueError("outputs.drainage requires a complete drainage configuration") return self - - -def _validate_sweep[T](value: T) -> T: - """Validate one scalar-or-list sweep field after Pydantic's type validation.""" - values = value if isinstance(value, list) else [value] - if len(values) != len(set(values)): - raise ValueError("contains duplicate semantic values") - return value diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py index cf1f38d9..d7436928 100644 --- a/src/itzi/ensemble/yaml.py +++ b/src/itzi/ensemble/yaml.py @@ -22,7 +22,7 @@ from typing import cast import yaml -from itzi_core import InfiltrationModelType, TemporalType +from itzi_core import InfiltrationModelType, SurfaceFlowParameters, TemporalType from itzi_core.const import DefaultValues from pydantic import JsonValue as PydanticJsonValue from pydantic import TypeAdapter, ValidationError @@ -33,7 +33,6 @@ from itzi.ensemble.models import ( DIRECT_INPUT_KEYS, MAX_ENSEMBLE_MEMBERS, - SURFACE_FLOW_DEFAULTS, DocumentFailure, EnsembleError, ExpandedEnsemble, @@ -443,7 +442,7 @@ def _normalize_infiltration(value: InfiltrationAlternative) -> NormalizedInfiltr def _selected_options( options: OptionSweepConfig, selected: dict[str, SweepScalar] ) -> dict[str, float]: - values = dict(SURFACE_FLOW_DEFAULTS) + values: dict[str, float] = SurfaceFlowParameters().model_dump() values["dtinf"] = DefaultValues.DTINF for key in type(options).model_fields: value = selected.get(f"options.{key}", cast(SweepFloat | None, getattr(options, key))) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 104c6247..fc8950f1 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -144,16 +144,10 @@ def resolver_worker( def preflight_worker(simulation: ResolvedSimulation) -> str | None: """Validate one resolved member in a read-only spawned GRASS process.""" - msgr.raise_on_error = True - msgr._itzi_logger.set_verbosity(msgr.verbosity()) try: - with GrassSessionManager(simulation.grass_params): - from itzi.ensemble.preflight import preflight_simulation - - preflight_simulation(simulation) + return preflight_ensemble_worker((simulation,)).get(simulation.simulation_id) except Exception as error: return f"{type(error).__name__}: {error}" - return None def preflight_ensemble_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index a939ae6b..c2886607 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -7,7 +7,8 @@ from types import SimpleNamespace import pytest -from itzi_core import InfiltrationModelType, TemporalType +from itzi_core import InfiltrationModelType, SurfaceFlowParameters, TemporalType +from itzi_core.const import DefaultValues from pydantic import ValidationError from itzi.ensemble import load_yaml_stream @@ -74,6 +75,9 @@ def test_scalar_document_expands_to_one_member(tmp_path): assert simulation.time.start is None assert simulation.time.end is None assert simulation.time.duration == timedelta(hours=2) + assert dict(simulation.options) == SurfaceFlowParameters().model_dump() | { + "dtinf": DefaultValues.DTINF + } @pytest.mark.parametrize( From 1b9bcfb34dd0dd8e772ff897f7ed9044b60dc050 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 18:10:43 -0600 Subject: [PATCH 58/85] fix providers --- src/itzi/providers/grass_input.py | 8 +------- src/itzi/providers/grass_output.py | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 6fae429d..0a7adb42 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Literal import numpy as np -from itzi_core import INPUT_ARRAY_KEYS, DomainData +from itzi_core import DomainData from itzi_core.providers import RasterInputProvider if TYPE_CHECKING: @@ -38,12 +38,6 @@ def __init__( ) -> None: from itzi.grass.utils import resolve_input_map_lists - invalid_input_keys = sorted(set(input_map_names) - INPUT_ARRAY_KEYS) - if invalid_input_keys: - raise ValueError( - f"input_map_names contains invalid input keys: {', '.join(invalid_input_keys)}" - ) - self.grass_interface = grass_interface self.start_time = default_start_time self.end_time = default_end_time diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 65d5e353..8b624431 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING import numpy as np +from itzi_core import OUTPUT_ARRAY_KEYS from itzi_core.providers import RasterOutputProvider, VectorOutputProvider from itzi.grass.names import derived_record_name @@ -25,7 +26,7 @@ if TYPE_CHECKING: from datetime import datetime, timedelta - from itzi_core import OUTPUT_ARRAY_KEYS, TemporalType + from itzi_core import TemporalType from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology from itzi.grass.interface import GrassInterface From c325fcf7e1021fbb705637995bdaa3712133f320 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 18:38:43 -0600 Subject: [PATCH 59/85] use only one worker for preflight. Remove background raster writer --- src/itzi/ensemble/preflight.py | 1 - src/itzi/ensemble/schema.py | 31 ++++++----------- src/itzi/grass/interface.py | 56 +++++------------------------- src/itzi/grass/session.py | 3 -- src/itzi/grass/utils.py | 38 +++++--------------- src/itzi/itzi.py | 26 +++----------- src/itzi/manifest.py | 11 ++---- src/itzi/providers/grass_output.py | 2 -- src/itzi/simulation_runner.py | 3 -- tests/cli/test_cli.py | 4 +-- tests/cli/test_ensemble_config.py | 22 ++++++++++++ tests/grass/test_ensemble.py | 19 ++++++---- tests/grass/test_grass_input.py | 11 +++--- tests/grass/test_itzi.py | 4 +-- tests/test_grass_session.py | 1 - 15 files changed, 77 insertions(+), 155 deletions(-) diff --git a/src/itzi/ensemble/preflight.py b/src/itzi/ensemble/preflight.py index a65d16ac..a9025da9 100644 --- a/src/itzi/ensemble/preflight.py +++ b/src/itzi/ensemble/preflight.py @@ -43,7 +43,6 @@ def preflight_simulation(simulation: ResolvedSimulation) -> None: region_id=simulation.grass_params.region, raster_mask_id=simulation.grass_params.mask, effective_mask=(simulation.effective_mask.mode, simulation.effective_mask.source), - non_blocking_write=False, ) try: _validate_inputs(simulation, interface) diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index 113e23c1..372b2f21 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -50,28 +50,19 @@ class EnsembleMetadata(StrictModel): name: StrictStr | None = None -class RelativeTimeConfig(StrictModel): - duration: StrictStr +class TimeConfig(StrictModel): + start: StrictStr | None = None + end: StrictStr | None = None record_step: StrictStr - start: None = None - end: None = None + duration: StrictStr | None = None - -class AbsoluteDurationTimeConfig(StrictModel): - start: StrictStr - duration: StrictStr - record_step: StrictStr - end: None = None - - -class AbsoluteEndTimeConfig(StrictModel): - start: StrictStr - end: StrictStr - record_step: StrictStr - duration: None = None - - -type TimeConfig = RelativeTimeConfig | AbsoluteDurationTimeConfig | AbsoluteEndTimeConfig + @model_validator(mode="after") + def validate_combination(self) -> TimeConfig: + if (self.start is None and self.end is None and self.duration is not None) or ( + self.start is not None and (self.end is None) != (self.duration is None) + ): + return self + raise ValueError("time requires duration alone, start and duration, or start and end") class NoInfiltration(StrictModel): diff --git a/src/itzi/grass/interface.py b/src/itzi/grass/interface.py index 69dd040e..dab86c9b 100644 --- a/src/itzi/grass/interface.py +++ b/src/itzi/grass/interface.py @@ -16,8 +16,6 @@ import os from collections.abc import Mapping from datetime import datetime, timedelta -from queue import Queue -from threading import Lock, Thread from types import MappingProxyType from typing import ClassVar, NamedTuple @@ -41,14 +39,12 @@ from itzi.grass.utils import ( MapData, RasterMapType, - RasterWriteInstructions, STDSType, format_id, get_crs_wkt, has_mask, name_is_map, output_name, - raster_writer, replace_cell_null_sentinel, split_identifier, write_raster_map_blocking, @@ -108,7 +104,6 @@ def __init__( region_id: str | None, raster_mask_id: str | None, effective_mask: tuple[str, str | None] | None = None, - non_blocking_write=True, ) -> None: assert isinstance(start_time, datetime), "start_time not a datetime object!" assert isinstance(end_time, datetime), "end_time not a datetime object!" @@ -119,7 +114,6 @@ def __init__( self.start_time = start_time self.end_time = end_time self.dtype = dtype - self.non_blocking_write = non_blocking_write # LatLon is not supported if gscript.locn_is_latlong(): @@ -144,43 +138,21 @@ def __init__( } self.mask_mode, self.mask_source = self._select_effective_mask(effective_mask) self.overwrite = gscript.overwrite() - # Create thread and queue for writing raster maps - if self.non_blocking_write: - self.raster_lock = Lock() - self.raster_writer_queue: Queue[RasterWriteInstructions | None] = Queue(maxsize=20) - self.raster_writer_thread = Thread( - name="RasterWriter", - target=raster_writer, - args=(self.raster_writer_queue, self.raster_lock), - ) - self.raster_writer_thread.start() def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): - self.finalize() self.cleanup() def __del__(self): - self.finalize() self.cleanup() - def finalize(self) -> None: - """Make sure that all maps are written.""" - if self.non_blocking_write: - msgr.debug("Writing last maps...") - self.raster_writer_queue.join() - def cleanup(self) -> None: - """Remove the temporary region and stop the raster writer.""" + """Remove the temporary region.""" if self.region_id: msgr.debug("Remove temp region...") gscript.del_temp_region() - # Thread killswitch - if self.non_blocking_write: - self.raster_writer_queue.put(None) - self.raster_writer_thread.join() def grass_dtype(self, dtype: str) -> RasterMapType: if dtype in self.dtype_conv["DCELL"]: @@ -380,13 +352,6 @@ def read_raster_map(self, raster_id: str) -> np.ndarray: effective mask selected for the simulation is applied separately by :meth:`get_npmask` and passed to itzi-core. """ - if self.non_blocking_write: - with self.raster_lock: - return self._read_raster_map_nomask(raster_id) - return self._read_raster_map_nomask(raster_id) - - def _read_raster_map_nomask(self, raster_id: str) -> np.ndarray: - """Read the current computational region while bypassing GRASS MASK.""" from grass.lib import raster as libraster name, mapset = split_identifier(raster_id) @@ -424,19 +389,14 @@ def _read_raster_map_nomask(self, raster_id: str) -> np.ndarray: def write_raster_map(self, arr: np.ndarray, raster_id: str, mkey: str, hmin: float) -> None: """Take a numpy array and write it to GRASS DB""" - mtype = self.grass_dtype(str(arr.dtype)) - write_instructions = RasterWriteInstructions( - array=arr, - raster_name=output_name(raster_id), - map_type=mtype, - map_key=mkey, - hmin=hmin, - overwrite=self.overwrite, + write_raster_map_blocking( + arr, + output_name(raster_id), + self.grass_dtype(str(arr.dtype)), + mkey, + hmin, + self.overwrite, ) - if self.non_blocking_write: - self.raster_writer_queue.put(write_instructions) - else: - write_raster_map_blocking(write_instructions) def create_db_links( self, vect_map: VectorTopo, linking_elem: dict[str, DBLayerDescription] diff --git a/src/itzi/grass/session.py b/src/itzi/grass/session.py index 5834f30e..7cf0ee93 100644 --- a/src/itzi/grass/session.py +++ b/src/itzi/grass/session.py @@ -14,7 +14,6 @@ from __future__ import annotations -import importlib.util import os import subprocess import sys @@ -56,8 +55,6 @@ def current_context() -> tuple[str, str, str] | None: # independent from GRASS imports. if not os.environ.get("GISRC"): return None - if importlib.util.find_spec("grass") is None and "grass" not in sys.modules: - return None try: from grass.pygrass.utils import getenv diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index ab87c2a8..caacb38c 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -17,8 +17,6 @@ from collections.abc import Mapping from datetime import datetime from pathlib import Path -from queue import Queue -from threading import Lock from typing import TYPE_CHECKING, Literal, NamedTuple import grass.pygrass.utils as gutils @@ -61,15 +59,6 @@ type RasterMapType = Literal["CELL", "FCELL", "DCELL"] -class RasterWriteInstructions(NamedTuple): - array: np.ndarray - raster_name: str - map_type: RasterMapType - map_key: str - hmin: float - overwrite: bool - - class MapData(NamedTuple): id: str start_time: datetime @@ -99,29 +88,20 @@ def check_output_files(file_list: list[str]) -> None: msgr.fatal(f"File {map_name} exists and will not be overwritten") -def raster_writer(q: Queue[RasterWriteInstructions | None], lock: Lock): - """Write a raster map in GRASS""" - while True: - write_instructions = q.get() - if write_instructions is None: - break - with lock: - write_raster_map_blocking(write_instructions) - q.task_done() - - -def write_raster_map_blocking(writer_instructions: RasterWriteInstructions) -> None: - raster_name = writer_instructions.raster_name - array = writer_instructions.array - map_key = writer_instructions.map_key - map_type = writer_instructions.map_type - hmin = writer_instructions.hmin +def write_raster_map_blocking( + array: np.ndarray, + raster_name: str, + map_type: RasterMapType, + map_key: str, + hmin: float, + overwrite: bool, +) -> None: with raster.RasterRow( raster_name, mapset=get_current_mapset(), mode="w", mtype=map_type, - overwrite=writer_instructions.overwrite, + overwrite=overwrite, ) as newraster: newrow = raster.Buffer((array.shape[1],), mtype=map_type) for row in array: diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index fc8950f1..90a67a36 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -142,15 +142,7 @@ def resolver_worker( return resolve_ensemble(expanded) -def preflight_worker(simulation: ResolvedSimulation) -> str | None: - """Validate one resolved member in a read-only spawned GRASS process.""" - try: - return preflight_ensemble_worker((simulation,)).get(simulation.simulation_id) - except Exception as error: - return f"{type(error).__name__}: {error}" - - -def preflight_ensemble_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: +def preflight_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: """Preflight selected members in one GRASS session, retaining individual failures.""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) @@ -445,7 +437,7 @@ def _run_one_ensemble( if states[simulation.simulation_id]["status"] != "planned": continue if member_started: - detail = _preflight_simulation_in_subprocess(simulation) + detail = _preflight_in_subprocess((simulation,)).get(simulation.simulation_id) if detail is not None: states[simulation.simulation_id]["status"] = "validation_failed" states[simulation.simulation_id]["failure"] = { @@ -513,20 +505,12 @@ def _run_simulation_in_subprocess(simulation: ResolvedSimulation) -> tuple[str, return "execution_failed", f"{type(error).__name__}: {error}" -def _preflight_simulation_in_subprocess(simulation: ResolvedSimulation) -> str | None: - try: - with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: - return executor.submit(preflight_worker, simulation).result() - except Exception as error: - return f"{type(error).__name__}: {error}" - - -def _preflight_ensemble_in_subprocess( +def _preflight_in_subprocess( simulations: tuple[ResolvedSimulation, ...], ) -> dict[str, str]: try: with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: - return executor.submit(preflight_ensemble_worker, simulations).result() + return executor.submit(preflight_worker, simulations).result() except Exception as error: detail = f"{type(error).__name__}: {error}" return {simulation.simulation_id: detail for simulation in simulations} @@ -564,7 +548,7 @@ def _preflight_ensemble( if simulation.simulation_id in selected_ids and simulation.simulation_id not in failures ) if to_preflight: - for simulation_id, detail in _preflight_ensemble_in_subprocess(to_preflight).items(): + for simulation_id, detail in _preflight_in_subprocess(to_preflight).items(): failures[simulation_id] = {"phase": "preflight", "detail": detail} return failures, manifest_failure diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py index 7fdc7d88..f2fd1308 100644 --- a/src/itzi/manifest.py +++ b/src/itzi/manifest.py @@ -30,6 +30,7 @@ ValidationFailure, render_template, ) +from itzi.ensemble.resolution import _validate_protected_file_output def _manifest_path(ensemble: ExpandedEnsemble) -> Path: @@ -73,15 +74,7 @@ def _validate_manifest_destination( protected.add(simulation.simulation_config.swmm_inp.resolve()) if simulation.artifacts.statistics_file is not None: protected.add(simulation.artifacts.statistics_file.resolve()) - if manifest_path in protected: - raise EnsembleError( - f"manifest <{manifest_path}> aliases a protected input or member artifact" - ) - for path in protected: - if manifest_path.exists() and path.exists() and manifest_path.samefile(path): - raise EnsembleError( - f"manifest <{manifest_path}> aliases a protected input or member artifact" - ) + _validate_protected_file_output(manifest_path, protected) validate_file_destination(manifest_path, overwrite=overwrite, description="manifest") diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 8b624431..af2244df 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -85,8 +85,6 @@ def write_arrays( def finalize(self) -> None: """Finalize outputs and cleanup.""" - # Write the final raster maps - self.grass_interface.finalize() # register in GRASS temporal framework for map_key, lst in self.output_maplist.items(): strds_id = self.out_map_names[map_key] diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 06307c4b..3d1bae2a 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -106,7 +106,6 @@ def initialize(self) -> Self: effective_mask=(self.effective_mask.mode, self.effective_mask.source) if self.effective_mask is not None else None, - non_blocking_write=False, ) msgr.verbose("Setting up GRASS simulation...") @@ -171,7 +170,6 @@ def finalize(self): self.sim.finalize() # Cleanup the grass interface object if hasattr(self, "g_interface"): - self.g_interface.finalize() self.g_interface.cleanup() return self @@ -187,5 +185,4 @@ def origin(self): def __del__(self): # Cleanup the grass interface object if hasattr(self, "g_interface"): - self.g_interface.finalize() self.g_interface.cleanup() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index da61ecd8..87eeb4aa 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -233,8 +233,8 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo ), ) monkeypatch.setattr( - "itzi.itzi._preflight_simulation_in_subprocess", - lambda simulation: preflights.append(simulation.simulation_id) or None, + "itzi.itzi._preflight_in_subprocess", + lambda simulations: preflights.append(simulations[0].simulation_id) or {}, ) monkeypatch.setattr( "itzi.itzi._run_simulation_in_subprocess", diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index c2886607..b1000baf 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -24,6 +24,7 @@ ManifestOutputs, OptionSweepConfig, StatisticsOutputs, + TimeConfig, ) from itzi.ensemble.resolution import ( _last_record_index, @@ -80,6 +81,27 @@ def test_scalar_document_expands_to_one_member(tmp_path): } +def test_time_config_accepts_only_supported_combinations(): + start = "2026-09-01T00:00:00" + end = "2026-09-01T02:00:00" + duration = "02:00:00" + for fields in ( + {"duration": duration}, + {"start": start, "duration": duration}, + {"start": start, "end": end}, + ): + TimeConfig.model_validate({"record_step": "00:05:00", **fields}) + for fields in ( + {}, + {"start": start}, + {"end": end}, + {"end": end, "duration": duration}, + {"start": start, "end": end, "duration": duration}, + ): + with pytest.raises(ValidationError): + TimeConfig.model_validate({"record_step": "00:05:00", **fields}) + + @pytest.mark.parametrize( ("model", "value"), ( diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index cdc570d6..45f7bcab 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -87,7 +87,7 @@ def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( "rainfall_rate": f"stage_series@{mapset}", } assert dict(resolved.input_kinds)["rainfall_rate"] == "strds" - assert preflight_worker(resolved) is None + assert preflight_worker((resolved,)).get(resolved.simulation_id) is None path = write_study(tmp_path, input={"rainfall_rate": f"stage_series@{mapset}"}) qualified = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] @@ -173,7 +173,6 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path None, "stage_mask_values", ("explicit", resolved.effective_mask.source), - non_blocking_write=False, ) try: explicit = interface.get_npmask() @@ -192,9 +191,9 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path finally: interface.cleanup() - assert preflight_worker(resolved) is None + assert preflight_worker((resolved,)).get(resolved.simulation_id) is None gscript.run_command("r.mask", flags="r") - assert preflight_worker(resolved) is None + assert preflight_worker((resolved,)).get(resolved.simulation_id) is None active_path = write_study(tmp_path, domain={}) active_resolved = _resolve_ensemble_in_subprocess( @@ -203,7 +202,9 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path assert isinstance(active_resolved, ResolvedSimulation) assert active_resolved.effective_mask.mode == "none" gscript.run_command("g.copy", raster="stage_mask_values,MASK") - assert "effective GRASS mask changed" in (preflight_worker(active_resolved) or "") + assert "effective GRASS mask changed" in ( + preflight_worker((active_resolved,)).get(active_resolved.simulation_id) or "" + ) assert gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET")) finally: gscript.run_command("r.mask", flags="r") @@ -218,10 +219,14 @@ def test_worker_rejects_region_and_input_changes_after_resolution(tmp_path: Path assert isinstance(resolved, ResolvedSimulation) gscript.run_command("g.region", n=60) - assert "GRASS domain changed" in (preflight_worker(resolved) or "") + assert "GRASS domain changed" in ( + preflight_worker((resolved,)).get(resolved.simulation_id) or "" + ) gscript.run_command("g.region", n=50) gscript.run_command("g.remove", flags="f", type="raster", name="stage_worker_friction") - assert "stage_worker_friction" in (preflight_worker(resolved) or "") + assert "stage_worker_friction" in ( + preflight_worker((resolved,)).get(resolved.simulation_id) or "" + ) assert not (tmp_path / "results").exists() diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index cfde66fb..25575cda 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -75,7 +75,6 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No import grass.script as gscript from itzi.grass.utils import ( - RasterWriteInstructions, format_id, output_name, qualify_output_id, @@ -98,9 +97,7 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) write_raster_map_blocking( - RasterWriteInstructions( - np.ones((3, 3), dtype=np.int32), "visible_output", "CELL", "ground_elevation", 0, False - ) + np.ones((3, 3), dtype=np.int32), "visible_output", "CELL", "ground_elevation", 0, False ) assert format_id("visible_output") == "visible_output@child" @@ -111,8 +108,10 @@ def test_direct_runner_qualifies_ids_before_creating_providers(test_data_path: s from pathlib import Path from itzi.configreader import ConfigReader + from itzi.grass.session import GrassSessionManager from itzi.simulation_runner import SimulationRunner + GrassSessionManager.ensure_temporal_initialized() reader = ConfigReader(str(Path(test_data_path) / "5by5" / "5by5.ini")) config = reader.sim_config.model_copy( update={"output_map_names": {"water_depth": "direct_id_check"}} @@ -132,9 +131,7 @@ def test_vector_writer_uses_bare_name_from_qualified_output_id() -> None: from itzi.grass.interface import GrassInterface start = datetime(2020, 1, 1) - with GrassInterface( - start, start + timedelta(seconds=1), np.float32, None, None, non_blocking_write=False - ) as interface: + with GrassInterface(start, start + timedelta(seconds=1), np.float32, None, None) as interface: interface.write_vector_map( DrainageNetworkTopology(nodes=(), links=()), DrainageNetworkAttributes(nodes=(), links=()), diff --git a/tests/grass/test_itzi.py b/tests/grass/test_itzi.py index 32de2b32..68e3f148 100644 --- a/tests/grass/test_itzi.py +++ b/tests/grass/test_itzi.py @@ -9,6 +9,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader +from itzi.grass.session import GrassSessionManager def _create_timed_rain_inputs(name_prefix: str) -> dict[str, str]: @@ -89,6 +90,7 @@ def _build_timed_rain_runner( parser.write(file_handle) conf_data = ConfigReader(config_file) + GrassSessionManager.ensure_temporal_initialized() return SimulationRunner( conf_data.sim_config, conf_data.grass_params, @@ -339,7 +341,6 @@ def test_timed_grass_rain_switches_cleanly_around_boundary( seconds=expected_input_deadline_seconds ) finally: - sim_runner.g_interface.finalize() sim_runner.g_interface.cleanup() @@ -370,5 +371,4 @@ def test_timed_grass_rain_is_applied_before_a_step_crosses_its_boundary(test_dat 360.0 / (1000 * 3600), ) finally: - sim_runner.g_interface.finalize() sim_runner.g_interface.cleanup() diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index e5200c63..7b0ea31a 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -21,7 +21,6 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: grass_package = ModuleType("grass") grass_package.script = grass_script - monkeypatch.setattr("itzi.grass.session.importlib.util.find_spec", lambda _: None) monkeypatch.setattr("itzi.grass.session.os.access", lambda *_: True) monkeypatch.setattr(GrassSessionManager, "ensure_temporal_initialized", lambda self: None) monkeypatch.setattr( From 2a2f74782d586eac7bc6f44d0c118a3b90b11e7d Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 18:51:17 -0600 Subject: [PATCH 60/85] fix type checking error --- src/itzi/ensemble/resolution.py | 39 +++++++++++++++---------------- tests/cli/test_ensemble_config.py | 27 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index e67611e8..ad698806 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -196,6 +196,7 @@ def _simulation_identity( """Build the versioned identity payload after all source normalization.""" drainage = dict(expanded.drainage) if expanded.drainage is not None else None if drainage is not None: + assert swmm_digest is not None drainage["swmm_digest"] = swmm_digest drainage["swmm_input"] = str( (expanded.source.path.parent / str(drainage["swmm_input"])).resolve() @@ -344,27 +345,25 @@ def _build_simulation_config( assert end is not None options = dict(expanded.options) surface_options = {key: value for key, value in options.items() if key != "dtinf"} - drainage = dict(expanded.drainage) if expanded.drainage is not None else {} + # Create the dict first to satisfy the type checker + config_data = { + "start_time": start, + "end_time": end, + "record_step": expanded.time.record_step, + "temporal_type": expanded.time.temporal_type, + "hotstart_config": None, + "input_map_names": input_names, + "output_map_names": dict(artifacts.output_map_names), + "surface_flow_parameters": SurfaceFlowParameters(**surface_options), + "dtinf": options["dtinf"], + "infiltration_model": expanded.infiltration.model, + "swmm_inp": swmm_path, + "drainage_output": artifacts.drainage_output, + } + if expanded.drainage is not None: + config_data.update((key, value) for key, value in expanded.drainage if key != "swmm_input") try: - return SimulationConfig( - start_time=start, - end_time=end, - record_step=expanded.time.record_step, - temporal_type=expanded.time.temporal_type, - hotstart_config=None, - input_map_names=input_names, - output_map_names=dict(artifacts.output_map_names), - surface_flow_parameters=SurfaceFlowParameters(**surface_options), - dtinf=options["dtinf"], - infiltration_model=expanded.infiltration.model, - swmm_inp=swmm_path, - drainage_output=artifacts.drainage_output, - **{ - key: drainage[key] - for key in ("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") - if key in drainage - }, - ) + return SimulationConfig.model_validate(config_data) except ValidationError as error: details = "; ".join(item["msg"] for item in error.errors(include_url=False)) raise EnsembleError(details) from error diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index b1000baf..866c1c0a 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from datetime import datetime, timedelta import sys from types import SimpleNamespace @@ -14,6 +15,7 @@ from itzi.ensemble import load_yaml_stream from itzi.ensemble.models import ( MAX_ENSEMBLE_MEMBERS, + ArtifactSummary, format_duration, parse_duration, render_template, @@ -27,6 +29,7 @@ TimeConfig, ) from itzi.ensemble.resolution import ( + _build_simulation_config, _last_record_index, _render_artifacts, _resolve_inputs, @@ -81,6 +84,30 @@ def test_scalar_document_expands_to_one_member(tmp_path): } +def test_drainage_coefficients_are_forwarded(tmp_path): + expanded = load_yaml_stream(write_yaml(tmp_path, document())).ensembles[0].simulations[0] + expanded = replace( + expanded, + drainage=( + ("swmm_input", "network.inp"), + ("orifice_coeff", 0.2), + ("free_weir_coeff", 0.3), + ("submerged_weir_coeff", 0.4), + ), + ) + config = _build_simulation_config( + expanded, + dict(expanded.input_maps), + ArtifactSummary((), None, None), + tmp_path / "network.inp", + ) + assert ( + config.orifice_coeff, + config.free_weir_coeff, + config.submerged_weir_coeff, + ) == (0.2, 0.3, 0.4) + + def test_time_config_accepts_only_supported_combinations(): start = "2026-09-01T00:00:00" end = "2026-09-01T02:00:00" From a87d5bcd097fdb02f14f790fe314e7fddd42a122 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 19:07:40 -0600 Subject: [PATCH 61/85] simplify code --- src/itzi/configreader.py | 2 - src/itzi/ensemble/resolution.py | 7 +- src/itzi/grass/interface.py | 22 ++----- src/itzi/grass/session.py | 5 +- tests/grass/test_ensemble.py | 110 ++++++++------------------------ 5 files changed, 35 insertions(+), 111 deletions(-) diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index e3475aee..b87cbb62 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -277,8 +277,6 @@ def from_raw_values(cls, raw_values: dict[str, str | None]) -> SimulationTimes: end: datetime = start + duration if start >= end: msgr.fatal("Simulation duration must be positive") - if duration is None: - duration: timedelta = end - start return cls( start=start, diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index ad698806..0c1b8b84 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -167,7 +167,7 @@ def _resolve_inputs( def _resolve_swmm_input( - expanded: ExpandedSimulation, cache: dict[Path, str] | None = None + expanded: ExpandedSimulation, cache: dict[Path, str] ) -> tuple[Path | None, str | None]: if expanded.drainage is None: return None, None @@ -177,11 +177,10 @@ def _resolve_swmm_input( path = path.resolve() if not path.is_file(): raise EnsembleError(f"SWMM input file <{path}> not found") - if cache is not None and path in cache: + if path in cache: return path, cache[path] digest = f"blake2b:{hashlib.blake2b(path.read_bytes(), digest_size=32).hexdigest()}" - if cache is not None: - cache[path] = digest + cache[path] = digest return path, digest diff --git a/src/itzi/grass/interface.py b/src/itzi/grass/interface.py index dab86c9b..d3e98759 100644 --- a/src/itzi/grass/interface.py +++ b/src/itzi/grass/interface.py @@ -40,12 +40,11 @@ MapData, RasterMapType, STDSType, - format_id, - get_crs_wkt, - has_mask, name_is_map, output_name, + read_domain, replace_cell_null_sentinel, + resolve_effective_mask, split_identifier, write_raster_map_blocking, ) @@ -194,22 +193,11 @@ def _select_effective_mask( if source is None: msgr.fatal(f"Effective {mode} mask has no source") return mode, source - if self.raster_mask_id: - return "explicit", format_id(self.raster_mask_id) - if has_mask(): - return "active", f"MASK@{gutils.getenv('MAPSET')}" - return "none", None + selected = resolve_effective_mask(self.raster_mask_id) + return selected.mode, selected.source def get_domain_data(self) -> DomainData: - return DomainData( - north=self.region.north, - south=self.region.south, - east=self.region.east, - west=self.region.west, - rows=self.region.rows, - cols=self.region.cols, - crs_wkt=get_crs_wkt(), - ) + return read_domain(None) def get_npmask(self) -> np.ndarray: """Return a boolean numpy ndarray where True is outside the domain.""" diff --git a/src/itzi/grass/session.py b/src/itzi/grass/session.py index 7cf0ee93..69a95dc8 100644 --- a/src/itzi/grass/session.py +++ b/src/itzi/grass/session.py @@ -45,7 +45,6 @@ class GrassSessionManager: def __init__(self, grass_params: GrassParams): self.grass_params = grass_params self.grass_session = None - self._owns_session = False @staticmethod def current_context() -> tuple[str, str, str] | None: @@ -147,7 +146,6 @@ def open(self) -> None: self.grass_session = gscript.setup.init( path=gisdb, location=location, mapset=mapset, grass_path=grassbin ) - self._owns_session = True if not os.environ.get("GISRC"): session_env = getattr(self.grass_session, "env", None) if session_env and session_env.get("GISRC"): @@ -162,14 +160,13 @@ def open(self) -> None: def close(self) -> None: """Stop GRASS session.""" global _initialized_temporal_session - if self.grass_session is not None and self._owns_session: + if self.grass_session is not None: try: self.grass_session.finish() except Exception as e: print(f"Warning: Error cleaning up GRASS session: {e}") _initialized_temporal_session = None self.grass_session = None - self._owns_session = False def __enter__(self): self.open() diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 45f7bcab..ff7a10d4 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -32,6 +32,7 @@ def write_study(tmp_path: Path, **changes: dict) -> Path: "options": {"dtmax": 0.3}, "outputs": {}, } + document["options"] = changes.pop("options", document["options"]) for section, fields in changes.items(): document[section].update(fields) path = tmp_path / "study.yaml" @@ -397,27 +398,11 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da @pytest.mark.usefixtures("grass_5by5") def test_dry_run_preflights_multiple_members_in_one_session(tmp_path): prefix = f"dry_multi_{uuid4().hex[:8]}" - config_path = tmp_path / "dry-multi.yaml" - config_path.write_text( - f"""\ -schema_version: 1 -ensemble: - id: dry-multi -domain: {{}} -time: - duration: "00:01:00" - record_step: "00:00:30" -input: - ground_elevation: "z" - friction: "n" -options: - cfl: [0.2, 0.3] -outputs: - rasters: - prefix: "{prefix}_{{simulation}}" - variables: [water_depth] -""", - encoding="utf-8", + config_path = write_study( + tmp_path, + ensemble={"id": "dry-multi"}, + options={"cfl": [0.2, 0.3]}, + outputs={"rasters": {"prefix": f"{prefix}_{{simulation}}", "variables": ["water_depth"]}}, ) itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) @@ -430,28 +415,15 @@ def test_dry_run_preflights_multiple_members_in_one_session(tmp_path): @pytest.mark.usefixtures("grass_5by5") def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): gscript.mapcalc("dry_null_dem=null()") - config_path = tmp_path / "dry.yaml" - config_path.write_text( - """\ -schema_version: 1 -ensemble: - id: dry -domain: {} -time: - duration: "00:01:00" - record_step: "00:00:30" -input: - ground_elevation: "dry_null_dem" - friction: "n" -options: {} -outputs: - rasters: - prefix: "dry_output" - variables: [water_depth] - statistics: - file: "dry-results/statistics.csv" -""", - encoding="utf-8", + config_path = write_study( + tmp_path, + ensemble={"id": "dry"}, + input={"ground_elevation": "dry_null_dem"}, + options={}, + outputs={ + "rasters": {"prefix": "dry_output", "variables": ["water_depth"]}, + "statistics": {"file": "dry-results/statistics.csv"}, + }, ) with pytest.raises(RuntimeError, match="YAML batch validation failed"): @@ -497,27 +469,12 @@ def test_dry_run_does_not_read_rainfall_cells(tmp_path): increment="30", unit="seconds", ) - config_path = tmp_path / "dry-rain.yaml" - config_path.write_text( - """\ -schema_version: 1 -ensemble: - id: dry-rain -domain: {} -time: - duration: "00:01:00" - record_step: "00:00:30" -input: - ground_elevation: "z" - friction: "n" - rainfall_rate: "dry_rain_series" -options: {} -outputs: - rasters: - prefix: "dry_rain_output" - variables: [water_depth] -""", - encoding="utf-8", + config_path = write_study( + tmp_path, + ensemble={"id": "dry-rain"}, + input={"rainfall_rate": "dry_rain_series"}, + options={}, + outputs={"rasters": {"prefix": "dry_rain_output", "variables": ["water_depth"]}}, ) itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) @@ -528,26 +485,11 @@ def test_dry_run_does_not_read_rainfall_cells(tmp_path): @pytest.mark.usefixtures("grass_5by5") def test_dry_run_checks_every_generated_output_record(test_data_temp_path): gscript.mapcalc("existing_output_water_depth_0002=1") - config_path = Path(test_data_temp_path) / "existing-output.yaml" - config_path.write_text( - """\ -schema_version: 1 -ensemble: - id: existing-output -domain: {} -time: - duration: "00:01:00" - record_step: "00:00:30" -input: - ground_elevation: "z" - friction: "n" -options: {} -outputs: - rasters: - prefix: "existing_output" - variables: [water_depth] -""", - encoding="utf-8", + config_path = write_study( + Path(test_data_temp_path), + ensemble={"id": "existing-output"}, + options={}, + outputs={"rasters": {"prefix": "existing_output", "variables": ["water_depth"]}}, ) with pytest.raises(RuntimeError, match="YAML batch validation failed"): From f0a0289b0c7b05dee29a7423d9a56847178c7207 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 19:15:04 -0600 Subject: [PATCH 62/85] remove exception caching from input resolution and remove has_mask --- src/itzi/ensemble/resolution.py | 13 +++---------- src/itzi/grass/utils.py | 7 +------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index 0c1b8b84..7dcd8513 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -52,7 +52,7 @@ def resolve_ensemble( actual_params = active_grass_params(requested_params) domain = read_domain(simulations[0].domain.region) effective_mask = resolve_effective_mask(simulations[0].domain.mask) - input_cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] = {} + input_cache: dict[str, tuple[str, Literal["raster", "strds"]]] = {} swmm_cache: dict[Path, str] = {} results: list[ResolvedSimulation | ValidationFailure] = [] for expanded in simulations: @@ -141,7 +141,7 @@ def _build_resolved_simulation( def _resolve_inputs( inputs: dict[str, str], *, - cache: dict[str, tuple[str, Literal["raster", "strds"]] | Exception] | None = None, + cache: dict[str, tuple[str, Literal["raster", "strds"]]] | None = None, ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: from itzi.grass.utils import resolve_input_identifier @@ -150,16 +150,9 @@ def _resolve_inputs( for key, identifier in inputs.items(): result = cache.get(identifier) if cache is not None else None if result is None: - try: - result = resolve_input_identifier(identifier) - except Exception as error: - if cache is not None: - cache[identifier] = error - raise + result = resolve_input_identifier(identifier) if cache is not None: cache[identifier] = result - if isinstance(result, Exception): - raise result source, kind = result resolved[key] = source kinds[key] = kind diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index caacb38c..a91c2019 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -125,16 +125,11 @@ def resolve_effective_mask(mask: str | None) -> EffectiveMask: if not mapset: raise EnsembleError(f"explicit mask {mask!r} was not found") return EffectiveMask("explicit", f"{name}@{mapset}") - if has_mask(): + if gutils.get_mapset_raster("MASK", current_mapset): return EffectiveMask("active", f"MASK@{current_mapset}") return EffectiveMask("none", None) -def has_mask() -> bool: - """Return True if the mapset has a mask, False otherwise.""" - return bool(gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET"))) - - def format_id(name: str) -> str: """Qualify a raster name using the GRASS search path if it exists.""" if "@" in name: From 7c6f619337ba41817b9cefe82de47763beb4829b Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 19:18:51 -0600 Subject: [PATCH 63/85] fix cli tests --- tests/cli/test_cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 87eeb4aa..ebd97df1 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -20,7 +20,7 @@ itzi_run, itzi_run_one, main, - preflight_ensemble_worker, + preflight_worker, reconcile_hotstart_commands, sim_runner_worker, ) @@ -145,7 +145,7 @@ def display(resolved, *_args, **_kwargs): monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_subprocess", resolve) monkeypatch.setattr("itzi.itzi.validate_resolved_ensemble", validate) monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_, **__: None) - monkeypatch.setattr("itzi.itzi._preflight_ensemble_in_subprocess", preflight) + monkeypatch.setattr("itzi.itzi._preflight_in_subprocess", preflight) monkeypatch.setattr("itzi.itzi._run_one_ensemble", run) monkeypatch.setattr("itzi.itzi._display_dry_plan", display) @@ -172,7 +172,7 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): calls = [] monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) monkeypatch.setattr( - "itzi.itzi._preflight_ensemble_in_subprocess", + "itzi.itzi._preflight_in_subprocess", lambda simulations: ( calls.append(tuple(s.simulation_id for s in simulations)) or {"sim-failed": "invalid input"} @@ -268,7 +268,7 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo } -def test_preflight_ensemble_worker_keeps_member_failures(monkeypatch): +def test_preflight_worker_keeps_member_failures(monkeypatch): calls = [] class FakeGrassSessionManager: @@ -295,7 +295,7 @@ def preflight(simulation): for simulation_id in ("good", "bad", "another") ) - assert preflight_ensemble_worker(simulations) == {"bad": "ValueError: invalid input"} + assert preflight_worker((simulations)) == {"bad": "ValueError: invalid input"} assert calls == ["open", "good", "bad", "another", "close"] From d93941f0ab55403bd8f08758fc1e4d844cf5262d Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 19:41:57 -0600 Subject: [PATCH 64/85] fix typing errors in test, and init tgis in conftest --- tests/cli/test_cli.py | 66 ++++++++++++++++++++++--------- tests/cli/test_ensemble_config.py | 18 +++++---- tests/grass/conftest.py | 3 +- tests/grass/test_ensemble.py | 8 +++- tests/grass/test_grass_input.py | 5 ++- tests/test_grass_session.py | 16 ++++---- 6 files changed, 76 insertions(+), 40 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index ebd97df1..d9f1cd21 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -5,12 +5,21 @@ import sys from pathlib import Path from types import ModuleType, SimpleNamespace +from typing import cast import pytest +from itzi_core import DomainData, SimulationConfig import itzi.messenger as msgr from itzi.cli_parser import build_parser -from itzi.ensemble.models import ResolvedEnsemble, ResolvedSimulation +from itzi.ensemble.models import ( + ArtifactSummary, + EffectiveMask, + ExpandedEnsemble, + ResolvedEnsemble, + ResolvedSimulation, +) +from itzi.grass.session import GrassParams from itzi.itzi import ( VerbosityLevel, _preflight_ensemble, @@ -77,7 +86,7 @@ def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_, **__: None) _run_ensemble_batch( - SimpleNamespace(config_file=["study.yaml"], resume_from=[], member=[], dry=True) + argparse.Namespace(config_file=["study.yaml"], resume_from=[], member=[], dry=True) ) assert calls == [simulations] @@ -100,19 +109,24 @@ def test_ensembles_are_checked_and_run_in_order_without_cross_ensemble_validatio name: ResolvedSimulation( simulation_id=name, coordinates=(), - grass_params=None, - domain_data=None, - effective_mask=None, + grass_params=cast(GrassParams, None), + domain_data=cast(DomainData, None), + effective_mask=cast(EffectiveMask, None), input_kinds=(), - simulation_config=SimpleNamespace( - input_map_names={"friction": "first_water_depth@PERMANENT"} - if name == "second" - else {} + simulation_config=cast( + SimulationConfig, + SimpleNamespace( + input_map_names={"friction": "first_water_depth@PERMANENT"} + if name == "second" + else {} + ), ), - artifacts=SimpleNamespace( + artifacts=ArtifactSummary( output_map_names=(("water_depth", "first_water_depth"),) if name == "first" else (), + drainage_output=None, + statistics_file=None, ), normalized_payload=name, ) @@ -150,7 +164,7 @@ def display(resolved, *_args, **_kwargs): monkeypatch.setattr("itzi.itzi._display_dry_plan", display) _run_ensemble_batch( - SimpleNamespace(config_file=["first.yaml", "second.yaml"], member=[], dry=dry, o=True) + argparse.Namespace(config_file=["first.yaml", "second.yaml"], member=[], dry=dry, o=True) ) assert calls == [ @@ -181,7 +195,12 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): member_failures, manifest_failure = _preflight_ensemble( ResolvedEnsemble( - ensemble, (selected_simulation, selected_with_failure, unselected_simulation), () + cast(ExpandedEnsemble, ensemble), + cast( + tuple[ResolvedSimulation, ...], + (selected_simulation, selected_with_failure, unselected_simulation), + ), + (), ), {"sim-selected", "sim-failed"}, has_selectors=True, @@ -195,7 +214,9 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): def test_select_members_only_matches_current_ensemble(): resolved = ResolvedEnsemble( - SimpleNamespace(ensemble_id="first"), (SimpleNamespace(simulation_id="member"),), () + cast(ExpandedEnsemble, SimpleNamespace(ensemble_id="first")), + cast(tuple[ResolvedSimulation, ...], (SimpleNamespace(simulation_id="member"),)), + (), ) assert _select_members(resolved, [("second#member", "second#member")]) == set() assert _select_members(resolved, [("first#member", "first#member")]) == {"member"} @@ -207,7 +228,7 @@ def test_unqualified_selector_requires_one_ensemble(monkeypatch): with pytest.raises(RuntimeError, match="Unqualified --member IDs"): _run_ensemble_batch( - SimpleNamespace(config_file=["first.yaml", "second.yaml"], member=["member"]) + argparse.Namespace(config_file=["first.yaml", "second.yaml"], member=["member"]) ) @@ -249,7 +270,9 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo ) count = _run_one_ensemble( - ResolvedEnsemble(ensemble, simulations, ()), + ResolvedEnsemble( + cast(ExpandedEnsemble, ensemble), cast(tuple[ResolvedSimulation, ...], simulations), () + ), {"good", "bad", "preflight"}, {"preflight": {"phase": "preflight", "detail": "invalid input"}}, None, @@ -288,14 +311,16 @@ def preflight(simulation): monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) fake_preflight = ModuleType("itzi.ensemble.preflight") - fake_preflight.preflight_simulation = preflight + setattr(fake_preflight, "preflight_simulation", preflight) monkeypatch.setitem(sys.modules, "itzi.ensemble.preflight", fake_preflight) simulations = tuple( SimpleNamespace(simulation_id=simulation_id, grass_params="shared") for simulation_id in ("good", "bad", "another") ) - assert preflight_worker((simulations)) == {"bad": "ValueError: invalid input"} + assert preflight_worker(cast(tuple[ResolvedSimulation, ...], simulations)) == { + "bad": "ValueError: invalid input" + } assert calls == ["open", "good", "bad", "another", "close"] @@ -473,7 +498,10 @@ def test_reconcile_hotstart_commands_accepts_single_resume_for_single_config(): def test_reconcile_hotstart_commands_matches_multiple_named_values(): config_file_list = ["/tmp/a.ini", "/tmp/b.ini", "/tmp/c.ini"] - resume_from_list = [("c.ini", "restart_c.zip"), ("a.ini", "restart_a.zip")] + resume_from_list: list[tuple[str | None, str]] = [ + ("c.ini", "restart_c.zip"), + ("a.ini", "restart_a.zip"), + ] assert reconcile_hotstart_commands(config_file_list, resume_from_list) == [ ("/tmp/a.ini", "restart_a.zip"), @@ -484,7 +512,7 @@ def test_reconcile_hotstart_commands_matches_multiple_named_values(): def test_reconcile_hotstart_commands_accepts_duplicate_basenames_with_paths(): config_file_list = ["./sim1/config.ini", "sim2/config.ini"] - resume_from_list = [ + resume_from_list: list[tuple[str | None, str]] = [ ("./sim1/config.ini", "sim1/hotstart.zip"), ("sim2/config.ini", "sim2/hotstart.zip"), ] diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 866c1c0a..8b7c1cf1 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta import sys from types import SimpleNamespace +from typing import cast import pytest from itzi_core import InfiltrationModelType, SurfaceFlowParameters, TemporalType @@ -16,6 +17,7 @@ from itzi.ensemble.models import ( MAX_ENSEMBLE_MEMBERS, ArtifactSummary, + ResolvedSimulation, format_duration, parse_duration, render_template, @@ -365,17 +367,15 @@ def test_duration_and_template_helpers_are_strict(): def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): - expanded = SimpleNamespace( - ensemble_id="study", - source=SimpleNamespace(path=tmp_path / "study.yaml"), - outputs=SimpleNamespace( + expanded = load_yaml_stream(write_yaml(tmp_path, document())).ensembles[0].simulations[0] + expanded = replace( + expanded, + outputs=replace( + expanded.outputs, raster_prefix=None, raster_variables=(), statistics_file="results/{simulation}.csv", - drainage_dataset=None, ), - drainage=None, - time=SimpleNamespace(duration=timedelta(hours=1), record_step=timedelta(minutes=5)), ) monkeypatch.setattr("itzi.ensemble.resolution._validate_output_names", lambda *_: None) @@ -430,7 +430,9 @@ def test_later_generated_output_cannot_alias_an_input(): ) with pytest.raises(ValueError, match="result_0002@PERMANENT.*aliases"): - validate_resolved_ensemble((output, input_simulation)) + validate_resolved_ensemble( + (cast(ResolvedSimulation, output), cast(ResolvedSimulation, input_simulation)) + ) def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index f675144a..da5d35c5 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -169,7 +169,8 @@ def grass_5by5(grass_xy_session, test_data_path): gscript.mapcalc("infiltration_rate=2") gscript.mapcalc("loss_rate=1.5") gscript.mapcalc("inflow_rate=0.1") - return None + with session.GrassSessionManager(session.GrassParams()): + yield @pytest.fixture(scope="class") diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index ff7a10d4..72475a12 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from pathlib import Path from types import SimpleNamespace +from typing import cast from uuid import uuid4 import grass.script as gscript @@ -34,7 +35,9 @@ def write_study(tmp_path: Path, **changes: dict) -> Path: } document["options"] = changes.pop("options", document["options"]) for section, fields in changes.items(): - document[section].update(fields) + values = document[section] + assert isinstance(values, dict) + values.update(fields) path = tmp_path / "study.yaml" path.write_text(yaml.safe_dump(document), encoding="utf-8") return path @@ -506,6 +509,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): def test_preflight_accepts_unoccupied_drainage_id(): from itzi.grass.utils import get_current_mapset from itzi.ensemble.preflight import _validate_grass_outputs + from itzi.grass.interface import GrassInterface GrassSessionManager.ensure_temporal_initialized() name = f"drainage_collision_{uuid4().hex[:8]}" @@ -524,4 +528,4 @@ def test_preflight_accepts_unoccupied_drainage_id(): validate_output_stds_temporal_type=lambda *_: None, ) - _validate_grass_outputs(simulation, interface) + _validate_grass_outputs(cast(ResolvedSimulation, simulation), cast(GrassInterface, interface)) diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index 25575cda..01e5b097 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Literal, cast import numpy as np import pytest @@ -41,6 +41,7 @@ def test_input_kinds_share_raster_and_strds_construction( monkeypatch: pytest.MonkeyPatch, ) -> None: from itzi.grass import utils + from itzi.grass.interface import GrassInterface from itzi.providers.grass_input import GrassRasterInputProvider from itzi.grass.utils import MapData @@ -58,7 +59,7 @@ def test_input_kinds_share_raster_and_strds_construction( interface = FakeGrassInterface() provider = GrassRasterInputProvider( - interface, + cast(GrassInterface, interface), {"ground_elevation": "raster", "rainfall_rate": "series"}, start, end, diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index 7b0ea31a..96ed0c76 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -17,9 +17,9 @@ def finish(self) -> None: def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: session = FakeGrassSession({"GISRC": "/tmp/worker-gisrc"}) grass_script = ModuleType("grass.script") - grass_script.setup = SimpleNamespace(init=lambda **_: session) + grass_script.__dict__["setup"] = SimpleNamespace(init=lambda **_: session) grass_package = ModuleType("grass") - grass_package.script = grass_script + grass_package.__dict__["script"] = grass_script monkeypatch.setattr("itzi.grass.session.os.access", lambda *_: True) monkeypatch.setattr(GrassSessionManager, "ensure_temporal_initialized", lambda self: None) @@ -41,7 +41,7 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: monkeypatch.setitem(sys.modules, "grass", grass_package) monkeypatch.setitem(sys.modules, "grass.script", grass_script) - grass_params = SimpleNamespace( + grass_params = GrassParams( grassdata="/grassdata", location="location", mapset="mapset", @@ -59,13 +59,13 @@ def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: def test_temporal_initialization_once_per_active_session(monkeypatch) -> None: calls: list[str] = [] script = ModuleType("grass.script") - script.set_raise_on_error = lambda _: calls.append("script") + script.__dict__["set_raise_on_error"] = lambda _: calls.append("script") temporal = ModuleType("grass.temporal") - temporal.init = lambda **_: calls.append("init") - temporal.set_raise_on_error = lambda _: calls.append("temporal") + temporal.__dict__["init"] = lambda **_: calls.append("init") + temporal.__dict__["set_raise_on_error"] = lambda _: calls.append("temporal") package = ModuleType("grass") - package.script = script - package.temporal = temporal + package.__dict__["script"] = script + package.__dict__["temporal"] = temporal monkeypatch.setitem(sys.modules, "grass", package) monkeypatch.setitem(sys.modules, "grass.script", script) monkeypatch.setitem(sys.modules, "grass.temporal", temporal) From 847176b099b12dfcceffa7cd903d68cd8e8857c1 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 21:02:05 -0600 Subject: [PATCH 65/85] fix tutorial tests --- tests/grass/test_tutorial.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/grass/test_tutorial.py b/tests/grass/test_tutorial.py index 7657e613..46175694 100644 --- a/tests/grass/test_tutorial.py +++ b/tests/grass/test_tutorial.py @@ -14,6 +14,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader +from itzi.grass.session import GrassSessionManager DATA_EPSG = "3358" @@ -29,15 +30,20 @@ def grass_tutorial_session(test_data_temp_path): """Create a GRASS session in a new location and PERMANENT mapset""" # Keep all generated files in the test_data_temp_path os.chdir(test_data_temp_path) + workdir = tempfile.TemporaryDirectory(dir=test_data_temp_path) + os.chdir(workdir.name) tmpdir = tempfile.TemporaryDirectory() gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) grass_session = gscript.setup.init( path=tmpdir.name, location="itzi_tutorial", mapset="PERMANENT" ) + GrassSessionManager.ensure_temporal_initialized() os.environ["GRASS_VERBOSE"] = "1" yield grass_session grass_session.finish() tmpdir.cleanup() + os.chdir(test_data_temp_path) + workdir.cleanup() @pytest.fixture(scope="class") @@ -95,7 +101,7 @@ def itzi_tutorial(grass_tutorial_session, tutorial_test_file): @pytest.mark.slow @pytest.mark.usefixtures("itzi_tutorial", "test_data_path") class TestItziTutorial: - def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): + def test_tutorial(itzi_tutorial, test_data_path): """Run the tutorial simulation. Check the results.""" # Run the simulation config_file = os.path.join(test_data_path, "tutorial_files", "tutorial.ini") @@ -105,7 +111,7 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): conf_data.sim_config, conf_data.grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() sim_runner.run().finalize() # Check the results hmax_maps = gscript.list_grouped("raster", pattern="nc_itzi_tutorial_max_water_depth_*")[ @@ -116,7 +122,7 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): assert float(h_max_univar["mean_of_abs"]) == pytest.approx(0.0355, abs=1e-3) # Test consistency of stats file - stat_file_path = pathlib.Path(test_data_temp_path) / pathlib.Path("nc_itzi_tutorial.csv") + stat_file_path = pathlib.Path("nc_itzi_tutorial.csv") df_stats = pd.read_csv(stat_file_path, na_values="-") df_stats.set_index("simulation_time", drop=True, inplace=True, verify_integrity=True) df_stats.index = pd.to_timedelta(df_stats.index) @@ -134,7 +140,7 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): "'volume_change' inconsistent with other values" ) - def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, helpers): + def test_tutorial_drainage(itzi_tutorial, test_data_path, helpers): """Run the tutorial simulation with drainage.""" # Set the config file dynamically to make sure it can find the INP file inp_file = os.path.join(test_data_path, "tutorial_files", "tutorial_drainage.inp") @@ -157,7 +163,7 @@ def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, h } parser = ConfigParser() parser.read_dict(config_dict) - config_file = os.path.join(test_data_temp_path, "tutorial_drainage.ini") + config_file = "tutorial_drainage.ini" with open(config_file, "w") as f: parser.write(f) @@ -167,13 +173,11 @@ def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, h conf_data.sim_config, conf_data.grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() sim_runner.run().finalize() # Test consistency of stats file - stat_file_path = pathlib.Path(test_data_temp_path) / pathlib.Path( - "nc_itzi_tutorial_drainage.csv" - ) + stat_file_path = pathlib.Path("nc_itzi_tutorial_drainage.csv") df_stats = pd.read_csv(stat_file_path, na_values="-") df_stats.set_index("simulation_time", drop=True, inplace=True, verify_integrity=True) df_stats.index = pd.to_timedelta(df_stats.index) @@ -278,7 +282,7 @@ def test_tutorial_drainage_without_vector_output(itzi_tutorial, test_data_path): conf_data.sim_config, conf_data.grass_params, stats_file=conf_data.stats_file, - ) + ).initialize() try: assert sim_runner.sim.report.vector_provider is None From fc92781e9cfcc82ff65113df245a44bc237cb9e0 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Tue, 29 Sep 2026 21:21:25 -0600 Subject: [PATCH 66/85] put validation in GrassParams --- src/itzi/configreader.py | 18 ++++++------------ src/itzi/ensemble/models.py | 6 +----- src/itzi/grass/session.py | 19 +++++++++++-------- tests/cli/test_configreader.py | 14 +++++++++++++- tests/cli/test_ensemble_config.py | 6 ++++++ tests/test_grass_session.py | 20 ++++++++++++++++++++ 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index b87cbb62..8a127dd9 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -79,8 +79,7 @@ "capillary_pressure", "hydraulic_conductivity", ) -GRASS_MANDATORY_KEYS = ("grassdata", "location", "mapset") -GRASS_OPTION_KEYS = (*GRASS_MANDATORY_KEYS, "region", "mask", "grass_bin") +GRASS_OPTION_KEYS = ("grassdata", "location", "mapset", "region", "mask", "grass_bin") SURFACE_FLOW_OPTION_KEYS = tuple(SurfaceFlowParameters.model_fields) @@ -248,7 +247,11 @@ def _resolve_swmm_input_path(swmm_inp: str, config_file: str) -> Path: def _read_grass_params(params: ConfigParser) -> GrassParams: """Build GRASS session parameters from the config file.""" - return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) + try: + return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) + except ValueError as error: + msgr.fatal(str(error)) + raise AssertionError("unreachable") from error class SimulationTimes(BaseModel): @@ -358,7 +361,6 @@ def __init__(self, filename: str | None) -> None: infiltration_model = self._resolve_infiltration_model(self.input_map_names) self.grass_params = _read_grass_params(params) - self._check_grass_params(self.grass_params) surface_flow_parameters = _read_surface_flow_parameters(params) assert self.sim_times.record_step is not None @@ -402,14 +404,6 @@ def __init__(self, filename: str | None) -> None: except ValidationError as error: _fatal_validation_error(error) - def _check_grass_params(self, grass_params: GrassParams) -> None: - """Ensure mandatory GRASS settings are provided together.""" - grass_values = grass_params.model_dump() - grass_any = any(grass_values[key] for key in GRASS_MANDATORY_KEYS) - grass_all = all(grass_values[key] for key in GRASS_MANDATORY_KEYS) - if grass_any and not grass_all: - msgr.fatal(f"{GRASS_MANDATORY_KEYS} are mutualy inclusive") - def _resolve_infiltration_model( self, input_map_names: dict[str, str | None] ) -> InfiltrationModelType: diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index 2cc5bc96..192cec4a 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -96,11 +96,7 @@ class GrassContextConfig(StrictModel): @model_validator(mode="after") def validate_context(self) -> GrassContextConfig: - values = (self.database, self.project, self.mapset) - if any(value is not None for value in values) and not all( - value is not None for value in values - ): - raise ValueError("database, project, and mapset must be supplied together") + GrassParams(grassdata=self.database, location=self.project, mapset=self.mapset) if self.executable is not None and self.database is None: raise ValueError("executable requires database, project, and mapset") return self diff --git a/src/itzi/grass/session.py b/src/itzi/grass/session.py index 69a95dc8..46bae5a6 100644 --- a/src/itzi/grass/session.py +++ b/src/itzi/grass/session.py @@ -17,20 +17,18 @@ import os import subprocess import sys +from dataclasses import dataclass from pathlib import Path -from pydantic import BaseModel, ConfigDict - import itzi.messenger as msgr _initialized_temporal_session: tuple[int, str, tuple[str, str, str]] | None = None -class GrassParams(BaseModel): +@dataclass(frozen=True) +class GrassParams: """Parameters for GRASS GIS session.""" - model_config = ConfigDict(frozen=True) - grassdata: str | None = None location: str | None = None mapset: str | None = None @@ -38,6 +36,13 @@ class GrassParams(BaseModel): mask: str | None = None grass_bin: str | None = None + def __post_init__(self) -> None: + context = (self.grassdata, self.location, self.mapset) + if any(value is not None for value in context) and not all( + value is not None for value in context + ): + raise ValueError("GRASS database, location, and mapset must be supplied together") + class GrassSessionManager: """Manages GRASS session lifecycle.""" @@ -74,10 +79,8 @@ def _validate_requested_context(self) -> None: self.grass_params.location, self.grass_params.mapset, ) - if not any((gisdb, location, mapset)): + if gisdb is None: return - if not gisdb or not location or not mapset: - msgr.fatal("GRASS database, location, and mapset must be supplied together") requested_database = str(Path(gisdb).expanduser().resolve()) active_database = str(Path(active[0]).expanduser().resolve()) if (requested_database, location, mapset) != ( diff --git a/tests/cli/test_configreader.py b/tests/cli/test_configreader.py index 90999465..4c667e37 100644 --- a/tests/cli/test_configreader.py +++ b/tests/cli/test_configreader.py @@ -2,6 +2,7 @@ import logging from configparser import ConfigParser +from dataclasses import asdict from datetime import datetime, timedelta from pathlib import Path @@ -72,7 +73,7 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): assert sim_config.submerged_weir_coeff == DefaultValues.SUBMERGED_WEIR_COEFF assert sim_config.input_map_names == {"ground_elevation": "z", "friction": "n"} assert sim_config.output_map_names == {"water_depth": "out_water_depth"} - assert grass_params.model_dump() == { + assert asdict(grass_params) == { "grassdata": None, "location": None, "mapset": None, @@ -82,6 +83,17 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): } +def test_reader_rejects_partial_grass_context(tmp_path): + config_file = write_config_file( + tmp_path, make_config_dict(grass={"grassdata": "/grassdata", "location": "project"}) + ) + + with pytest.raises( + RuntimeError, match="GRASS database, location, and mapset must be supplied together" + ): + ConfigReader(config_file) + + @pytest.mark.parametrize( ("statistics", "expected_stats_file"), [ diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 8b7c1cf1..12756476 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -17,6 +17,7 @@ from itzi.ensemble.models import ( MAX_ENSEMBLE_MEMBERS, ArtifactSummary, + GrassContextConfig, ResolvedSimulation, format_duration, parse_duration, @@ -148,6 +149,11 @@ def test_schema_constraints_reject_invalid_values(model, value): model.model_validate(value) +def test_grass_context_requires_complete_context(): + with pytest.raises(ValidationError, match="mapset must be supplied together"): + GrassContextConfig(database="/grassdata", project="project") + + def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): content = document( extra="""\ diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index 96ed0c76..ca9bc1c8 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -2,9 +2,29 @@ import sys from types import ModuleType, SimpleNamespace +import pytest + from itzi.grass.session import GrassParams, GrassSessionManager +@pytest.mark.parametrize( + "values", + ( + {"grassdata": "/grassdata"}, + {"grassdata": "/grassdata", "location": "project"}, + {"mapset": "PERMANENT"}, + ), +) +def test_grass_params_requires_complete_context(values: dict[str, str]) -> None: + with pytest.raises( + ValueError, match="GRASS database, location, and mapset must be supplied together" + ): + GrassParams(**values) + + assert GrassParams(region="region").grassdata is None + assert GrassParams("/grassdata", "project", "PERMANENT").mapset == "PERMANENT" + + class FakeGrassSession: def __init__(self, env: dict[str, str]) -> None: self.env = env From 5de9d972e90d1dd32a346e087a447000a910e3c7 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 10:00:25 -0600 Subject: [PATCH 67/85] input_kinds should be supplied by the caller, simplify cache, and use pydantic to check field length --- src/itzi/ensemble/resolution.py | 11 +++++------ src/itzi/ensemble/schema.py | 4 +--- src/itzi/grass/utils.py | 22 ++++++++-------------- src/itzi/itzi.py | 2 +- src/itzi/providers/grass_input.py | 2 +- tests/grass/test_grass_input.py | 18 ++++-------------- 6 files changed, 20 insertions(+), 39 deletions(-) diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index 7dcd8513..cc04d255 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -145,15 +145,14 @@ def _resolve_inputs( ) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: from itzi.grass.utils import resolve_input_identifier + if cache is None: + cache = {} resolved: dict[str, str] = {} kinds: dict[str, Literal["raster", "strds"]] = {} for key, identifier in inputs.items(): - result = cache.get(identifier) if cache is not None else None - if result is None: - result = resolve_input_identifier(identifier) - if cache is not None: - cache[identifier] = result - source, kind = result + if identifier not in cache: + cache[identifier] = resolve_input_identifier(identifier) + source, kind = cache[identifier] resolved[key] = source kinds[key] = kind return resolved, kinds diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index 372b2f21..d1313515 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -134,13 +134,11 @@ class DrainageSweepConfig(StrictModel): class RasterOutputs(StrictModel): prefix: StrictStr - variables: list[StrictStr] + variables: list[StrictStr] = Field(min_length=1) @field_validator("variables") @classmethod def validate_variables(cls, value: list[str]) -> list[str]: - if not value: - raise ValueError("must not be empty") unknown = sorted(set(value) - set(OUTPUT_ARRAY_KEYS)) if unknown: raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index a91c2019..a5e2940c 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -305,7 +305,7 @@ def resolve_input_map_lists( map_names: Mapping[str, str], start_time: datetime, end_time: datetime, - input_kinds: Mapping[str, Literal["raster", "strds"]] | None, + input_kinds: Mapping[str, Literal["raster", "strds"]], ) -> dict[str, list[MapData] | None]: """Resolve provider-ready raster lists without creating an output provider.""" invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) @@ -316,21 +316,15 @@ def resolve_input_map_lists( for key, map_name in map_names.items(): if not map_name: continue - kind = input_kinds.get(key) if input_kinds is not None else None - if kind is None: - map_id, kind = resolve_input_identifier(map_name) - elif kind == "raster": - map_id = format_id(map_name) - else: - map_id = map_name if "@" in map_name else resolve_input_identifier(map_name)[0] + kind = input_kinds[key] if kind == "strds": - if not name_is_stds(map_id): - msgr.fatal(f"STRDS input <{map_id}> is no longer available") - elif kind == "raster" and not name_is_map(map_id): - msgr.fatal(f"raster input <{map_id}> is no longer available") + if not name_is_stds(map_name): + msgr.fatal(f"STRDS input <{map_name}> is no longer available") + elif kind == "raster" and not name_is_map(map_name): + msgr.fatal(f"raster input <{map_name}> is no longer available") if kind == "strds": - map_list = grass_interface.raster_list_from_strds(map_id) + map_list = grass_interface.raster_list_from_strds(map_name) else: - map_list = [MapData(id=map_id, start_time=start_time, end_time=end_time)] + map_list = [MapData(id=map_name, start_time=start_time, end_time=end_time)] map_lists[key] = map_list return map_lists diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 90a67a36..d74544d4 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -458,7 +458,7 @@ def _run_one_ensemble( status, detail = _run_simulation_in_subprocess(simulation) state = states[simulation.simulation_id] state["status"] = status - state["elapsed_seconds"] = format(time.monotonic() - started, ".2f") + state["elapsed_seconds"] = time.monotonic() - started if detail is not None: states[simulation.simulation_id]["failure"] = { "phase": "execution", diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 0a7adb42..ba98ba98 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -34,7 +34,7 @@ def __init__( input_map_names: Mapping[str, str], default_start_time: datetime, default_end_time: datetime, - input_kinds: Mapping[str, Literal["raster", "strds"]] | None, + input_kinds: Mapping[str, Literal["raster", "strds"]], ) -> None: from itzi.grass.utils import resolve_input_map_lists diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index 01e5b097..bcad3e75 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Literal, cast +from typing import TYPE_CHECKING, cast import numpy as np import pytest @@ -33,11 +33,7 @@ def raster_list_from_strds(self, strds_id: str) -> list[MapData]: return [MapData(strds_id, start, start + timedelta(hours=1))] -@pytest.mark.parametrize( - "input_kinds", [None, {"ground_elevation": "raster", "rainfall_rate": "strds"}] -) -def test_input_kinds_share_raster_and_strds_construction( - input_kinds: dict[str, Literal["raster", "strds"]] | None, +def test_resolved_input_kinds_share_raster_and_strds_construction( monkeypatch: pytest.MonkeyPatch, ) -> None: from itzi.grass import utils @@ -45,12 +41,6 @@ def test_input_kinds_share_raster_and_strds_construction( from itzi.providers.grass_input import GrassRasterInputProvider from itzi.grass.utils import MapData - monkeypatch.setattr( - utils, - "resolve_input_identifier", - lambda name: (f"{name}@test", "strds" if name == "series" else "raster"), - ) - monkeypatch.setattr(utils, "format_id", lambda name: f"{name}@test") monkeypatch.setattr(utils, "name_is_stds", lambda map_id: map_id == "series@test") monkeypatch.setattr(utils, "name_is_map", lambda map_id: map_id == "raster@test") @@ -60,10 +50,10 @@ def test_input_kinds_share_raster_and_strds_construction( provider = GrassRasterInputProvider( cast(GrassInterface, interface), - {"ground_elevation": "raster", "rainfall_rate": "series"}, + {"ground_elevation": "raster@test", "rainfall_rate": "series@test"}, start, end, - input_kinds, + {"ground_elevation": "raster", "rainfall_rate": "strds"}, ) assert provider.map_lists["ground_elevation"] == [MapData("raster@test", start, end)] From e0944290cd7908b3b8756d77f76e1d42744a2dbf Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 10:27:12 -0600 Subject: [PATCH 68/85] pass EffectiveMask directly, remove format_id --- src/itzi/ensemble/preflight.py | 2 +- src/itzi/grass/interface.py | 32 +++++++------------------------- src/itzi/grass/utils.py | 7 ------- src/itzi/simulation_runner.py | 4 +--- tests/grass/test_ensemble.py | 2 +- tests/grass/test_grass_input.py | 5 ----- 6 files changed, 10 insertions(+), 42 deletions(-) diff --git a/src/itzi/ensemble/preflight.py b/src/itzi/ensemble/preflight.py index a9025da9..c7c552d6 100644 --- a/src/itzi/ensemble/preflight.py +++ b/src/itzi/ensemble/preflight.py @@ -42,7 +42,7 @@ def preflight_simulation(simulation: ResolvedSimulation) -> None: dtype=np.float32, region_id=simulation.grass_params.region, raster_mask_id=simulation.grass_params.mask, - effective_mask=(simulation.effective_mask.mode, simulation.effective_mask.source), + effective_mask=simulation.effective_mask, ) try: _validate_inputs(simulation, interface) diff --git a/src/itzi/grass/interface.py b/src/itzi/grass/interface.py index d3e98759..f506519c 100644 --- a/src/itzi/grass/interface.py +++ b/src/itzi/grass/interface.py @@ -36,6 +36,7 @@ ) import itzi.messenger as msgr +from itzi.ensemble.models import EffectiveMask from itzi.grass.utils import ( MapData, RasterMapType, @@ -102,14 +103,13 @@ def __init__( dtype, region_id: str | None, raster_mask_id: str | None, - effective_mask: tuple[str, str | None] | None = None, + effective_mask: EffectiveMask | None = None, ) -> None: assert isinstance(start_time, datetime), "start_time not a datetime object!" assert isinstance(end_time, datetime), "end_time not a datetime object!" assert start_time <= end_time, "start_time > end_time!" self.region_id = region_id - self.raster_mask_id = raster_mask_id self.start_time = start_time self.end_time = end_time self.dtype = dtype @@ -127,15 +127,11 @@ def __init__( # Check if region is at least 3x3 if self.xr < 3 or self.yr < 3: msgr.fatal("GRASS Region should be at least 3 cells by 3 cells") - self.dx = self.region.ewres - self.dy = self.region.nsres - self.reg_bbox = { - "e": self.region.east, - "w": self.region.west, - "n": self.region.north, - "s": self.region.south, - } - self.mask_mode, self.mask_source = self._select_effective_mask(effective_mask) + selected_mask = effective_mask or resolve_effective_mask(raster_mask_id) + if selected_mask.mode != "none" and selected_mask.source is None: + msgr.fatal(f"Effective {selected_mask.mode} mask has no source") + self.mask_mode = selected_mask.mode + self.mask_source = selected_mask.source if selected_mask.mode != "none" else None self.overwrite = gscript.overwrite() def __enter__(self): @@ -182,20 +178,6 @@ def to_datetime(self, unit: str, time: int) -> datetime: """ return self.start_time + timedelta(seconds=self.to_s(unit, time)) - def _select_effective_mask( - self, effective_mask: tuple[str, str | None] | None - ) -> tuple[str, str | None]: - """Select a mask descriptor without modifying the mapset-wide MASK.""" - if effective_mask is not None: - mode, source = effective_mask - if mode == "none": - return mode, None - if source is None: - msgr.fatal(f"Effective {mode} mask has no source") - return mode, source - selected = resolve_effective_mask(self.raster_mask_id) - return selected.mode, selected.source - def get_domain_data(self) -> DomainData: return read_domain(None) diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py index a5e2940c..1a571734 100644 --- a/src/itzi/grass/utils.py +++ b/src/itzi/grass/utils.py @@ -130,13 +130,6 @@ def resolve_effective_mask(mask: str | None) -> EffectiveMask: return EffectiveMask("none", None) -def format_id(name: str) -> str: - """Qualify a raster name using the GRASS search path if it exists.""" - if "@" in name: - return name - return f"{name}@{gutils.get_mapset_raster(name) or gutils.getenv('MAPSET')}" - - def qualify_output_id(name: str, mapset: str) -> str: """Give a new output its destination mapset, never a searched mapset.""" map_name, requested_mapset = split_identifier(name) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py index 3d1bae2a..f1c62e47 100644 --- a/src/itzi/simulation_runner.py +++ b/src/itzi/simulation_runner.py @@ -103,9 +103,7 @@ def initialize(self) -> Self: dtype=data_type, region_id=self.grass_params.region, raster_mask_id=self.grass_params.mask, - effective_mask=(self.effective_mask.mode, self.effective_mask.source) - if self.effective_mask is not None - else None, + effective_mask=self.effective_mask, ) msgr.verbose("Setting up GRASS simulation...") diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 72475a12..d6977eec 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -176,7 +176,7 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path np.float32, None, "stage_mask_values", - ("explicit", resolved.effective_mask.source), + resolved.effective_mask, ) try: explicit = interface.get_npmask() diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py index bcad3e75..2c4adc28 100644 --- a/tests/grass/test_grass_input.py +++ b/tests/grass/test_grass_input.py @@ -66,7 +66,6 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No import grass.script as gscript from itzi.grass.utils import ( - format_id, output_name, qualify_output_id, write_raster_map_blocking, @@ -76,9 +75,6 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) gscript.mapcalc("visible_output=1") gscript.run_command("g.mapset", mapset="child", flags="c") - assert format_id("visible_output") == "visible_output@PERMANENT" - assert format_id("visible_output@PERMANENT") == "visible_output@PERMANENT" - output_id = qualify_output_id("visible_output", "child") assert output_id == "visible_output@child" assert derived_record_name(output_id, 0) == "visible_output_0000@child" @@ -90,7 +86,6 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No write_raster_map_blocking( np.ones((3, 3), dtype=np.int32), "visible_output", "CELL", "ground_elevation", 0, False ) - assert format_id("visible_output") == "visible_output@child" @pytest.mark.forked From b023595d9910ede0ea3b25ef33e48ba778a01120 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 10:36:41 -0600 Subject: [PATCH 69/85] remove unused init re-exports --- src/itzi/ensemble/__init__.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/itzi/ensemble/__init__.py b/src/itzi/ensemble/__init__.py index 25c82a2c..9774eeb3 100644 --- a/src/itzi/ensemble/__init__.py +++ b/src/itzi/ensemble/__init__.py @@ -1,11 +1,3 @@ """Configuration and execution plans for YAML ensembles.""" -from itzi.ensemble.yaml import ( - expand_yaml_document as expand_yaml_document, -) -from itzi.ensemble.yaml import ( - load_yaml_stream as load_yaml_stream, -) -from itzi.ensemble.yaml import ( - normalize_time as normalize_time, -) +from itzi.ensemble.yaml import load_yaml_stream as load_yaml_stream From fb1611d1229fffa0c526f8505489edc9dae1f928 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 11:05:06 -0600 Subject: [PATCH 70/85] rename preflight to run_validation --- .../{preflight.py => run_validation.py} | 2 +- src/itzi/itzi.py | 48 ++++++++++-------- tests/cli/test_cli.py | 50 ++++++++++--------- tests/grass/test_ensemble.py | 26 +++++----- 4 files changed, 66 insertions(+), 60 deletions(-) rename src/itzi/ensemble/{preflight.py => run_validation.py} (98%) diff --git a/src/itzi/ensemble/preflight.py b/src/itzi/ensemble/run_validation.py similarity index 98% rename from src/itzi/ensemble/preflight.py rename to src/itzi/ensemble/run_validation.py index c7c552d6..85b8e4f2 100644 --- a/src/itzi/ensemble/preflight.py +++ b/src/itzi/ensemble/run_validation.py @@ -28,7 +28,7 @@ from itzi.grass.interface import GrassInterface -def preflight_simulation(simulation: ResolvedSimulation) -> None: +def validate_run_requirements(simulation: ResolvedSimulation) -> None: """Validate one resolved simulation without creating user artifacts.""" from itzi.grass.interface import GrassInterface from itzi.grass.utils import ensure_min_version diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index d74544d4..f729c748 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -142,17 +142,17 @@ def resolver_worker( return resolve_ensemble(expanded) -def preflight_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: - """Preflight selected members in one GRASS session, retaining individual failures.""" +def run_validation_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: + """Validate selected members in one GRASS session, retaining individual failures.""" msgr.raise_on_error = True msgr._itzi_logger.set_verbosity(msgr.verbosity()) failures: dict[str, str] = {} with GrassSessionManager(simulations[0].grass_params): - from itzi.ensemble.preflight import preflight_simulation + from itzi.ensemble.run_validation import validate_run_requirements for simulation in simulations: try: - preflight_simulation(simulation) + validate_run_requirements(simulation) except Exception as error: failures[simulation.simulation_id] = f"{type(error).__name__}: {error}" return failures @@ -362,7 +362,7 @@ def _run_ensemble_batch(cli_args: Namespace) -> None: ensemble, tuple(successful), tuple(member_failures), artifact_failure ) selected_ids = _select_members(resolved, qualified_selectors) - preflight_failures, manifest_failure = _preflight_ensemble( + run_validation_failures, manifest_failure = _validate_ensemble_run_requirements( resolved, selected_ids, has_selectors=has_selectors, @@ -372,19 +372,21 @@ def _run_ensemble_batch(cli_args: Namespace) -> None: _display_dry_plan( resolved, selected_ids, - preflight_failures, + run_validation_failures, manifest_failure, has_selectors=has_selectors, ) dry_failed |= bool( - (resolved.failures and not has_selectors) or preflight_failures or manifest_failure + (resolved.failures and not has_selectors) + or run_validation_failures + or manifest_failure ) continue failure_count += _run_one_ensemble( resolved, selected_ids, - preflight_failures, + run_validation_failures, manifest_failure, has_selectors=has_selectors, overwrite=bool(cli_args.o), @@ -403,7 +405,7 @@ def _run_ensemble_batch(cli_args: Namespace) -> None: def _run_one_ensemble( resolved: ResolvedEnsemble, selected_ids: set[str], - preflight_failures: dict[str, dict[str, str]], + run_validation_failures: dict[str, dict[str, str]], manifest_failure: str | None, *, has_selectors: bool, @@ -417,7 +419,7 @@ def _run_one_ensemble( states = _initial_member_states( resolved.simulations, resolved.failures, selected_ids, has_selectors ) - for simulation_id, failure in preflight_failures.items(): + for simulation_id, failure in run_validation_failures.items(): states[simulation_id]["status"] = "validation_failed" states[simulation_id]["failure"] = failure if manifest_failure is not None: @@ -437,11 +439,13 @@ def _run_one_ensemble( if states[simulation.simulation_id]["status"] != "planned": continue if member_started: - detail = _preflight_in_subprocess((simulation,)).get(simulation.simulation_id) + detail = _validate_run_requirements_in_subprocess((simulation,)).get( + simulation.simulation_id + ) if detail is not None: states[simulation.simulation_id]["status"] = "validation_failed" states[simulation.simulation_id]["failure"] = { - "phase": "preflight", + "phase": "run_validation", "detail": detail, } _update_manifest(manifest_path, ensemble, states) @@ -505,18 +509,18 @@ def _run_simulation_in_subprocess(simulation: ResolvedSimulation) -> tuple[str, return "execution_failed", f"{type(error).__name__}: {error}" -def _preflight_in_subprocess( +def _validate_run_requirements_in_subprocess( simulations: tuple[ResolvedSimulation, ...], ) -> dict[str, str]: try: with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: - return executor.submit(preflight_worker, simulations).result() + return executor.submit(run_validation_worker, simulations).result() except Exception as error: detail = f"{type(error).__name__}: {error}" return {simulation.simulation_id: detail for simulation in simulations} -def _preflight_ensemble( +def _validate_ensemble_run_requirements( resolved: ResolvedEnsemble, selected_ids: set[str], *, @@ -542,14 +546,14 @@ def _preflight_ensemble( ) except EnsembleError as error: manifest_failure = str(error) - to_preflight = tuple( + to_validate = tuple( simulation for simulation in simulations if simulation.simulation_id in selected_ids and simulation.simulation_id not in failures ) - if to_preflight: - for simulation_id, detail in _preflight_in_subprocess(to_preflight).items(): - failures[simulation_id] = {"phase": "preflight", "detail": detail} + if to_validate: + for simulation_id, detail in _validate_run_requirements_in_subprocess(to_validate).items(): + failures[simulation_id] = {"phase": "run_validation", "detail": detail} return failures, manifest_failure @@ -575,7 +579,7 @@ def _select_members( def _display_dry_plan( resolved: ResolvedEnsemble, selected_ids: set[str], - preflight_failures: dict[str, dict[str, str]], + run_validation_failures: dict[str, dict[str, str]], manifest_failure: str | None, *, has_selectors: bool, @@ -585,8 +589,8 @@ def _display_dry_plan( for simulation in resolved.simulations: selection = "selected" if simulation.simulation_id in selected_ids else "not selected" msgr.message(f" {simulation.simulation_id} ({selection})") - if simulation.simulation_id in preflight_failures: - failure = preflight_failures[simulation.simulation_id] + if simulation.simulation_id in run_validation_failures: + failure = run_validation_failures[simulation.simulation_id] msgr.warning(f" {failure['phase']} failed: {failure['detail']}") for variable, name in simulation.artifacts.output_map_names: msgr.message(f" raster {variable}: {name}") diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index d9f1cd21..d6e24b00 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -22,14 +22,14 @@ from itzi.grass.session import GrassParams from itzi.itzi import ( VerbosityLevel, - _preflight_ensemble, + _validate_ensemble_run_requirements, _run_ensemble_batch, _run_one_ensemble, _select_members, itzi_run, itzi_run_one, main, - preflight_worker, + run_validation_worker, reconcile_hotstart_commands, sim_runner_worker, ) @@ -144,8 +144,8 @@ def validate(simulations, _sources): calls.append(("validate", simulations[0].simulation_id)) assert len(simulations) == 1 - def preflight(simulations): - calls.append(("preflight", simulations[0].simulation_id)) + def validate_run(simulations): + calls.append(("run_validation", simulations[0].simulation_id)) return {} def run(resolved, selected_ids, *_args, **_kwargs): @@ -159,7 +159,7 @@ def display(resolved, *_args, **_kwargs): monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_subprocess", resolve) monkeypatch.setattr("itzi.itzi.validate_resolved_ensemble", validate) monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_, **__: None) - monkeypatch.setattr("itzi.itzi._preflight_in_subprocess", preflight) + monkeypatch.setattr("itzi.itzi._validate_run_requirements_in_subprocess", validate_run) monkeypatch.setattr("itzi.itzi._run_one_ensemble", run) monkeypatch.setattr("itzi.itzi._display_dry_plan", display) @@ -170,11 +170,11 @@ def display(resolved, *_args, **_kwargs): assert calls == [ (step, name) for name in ("first", "second") - for step in ("resolve", "validate", "preflight", "display" if dry else "run") + for step in ("resolve", "validate", "run_validation", "display" if dry else "run") ] -def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): +def test_run_validation_only_checks_selected_members(tmp_path, monkeypatch): selected_simulation = SimpleNamespace(simulation_id="sim-selected") selected_with_failure = SimpleNamespace(simulation_id="sim-failed") unselected_simulation = SimpleNamespace(simulation_id="sim-unselected") @@ -186,14 +186,14 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): calls = [] monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) monkeypatch.setattr( - "itzi.itzi._preflight_in_subprocess", + "itzi.itzi._validate_run_requirements_in_subprocess", lambda simulations: ( calls.append(tuple(s.simulation_id for s in simulations)) or {"sim-failed": "invalid input"} ), ) - member_failures, manifest_failure = _preflight_ensemble( + member_failures, manifest_failure = _validate_ensemble_run_requirements( ResolvedEnsemble( cast(ExpandedEnsemble, ensemble), cast( @@ -208,7 +208,9 @@ def test_preflight_only_checks_selected_members(tmp_path, monkeypatch): ) assert calls == [("sim-selected", "sim-failed")] - assert member_failures == {"sim-failed": {"phase": "preflight", "detail": "invalid input"}} + assert member_failures == { + "sim-failed": {"phase": "run_validation", "detail": "invalid input"} + } assert manifest_failure is None @@ -241,11 +243,11 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo artifacts = SimpleNamespace(output_map_names=(), drainage_output=None, statistics_file=None) simulations = tuple( SimpleNamespace(simulation_id=name, coordinates=(), artifacts=artifacts) - for name in ("preflight", "unselected", "good", "bad") + for name in ("invalid", "unselected", "good", "bad") ) runs = [] updates = [] - preflights = [] + run_validations = [] monkeypatch.setattr("itzi.itzi._create_manifest", lambda *_args, **_kwargs: None) monkeypatch.setattr( "itzi.itzi._update_manifest", @@ -254,8 +256,8 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo ), ) monkeypatch.setattr( - "itzi.itzi._preflight_in_subprocess", - lambda simulations: preflights.append(simulations[0].simulation_id) or {}, + "itzi.itzi._validate_run_requirements_in_subprocess", + lambda simulations: run_validations.append(simulations[0].simulation_id) or {}, ) monkeypatch.setattr( "itzi.itzi._run_simulation_in_subprocess", @@ -273,8 +275,8 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo ResolvedEnsemble( cast(ExpandedEnsemble, ensemble), cast(tuple[ResolvedSimulation, ...], simulations), () ), - {"good", "bad", "preflight"}, - {"preflight": {"phase": "preflight", "detail": "invalid input"}}, + {"good", "bad", "invalid"}, + {"invalid": {"phase": "run_validation", "detail": "invalid input"}}, None, has_selectors=True, overwrite=False, @@ -282,16 +284,16 @@ def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, mo assert count == 2 assert runs == ["good", "bad"] - assert preflights == ["bad"] + assert run_validations == ["bad"] assert updates[-1] == { "good": "completed", "bad": "execution_failed", - "preflight": "validation_failed", + "invalid": "validation_failed", "unselected": "not_selected", } -def test_preflight_worker_keeps_member_failures(monkeypatch): +def test_run_validation_worker_keeps_member_failures(monkeypatch): calls = [] class FakeGrassSessionManager: @@ -304,21 +306,21 @@ def __enter__(self): def __exit__(self, *_): calls.append("close") - def preflight(simulation): + def validate_run_requirements(simulation): calls.append(simulation.simulation_id) if simulation.simulation_id == "bad": raise ValueError("invalid input") monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) - fake_preflight = ModuleType("itzi.ensemble.preflight") - setattr(fake_preflight, "preflight_simulation", preflight) - monkeypatch.setitem(sys.modules, "itzi.ensemble.preflight", fake_preflight) + fake_run_validation = ModuleType("itzi.ensemble.run_validation") + setattr(fake_run_validation, "validate_run_requirements", validate_run_requirements) + monkeypatch.setitem(sys.modules, "itzi.ensemble.run_validation", fake_run_validation) simulations = tuple( SimpleNamespace(simulation_id=simulation_id, grass_params="shared") for simulation_id in ("good", "bad", "another") ) - assert preflight_worker(cast(tuple[ResolvedSimulation, ...], simulations)) == { + assert run_validation_worker(cast(tuple[ResolvedSimulation, ...], simulations)) == { "bad": "ValueError: invalid input" } assert calls == ["open", "good", "bad", "another", "close"] diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index d6977eec..88ba2cd0 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -18,7 +18,7 @@ from itzi.cli_parser import build_parser from itzi.ensemble.models import ResolvedSimulation, ValidationFailure from itzi.grass.session import GrassSessionManager -from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, preflight_worker +from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, run_validation_worker from itzi.run_plan import load_batch @@ -91,7 +91,7 @@ def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( "rainfall_rate": f"stage_series@{mapset}", } assert dict(resolved.input_kinds)["rainfall_rate"] == "strds" - assert preflight_worker((resolved,)).get(resolved.simulation_id) is None + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None path = write_study(tmp_path, input={"rainfall_rate": f"stage_series@{mapset}"}) qualified = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] @@ -107,7 +107,7 @@ def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> None: +def test_temporal_run_validation_failure_does_not_stop_next_member(tmp_path: Path) -> None: from grass.pygrass import utils as gutils GrassSessionManager.ensure_temporal_initialized() @@ -143,7 +143,7 @@ def test_temporal_preflight_failure_does_not_stop_next_member(tmp_path: Path) -> assert sorted(member["status"] for member in members) == ["completed", "validation_failed"] failed = next(member for member in members if member["status"] == "validation_failed") completed = next(member for member in members if member["status"] == "completed") - assert failed["failure"]["phase"] == "preflight" + assert failed["failure"]["phase"] == "run_validation" assert ( "ends before simulation" in failed["failure"]["detail"] or "inadequate temporal" in failed["failure"]["detail"] @@ -195,9 +195,9 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path finally: interface.cleanup() - assert preflight_worker((resolved,)).get(resolved.simulation_id) is None + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None gscript.run_command("r.mask", flags="r") - assert preflight_worker((resolved,)).get(resolved.simulation_id) is None + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None active_path = write_study(tmp_path, domain={}) active_resolved = _resolve_ensemble_in_subprocess( @@ -207,7 +207,7 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path assert active_resolved.effective_mask.mode == "none" gscript.run_command("g.copy", raster="stage_mask_values,MASK") assert "effective GRASS mask changed" in ( - preflight_worker((active_resolved,)).get(active_resolved.simulation_id) or "" + run_validation_worker((active_resolved,)).get(active_resolved.simulation_id) or "" ) assert gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET")) finally: @@ -224,12 +224,12 @@ def test_worker_rejects_region_and_input_changes_after_resolution(tmp_path: Path gscript.run_command("g.region", n=60) assert "GRASS domain changed" in ( - preflight_worker((resolved,)).get(resolved.simulation_id) or "" + run_validation_worker((resolved,)).get(resolved.simulation_id) or "" ) gscript.run_command("g.region", n=50) gscript.run_command("g.remove", flags="f", type="raster", name="stage_worker_friction") assert "stage_worker_friction" in ( - preflight_worker((resolved,)).get(resolved.simulation_id) or "" + run_validation_worker((resolved,)).get(resolved.simulation_id) or "" ) assert not (tmp_path / "results").exists() @@ -399,7 +399,7 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_dry_run_preflights_multiple_members_in_one_session(tmp_path): +def test_dry_run_validates_multiple_members_in_one_session(tmp_path): prefix = f"dry_multi_{uuid4().hex[:8]}" config_path = write_study( tmp_path, @@ -443,7 +443,7 @@ def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): (tmp_path / "results" / "dry.manifest.yaml").read_text(encoding="utf-8") ) assert manifest["members"][0]["status"] == "validation_failed" - assert manifest["members"][0]["failure"]["phase"] == "preflight" + assert manifest["members"][0]["failure"]["phase"] == "run_validation" assert not (tmp_path / "dry-results").exists() assert not gscript.find_file(name="dry_output_water_depth_0000", element="cell").get("file") @@ -506,9 +506,9 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_preflight_accepts_unoccupied_drainage_id(): +def test_run_validation_accepts_unoccupied_drainage_id(): from itzi.grass.utils import get_current_mapset - from itzi.ensemble.preflight import _validate_grass_outputs + from itzi.ensemble.run_validation import _validate_grass_outputs from itzi.grass.interface import GrassInterface GrassSessionManager.ensure_temporal_initialized() From 206f49d77164334898c90d949730c9f7b17f050b Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 11:42:17 -0600 Subject: [PATCH 71/85] consolidate ensemble tests --- tests/grass/test_ensemble.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 88ba2cd0..84249bbf 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -155,7 +155,7 @@ def test_temporal_run_validation_failure_does_not_stop_next_member(tmp_path: Pat @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path) -> None: +def test_worker_rejects_changed_mask_region_and_inputs_after_resolution(tmp_path: Path) -> None: from grass.pygrass import utils as gutils from itzi.grass.interface import GrassInterface @@ -213,10 +213,6 @@ def test_mask_semantics_and_worker_detects_changed_effective_mask(tmp_path: Path finally: gscript.run_command("r.mask", flags="r") - -@pytest.mark.forked -@pytest.mark.usefixtures("grass_5by5") -def test_worker_rejects_region_and_input_changes_after_resolution(tmp_path: Path) -> None: gscript.run_command("g.copy", raster="n,stage_worker_friction") path = write_study(tmp_path, input={"friction": "stage_worker_friction"}) resolved = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] @@ -342,7 +338,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): @pytest.mark.forked @pytest.mark.usefixtures("grass_5by5") -def test_explicit_grass_context_runs_four_members_without_parent_session(test_data_temp_path): +def test_explicit_grass_context_runs_two_members_without_parent_session(test_data_temp_path): context = gscript.gisenv() config_path = Path(test_data_temp_path) / "explicit.yaml" config_path.write_text( @@ -364,7 +360,7 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da friction: "n" water_depth: "start_h" options: - cfl: [0.2, 0.3] + cfl: 0.2 theta: [0.8, 0.9] dtmax: 0.3 outputs: @@ -386,7 +382,7 @@ def test_explicit_grass_context_runs_four_members_without_parent_session(test_da manifest_path = Path(test_data_temp_path) / "explicit-manifest.yaml" manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) - assert len(manifest["members"]) == 4 + assert len(manifest["members"]) == 2 assert all(member["status"] == "completed" for member in manifest["members"]) mapset_path = ( Path(context["GISDBASE"]) / context["LOCATION_NAME"] / context["MAPSET"] / "cellhd" @@ -505,7 +501,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): @pytest.mark.forked -@pytest.mark.usefixtures("grass_5by5") +@pytest.mark.usefixtures("grass_xy_session") def test_run_validation_accepts_unoccupied_drainage_id(): from itzi.grass.utils import get_current_mapset from itzi.ensemble.run_validation import _validate_grass_outputs From c7d0126325be15456c96171559b6ed754df0cd15 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 12:52:16 -0600 Subject: [PATCH 72/85] assert that green ampt keys are part of input keys --- src/itzi/ensemble/models.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index 192cec4a..732f0283 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -33,12 +33,16 @@ from itzi.grass.session import GrassParams -GREEN_AMPT_KEYS = ( - "effective_porosity", - "capillary_pressure", - "hydraulic_conductivity", - "soil_water_content", +GREEN_AMPT_KEYS = frozenset( + { + "effective_porosity", + "capillary_pressure", + "hydraulic_conductivity", + "soil_water_content", + } ) +assert GREEN_AMPT_KEYS <= INPUT_ARRAY_KEYS + DIRECT_INPUT_KEYS = tuple( key for key in INPUT_ARRAY_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} ) From 21f63522c9e71cccc4f70bee925d69493f5188f2 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 13:18:27 -0600 Subject: [PATCH 73/85] rename options to parameters --- docs/conf_file.rst | 4 +- docs/yaml-ensemble-report.html | 195 ++++++++++++++++++++++++++++++ src/itzi/ensemble/models.py | 2 +- src/itzi/ensemble/resolution.py | 10 +- src/itzi/ensemble/schema.py | 4 +- src/itzi/ensemble/yaml.py | 22 ++-- tests/cli/test_ensemble_config.py | 26 ++-- tests/grass/test_ensemble.py | 20 +-- 8 files changed, 240 insertions(+), 43 deletions(-) create mode 100644 docs/yaml-ensemble-report.html diff --git a/docs/conf_file.rst b/docs/conf_file.rst index b4852769..483aa1cd 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -45,7 +45,7 @@ Basic YAML example rainfall_rate: ["rain_10yr@PERMANENT", "rain_100yr@PERMANENT"] infiltration: type: "none" - options: + parameters: cfl: [0.5, 0.7] outputs: rasters: @@ -56,7 +56,7 @@ Basic YAML example The example expands to four simulations: two rainfall maps multiplied by two ``cfl`` values. Scalars are fixed across an ensemble, while non-empty lists in -``input``, ``options``, or ``drainage`` are sweep dimensions. ``domain`` and +``input``, ``parameters``, or ``drainage`` are sweep dimensions. ``domain`` and ``time`` are always scalar. Output variable lists are selections, not sweeps. An ensemble may contain at most 100 candidate simulations; a full command may contain at most 200. diff --git a/docs/yaml-ensemble-report.html b/docs/yaml-ensemble-report.html new file mode 100644 index 00000000..831de0eb --- /dev/null +++ b/docs/yaml-ensemble-report.html @@ -0,0 +1,195 @@ + + + + + + YAML ensembles | Itzï branch report + + + +
+
Itzï · branch comparison: ensemble vs main
+

YAML configuration & ensemble runs

+

One YAML document now describes a study rather than just one run. Lists of inputs or settings turn it into several simulations, which Itzï checks and runs one at a time.

+
+
+
+

What changed

+
+

Before

Each INI file described one simulation. A batch ran several INI files, with the legacy hotstart and resume workflow.

+

On this branch

.yaml files contain one or more documents separated by ---. Each document defines an ensemble: a shared domain and clock, plus one or more combinations of input maps and numerical parameters. A command can process several YAML files and legacy INI files in order.

+
+

The .yaml suffix selects YAML parsing. Other suffixes use the deprecated INI reader; invalid YAML is never silently treated as INI.

+
+ +
+

A four-member study

+

This example assumes the GRASS session can see the named rasters. Save it as study.yaml:

+ 
schema_version: 1
+ensemble:
+  id: "study"
+domain: {}
+time:
+  duration: "02:00:00"
+  record_step: "00:05:00"
+input:
+  ground_elevation: "terrain"
+  friction: "manning"
+  rainfall_rate: ["rain_10yr", "rain_100yr"]
+parameters:
+  cfl: [0.5, 0.7]
+outputs:
+  rasters:
+    prefix: "study_{simulation}"
+    variables: [water_depth, flow_speed]
+  statistics:
+    file: "results/{simulation}.csv"
+

Two rainfall choices × two cfl choices = four simulations. Each has the same terrain, friction, region, duration and recording interval. variables chooses what to save; it does not multiply the number of simulations.

+
+ + + + + + + + + +
YAML fieldPlain-English meaning
schema_version, ensembleFormat version 1 and a unique study ID. An optional name is descriptive.
domainOne GRASS context for the ensemble. Leave grass out to use the active session; optionally specify grass.database, project, mapset together, and executable. An optional region and mask apply to all members.
timeOne clock for every member: duration alone, start + duration, or start + end, always with record_step. Durations are positive HH:MM:SS, including hours over 23.
inputTerrain (ground_elevation) and friction are required. Other raster/space-time raster inputs are optional. A scalar fixes a value; a nonempty list sweeps it.
parameters, drainageNumerical settings use core defaults unless supplied; drainage is optional and requires the SWMM input and three exchange coefficients. Their lists can also be swept.
outputsOptional raster, CSV statistics, drainage-vector and manifest destinations. The section itself is required, even if empty (outputs: {}).
+

input.infiltration is either {type: "none"} (the default), {type: "constant", rate: "map"}, or type: "green-ampt" with maps for effective porosity, capillary pressure and hydraulic conductivity (optionally soil water content). A list of complete infiltration alternatives counts as one sweep dimension.

+
+ +
+

From choices to runnable members

+
    +
  1. Read and expand. YAML is safely parsed, checked against a strict versioned schema, and every combination of list values is produced. Unknown fields, duplicate or merge keys, custom tags, empty documents and duplicate sweep values are rejected. An explicitly separated later document can still load after an earlier document fails.
    2. +
  2. Resolve against GRASS. A fresh process opens the requested or active GRASS context, finds the region and mask, resolves each input as a raster or space-time raster dataset, and builds a simulation configuration. Ambiguous or missing inputs fail for that member. Optional SWMM files are resolved relative to the YAML file.
    3. +
  3. Name and check outputs. Each successfully resolved member gets a sim-xxxxxxxx ID from a digest of its normalized domain, time, inputs, settings, mask and drainage information. This ID is based on the resolved run, not its position in the sweep. Itzï renders output templates, then checks for duplicate IDs, collisions between members, and output names that would overwrite inputs or the mask.
    4. +
  4. Validate run requirements and run. Selected members are checked for usable input data, temporal coverage, existing destinations and a still-matching GRASS environment. The real run writes a manifest, then runs eligible members sequentially in separate processes. After the first member, each next member's run requirements are validated again because earlier output may have changed what exists.
    5. +
+

The limits apply before member selection: at most 100 candidate simulations per document and 200 across the batch.

+
+ +
+

Execution flow

+

The CLI branches into the purple legacy INI path or the YAML path. Teal YAML cards run in spawned GRASS processes; YAML then branches into dry-run and real-run outcomes. Labels use Python module and function or class method names.

+
+
itzi.itzi.main() → itzi.itzi.itzi_run()Parse “itzi run” and choose a route
+
↓ choose a route ↓
+
+
+

Legacy INI-only

+
+
itzi.itzi._uses_legacy_execution_path()Only INI files; no --member or --dry-run
+
itzi.itzi._run_legacy_batch()Warn that INI is deprecated; process files in order
+
itzi.itzi.reconcile_hotstart_commands()Map optional --resume-from checkpoints, or start fresh
+
itzi.itzi.itzi_run_one()Launch one subprocess for each INI file
+
itzi.itzi.sim_runner_worker()ConfigReader.__init__() reads this INI file
+
GrassSessionManager.open()Enter the requested or active GRASS session
+
SimulationRunner.initialize()Set up providers; optionally load a hotstart
+
SimulationRunner.run() → SimulationRunner.step()Advance the model until the end time
+
SimulationRunner.finalize()Clean up; the worker exits with a status
+
itzi.itzi._run_legacy_batch()Repeat for the next INI; report times and failures
+
+
+
+

YAML ensemble

+
+
itzi.itzi._run_ensemble_batch()Coordinate ordered documents and collect errors
+
itzi.run_plan.load_batch() → itzi.ensemble.yaml.load_yaml_stream()Read each .yaml stream (or adapt an INI file)
+
itzi.ensemble.schema.YamlEnsembleDocumentV1.model_validate()itzi.ensemble.yaml.expand_yaml_document() builds the Cartesian members
+
itzi.itzi._resolve_ensemble_in_subprocess()itzi.itzi.resolver_worker() · one spawned GRASS process per ensemble
+
itzi.ensemble.resolution.resolve_ensemble()itzi.ensemble.resolution._build_resolved_simulation() creates member IDs
+
itzi.ensemble.resolution.validate_resolved_ensemble()Reject artifact collisions; itzi.itzi._select_members() applies --member
+
itzi.itzi._validate_ensemble_run_requirements()itzi.itzi._validate_run_requirements_in_subprocess() · one process per ensemble
+
itzi.itzi.run_validation_worker()itzi.ensemble.run_validation.validate_run_requirements() checks each selected member
+
+
↓ choose an outcome ↓
+
+
+

--dry-run

+
itzi.itzi._display_dry_plan()Print IDs, destinations and errors; write nothing
+
+
+

Real run

+
+
itzi.itzi._run_one_ensemble()itzi.manifest._create_manifest() records all members
+
itzi.itzi._run_one_ensemble()Later members: itzi.itzi._validate_run_requirements_in_subprocess(); itzi.manifest._update_manifest() marks members running
+
itzi.itzi._run_simulation_in_subprocess() → itzi.itzi.resolved_sim_runner_worker()New GRASS process for this member
+
SimulationRunner.initialize() → SimulationRunner.run() → SimulationRunner.step() → SimulationRunner.finalize()Build providers, advance the model, clean up
+
itzi.manifest._update_manifest()Record completion or execution failure
+
+
+
+
+
+
+

The INI path runs each file in a separate subprocess and supports mapped --resume-from checkpoints. The YAML real-run loop repeats the per-member run-requirements validation and execution boxes for each selected, eligible member. Each worker opens a GRASS session with GrassSessionManager.open(); separate processes prevent GRASS temporal state leaking across mapsets.

+
+ +
+

Using and reading the results

+ 
itzi run study.yaml --dry-run
+itzi run study.yaml --member 'study#sim-xxxxxxxx'
+itzi run study.yaml
+

First inspect the dry-run output for the actual sim-… ID; replace xxxxxxxx above with those eight hex digits. Repeat --member to choose several. If the command contains exactly one ensemble, the simulation ID may be unqualified. -o permits overwriting existing destinations.

+
    +
  • Raster variables become GRASS space-time raster outputs named from the prefix plus _variable; each record has its own map. Statistics go to CSV, and optional SWMM drainage results can go to a space-time vector dataset.
    • +
  • {ensemble} and {simulation} are the only template placeholders. The manifest path may use {ensemble}, but not {simulation}. Filesystem paths in YAML are relative to its file; output GRASS map names must be unqualified names in the execution mapset.
    • +
  • A real run writes results/study.manifest.yaml beside this example file by default. outputs.manifest.file overrides that path. The YAML manifest records source digests, member choices, destinations, statuses (planned, running, completed, validation_failed, execution_failed, not_selected) and failures/elapsed time when applicable.
    • +
  • Dry runs perform validation without creating manifests or model outputs. Document and member failures are reported; valid members in other documents can still be processed. The command ends with an error status when relevant validation or execution fails.
    • +
+

Compatibility: A plain INI-only run keeps its old execution and hotstart-resume path. INI files in the new batch planner (for example, mixed with YAML or using --dry-run) are adapted as single-member ensembles. YAML resume and checkpoint creation are not yet supported; --resume-from cannot be combined with the new ensemble path.

+
+
+ + + diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index 732f0283..abb5c52c 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -149,7 +149,7 @@ class ExpandedSimulation: time: NormalizedTime input_maps: tuple[tuple[str, str], ...] infiltration: NormalizedInfiltration - options: tuple[tuple[str, float], ...] + parameters: tuple[tuple[str, float], ...] drainage: tuple[tuple[str, str | float], ...] | None outputs: OutputTemplates diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py index cc04d255..12022eca 100644 --- a/src/itzi/ensemble/resolution.py +++ b/src/itzi/ensemble/resolution.py @@ -211,7 +211,7 @@ def _simulation_identity( "time": time_payload, "inputs": input_names, "infiltration": {"model": str(expanded.infiltration.model)}, - "options": dict(expanded.options), + "parameters": dict(expanded.parameters), "drainage": drainage, } canonical = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) @@ -334,8 +334,8 @@ def _build_simulation_config( start = datetime.min # noqa: DTZ901 # Core's established relative-time adapter. end = start + expanded.time.duration assert end is not None - options = dict(expanded.options) - surface_options = {key: value for key, value in options.items() if key != "dtinf"} + parameters = dict(expanded.parameters) + surface_parameters = {key: value for key, value in parameters.items() if key != "dtinf"} # Create the dict first to satisfy the type checker config_data = { "start_time": start, @@ -345,8 +345,8 @@ def _build_simulation_config( "hotstart_config": None, "input_map_names": input_names, "output_map_names": dict(artifacts.output_map_names), - "surface_flow_parameters": SurfaceFlowParameters(**surface_options), - "dtinf": options["dtinf"], + "surface_flow_parameters": SurfaceFlowParameters(**surface_parameters), + "dtinf": parameters["dtinf"], "infiltration_model": expanded.infiltration.model, "swmm_inp": swmm_path, "drainage_output": artifacts.drainage_output, diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index d1313515..153c6e42 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -113,7 +113,7 @@ def validate_initial_conditions(self) -> InputSweepConfig: return self -class OptionSweepConfig(StrictModel): +class ParameterSweepConfig(StrictModel): hmin: SweepFloat | None = None cfl: SweepFloat | None = None theta: SweepFloat | None = None @@ -172,7 +172,7 @@ class YamlEnsembleDocumentV1(StrictModel): domain: DomainConfig time: TimeConfig input: InputSweepConfig - options: OptionSweepConfig + parameters: ParameterSweepConfig drainage: DrainageSweepConfig | None = None outputs: OutputConfig diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py index d7436928..fed07268 100644 --- a/src/itzi/ensemble/yaml.py +++ b/src/itzi/ensemble/yaml.py @@ -54,7 +54,7 @@ InfiltrationAlternative, InputSweepConfig, NoInfiltration, - OptionSweepConfig, + ParameterSweepConfig, SweepFloat, SweepString, TimeConfig, @@ -353,7 +353,7 @@ def expand_yaml_document( (NoInfiltration, ConstantInfiltration, GreenAmptInfiltration), ) infiltration = _normalize_infiltration(infiltration_value) - options = _selected_options(document.options, selected) + parameters = _selected_parameters(document.parameters, selected) drainage = _selected_drainage(document.drainage, selected) outputs = _output_templates(document) simulations.append( @@ -365,7 +365,7 @@ def expand_yaml_document( time=normalized_time, input_maps=tuple(sorted(input_maps.items())), infiltration=infiltration, - options=tuple(sorted(options.items())), + parameters=tuple(sorted(parameters.items())), drainage=tuple(sorted(drainage.items())) if drainage is not None else None, outputs=outputs, ) @@ -397,10 +397,10 @@ def _collect_dimensions( cast(tuple[SweepScalar, ...], tuple(document.input.infiltration)), ) ) - for key in type(document.options).model_fields: - value = cast(SweepFloat | None, getattr(document.options, key)) + for key in type(document.parameters).model_fields: + value = cast(SweepFloat | None, getattr(document.parameters, key)) if isinstance(value, list): - dimensions.append((f"options.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) + dimensions.append((f"parameters.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) if document.drainage is not None: for key in type(document.drainage).model_fields: value = cast(SweepString | SweepFloat, getattr(document.drainage, key)) @@ -439,13 +439,15 @@ def _normalize_infiltration(value: InfiltrationAlternative) -> NormalizedInfiltr return NormalizedInfiltration(InfiltrationModelType.GREEN_AMPT, tuple(sorted(maps.items()))) -def _selected_options( - options: OptionSweepConfig, selected: dict[str, SweepScalar] +def _selected_parameters( + parameters: ParameterSweepConfig, selected: dict[str, SweepScalar] ) -> dict[str, float]: values: dict[str, float] = SurfaceFlowParameters().model_dump() values["dtinf"] = DefaultValues.DTINF - for key in type(options).model_fields: - value = selected.get(f"options.{key}", cast(SweepFloat | None, getattr(options, key))) + for key in type(parameters).model_fields: + value = selected.get( + f"parameters.{key}", cast(SweepFloat | None, getattr(parameters, key)) + ) if value is not None: assert isinstance(value, float) values[key] = value diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index 12756476..c3fa7249 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -27,7 +27,7 @@ EnsembleMetadata, InputSweepConfig, ManifestOutputs, - OptionSweepConfig, + ParameterSweepConfig, StatisticsOutputs, TimeConfig, ) @@ -53,7 +53,7 @@ def document(*, ensemble_id: str = "study", extra: str = "") -> str: input: ground_elevation: "elevation" friction: "manning" -options: {} +parameters: {} outputs: rasters: prefix: "results_{ensemble}_{simulation}" @@ -82,7 +82,7 @@ def test_scalar_document_expands_to_one_member(tmp_path): assert simulation.time.start is None assert simulation.time.end is None assert simulation.time.duration == timedelta(hours=2) - assert dict(simulation.options) == SurfaceFlowParameters().model_dump() | { + assert dict(simulation.parameters) == SurfaceFlowParameters().model_dump() | { "dtinf": DefaultValues.DTINF } @@ -138,8 +138,8 @@ def test_time_config_accepts_only_supported_combinations(): (EnsembleMetadata, {"id": "invalid!"}), (InputSweepConfig, {"ground_elevation": [], "friction": "manning"}), (InputSweepConfig, {"ground_elevation": ["elevation"], "friction": [["manning"]]}), - (OptionSweepConfig, {"cfl": [float("nan")]}), - (OptionSweepConfig, {"cfl": [0.5, 0.5]}), + (ParameterSweepConfig, {"cfl": [float("nan")]}), + (ParameterSweepConfig, {"cfl": [0.5, 0.5]}), (StatisticsOutputs, {"file": ""}), (ManifestOutputs, {"file": ""}), ), @@ -161,7 +161,7 @@ def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): ground_elevation: ["elevation_a", "elevation_b"] friction: "manning" rainfall_rate: ["rain_a", "rain_b"] -options: +parameters: cfl: [0.5, 0.7] outputs: rasters: @@ -178,10 +178,10 @@ def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): assert tuple(path for path, _ in simulations[0].coordinates) == ( "input.ground_elevation", "input.rainfall_rate", - "options.cfl", + "parameters.cfl", ) - assert dict(simulations[0].coordinates)["options.cfl"] == 0.5 - assert dict(simulations[-1].coordinates)["options.cfl"] == 0.7 + assert dict(simulations[0].coordinates)["parameters.cfl"] == 0.5 + assert dict(simulations[-1].coordinates)["parameters.cfl"] == 0.7 def test_loader_continues_after_failed_explicit_document(tmp_path): @@ -196,7 +196,7 @@ def test_loader_continues_after_failed_explicit_document(tmp_path): input: ground_elevation: elevation friction: manning -options: +parameters: cfl: "0.7" outputs: {} --- @@ -240,7 +240,7 @@ def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): input: ground_elevation: elevation friction: manning -options: +parameters: <<: *defaults outputs: {} --- @@ -275,7 +275,7 @@ def test_infiltration_alternatives_are_explicit(tmp_path): - type: none - type: constant rate: infiltration_rate -options: {} +parameters: {} outputs: {} """ ) @@ -334,7 +334,7 @@ def test_expansion_limit_is_checked_before_materialization(tmp_path): input: ground_elevation: elevation friction: manning -options: +parameters: cfl: [{values}] outputs: {{}} """ diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py index 84249bbf..a4f3154f 100644 --- a/tests/grass/test_ensemble.py +++ b/tests/grass/test_ensemble.py @@ -30,10 +30,10 @@ def write_study(tmp_path: Path, **changes: dict) -> Path: "domain": {}, "time": {"duration": "00:01:00", "record_step": "00:00:30"}, "input": {"ground_elevation": "z", "friction": "n"}, - "options": {"dtmax": 0.3}, + "parameters": {"dtmax": 0.3}, "outputs": {}, } - document["options"] = changes.pop("options", document["options"]) + document["parameters"] = changes.pop("parameters", document["parameters"]) for section, fields in changes.items(): values = document[section] assert isinstance(values, dict) @@ -295,7 +295,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): infiltration: - type: "constant" rate: "infiltration_rate" -options: +parameters: cfl: [0.2, 0.3] dtmax: 0.3 outputs: @@ -322,11 +322,11 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): assert [member["coordinates"] for member in manifest["members"]] == [ { "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, - "options.cfl": 0.2, + "parameters.cfl": 0.2, }, { "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, - "options.cfl": 0.3, + "parameters.cfl": 0.3, }, ] assert all( @@ -359,7 +359,7 @@ def test_explicit_grass_context_runs_two_members_without_parent_session(test_dat ground_elevation: "z" friction: "n" water_depth: "start_h" -options: +parameters: cfl: 0.2 theta: [0.8, 0.9] dtmax: 0.3 @@ -400,7 +400,7 @@ def test_dry_run_validates_multiple_members_in_one_session(tmp_path): config_path = write_study( tmp_path, ensemble={"id": "dry-multi"}, - options={"cfl": [0.2, 0.3]}, + parameters={"cfl": [0.2, 0.3]}, outputs={"rasters": {"prefix": f"{prefix}_{{simulation}}", "variables": ["water_depth"]}}, ) @@ -418,7 +418,7 @@ def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): tmp_path, ensemble={"id": "dry"}, input={"ground_elevation": "dry_null_dem"}, - options={}, + parameters={}, outputs={ "rasters": {"prefix": "dry_output", "variables": ["water_depth"]}, "statistics": {"file": "dry-results/statistics.csv"}, @@ -472,7 +472,7 @@ def test_dry_run_does_not_read_rainfall_cells(tmp_path): tmp_path, ensemble={"id": "dry-rain"}, input={"rainfall_rate": "dry_rain_series"}, - options={}, + parameters={}, outputs={"rasters": {"prefix": "dry_rain_output", "variables": ["water_depth"]}}, ) @@ -487,7 +487,7 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path): config_path = write_study( Path(test_data_temp_path), ensemble={"id": "existing-output"}, - options={}, + parameters={}, outputs={"rasters": {"prefix": "existing_output", "variables": ["water_depth"]}}, ) From 05ecced4d9c1a3ba4c992422385ed2c862463519 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 13:32:16 -0600 Subject: [PATCH 74/85] print an error message instead of raw traceback --- src/itzi/itzi.py | 5 ++++- tests/cli/test_cli.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index f729c748..bb48e371 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -325,7 +325,10 @@ def _run_ensemble_batch(cli_args: Namespace) -> None: """Resolve and execute each ensemble in order through spawn boundaries.""" if getattr(cli_args, "resume_from", []): msgr.fatal("Resume is not available for YAML ensembles.") - ensembles, document_failures = load_batch(cli_args.config_file) + try: + ensembles, document_failures = load_batch(cli_args.config_file) + except EnsembleError as error: + msgr.fatal(str(error)) for failure in document_failures: msgr.warning(failure.format()) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index d6e24b00..a235d778 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -349,6 +349,38 @@ def fail(_): assert "Traceback" not in stderr +@pytest.mark.parametrize("same_file", [True, False]) +def test_main_reports_duplicate_ensemble_ids(tmp_path, itzi_stderr, same_file): + document = """\ +schema_version: 1 +ensemble: + id: repeated +domain: {} +time: + duration: "01:00:00" + record_step: "00:05:00" +input: + ground_elevation: elevation + friction: manning +parameters: {} +outputs: {} +""" + first = tmp_path / "first.yaml" + second = tmp_path / "second.yaml" + if same_file: + first.write_text(f"{document}---\n{document}", encoding="utf-8") + paths = [str(first)] + else: + first.write_text(document, encoding="utf-8") + second.write_text(document, encoding="utf-8") + paths = [str(first), str(second)] + + assert main(["run", *paths]) == 1 + stderr = itzi_stderr.getvalue() + assert stderr.count("ERROR: duplicate ensemble ID 'repeated'") == 1 + assert "Traceback" not in stderr + + def test_main_propagates_unexpected_error(monkeypatch): def fail(_): raise ValueError("unexpected") From 9ecc5df77796099ff7190de40a60f78d22765593 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 13:46:56 -0600 Subject: [PATCH 75/85] rename ensemble.name to ensemble.description. Limit ensemble.id to 80 chars --- docs/conf_file.rst | 3 +- src/itzi/ensemble/models.py | 2 +- src/itzi/ensemble/schema.py | 4 +-- src/itzi/ensemble/yaml.py | 2 +- src/itzi/manifest.py | 2 +- tests/cli/test_ensemble_config.py | 59 ++++++++----------------------- 6 files changed, 21 insertions(+), 51 deletions(-) diff --git a/docs/conf_file.rst b/docs/conf_file.rst index 483aa1cd..5a09a61a 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -26,7 +26,7 @@ Basic YAML example schema_version: 1 ensemble: id: "central-city" - name: "Central city rainfall study" + description: "Central city rainfall study" domain: grass: database: "/srv/grassdata" @@ -58,6 +58,7 @@ The example expands to four simulations: two rainfall maps multiplied by two ``cfl`` values. Scalars are fixed across an ensemble, while non-empty lists in ``input``, ``parameters``, or ``drainage`` are sweep dimensions. ``domain`` and ``time`` are always scalar. Output variable lists are selections, not sweeps. +The optional ``ensemble.description`` is limited to 256 characters. An ensemble may contain at most 100 candidate simulations; a full command may contain at most 200. diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py index abb5c52c..09bb3d66 100644 --- a/src/itzi/ensemble/models.py +++ b/src/itzi/ensemble/models.py @@ -160,7 +160,7 @@ class ExpandedEnsemble: source: SourceDocument ensemble_id: str - ensemble_name: str | None + ensemble_description: str | None manifest_template: str | None simulations: tuple[ExpandedSimulation, ...] diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py index 153c6e42..ed2ad9b5 100644 --- a/src/itzi/ensemble/schema.py +++ b/src/itzi/ensemble/schema.py @@ -46,8 +46,8 @@ def _validate_sweep[T](values: list[T]) -> list[T]: class EnsembleMetadata(StrictModel): - id: StrictStr = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$") - name: StrictStr | None = None + id: StrictStr = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$", max_length=80) + description: StrictStr | None = Field(default=None, max_length=256) class TimeConfig(StrictModel): diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py index fed07268..66110efe 100644 --- a/src/itzi/ensemble/yaml.py +++ b/src/itzi/ensemble/yaml.py @@ -376,7 +376,7 @@ def expand_yaml_document( return ExpandedEnsemble( source=source, ensemble_id=document.ensemble.id, - ensemble_name=document.ensemble.name, + ensemble_description=document.ensemble.description, manifest_template=manifest_template, simulations=tuple(simulations), ) diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py index f2fd1308..253c5b31 100644 --- a/src/itzi/manifest.py +++ b/src/itzi/manifest.py @@ -127,7 +127,7 @@ def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> d return { "manifest_version": 1, "last_updated_at": datetime.now().astimezone().isoformat(), - "ensemble": {"id": ensemble.ensemble_id, "name": ensemble.ensemble_name}, + "ensemble": {"id": ensemble.ensemble_id, "description": ensemble.ensemble_description}, "source": { "path": str(ensemble.source.path), "document_index": ensemble.source.document_index, diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py index c3fa7249..27eec93d 100644 --- a/tests/cli/test_ensemble_config.py +++ b/tests/cli/test_ensemble_config.py @@ -23,14 +23,7 @@ parse_duration, render_template, ) -from itzi.ensemble.schema import ( - EnsembleMetadata, - InputSweepConfig, - ManifestOutputs, - ParameterSweepConfig, - StatisticsOutputs, - TimeConfig, -) +from itzi.ensemble.schema import TimeConfig from itzi.ensemble.resolution import ( _build_simulation_config, _last_record_index, @@ -38,6 +31,7 @@ _resolve_inputs, validate_resolved_ensemble, ) +from itzi.manifest import _manifest_document def write_yaml(tmp_path, content: str): @@ -87,6 +81,18 @@ def test_scalar_document_expands_to_one_member(tmp_path): } +def test_ensemble_description_reaches_manifest(tmp_path): + description = "Central city rainfall study" + content = document().replace(" id: study", f" id: study\n description: {description}") + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + assert _manifest_document(stream.ensembles[0], {})["ensemble"] == { + "id": "study", + "description": description, + } + + def test_drainage_coefficients_are_forwarded(tmp_path): expanded = load_yaml_stream(write_yaml(tmp_path, document())).ensembles[0].simulations[0] expanded = replace( @@ -132,23 +138,6 @@ def test_time_config_accepts_only_supported_combinations(): TimeConfig.model_validate({"record_step": "00:05:00", **fields}) -@pytest.mark.parametrize( - ("model", "value"), - ( - (EnsembleMetadata, {"id": "invalid!"}), - (InputSweepConfig, {"ground_elevation": [], "friction": "manning"}), - (InputSweepConfig, {"ground_elevation": ["elevation"], "friction": [["manning"]]}), - (ParameterSweepConfig, {"cfl": [float("nan")]}), - (ParameterSweepConfig, {"cfl": [0.5, 0.5]}), - (StatisticsOutputs, {"file": ""}), - (ManifestOutputs, {"file": ""}), - ), -) -def test_schema_constraints_reject_invalid_values(model, value): - with pytest.raises(ValidationError): - model.model_validate(value) - - def test_grass_context_requires_complete_context(): with pytest.raises(ValidationError, match="mapset must be supplied together"): GrassContextConfig(database="/grassdata", project="project") @@ -288,30 +277,10 @@ def test_infiltration_alternatives_are_explicit(tmp_path): InfiltrationModelType.CONSTANT, ] - labelled = content.replace(" - type: none", ' - type: none\n label: "first"') - labelled_stream = load_yaml_stream(write_yaml(tmp_path, labelled)) - assert labelled_stream.failures[0].phase == "schema" - duplicate = content.replace(" - type: none\n", " - type: none\n - type: none\n") assert load_yaml_stream(write_yaml(tmp_path, duplicate)).failures[0].phase == "schema" -def test_yaml_rejects_unsupported_hotstart(tmp_path): - content = ( - document() - + """\ -hotstart: - wallclock_step: "00:01:00" - file: "checkpoint.zip" -""" - ) - - stream = load_yaml_stream(write_yaml(tmp_path, content)) - - assert stream.ensembles == () - assert stream.failures[0].phase == "schema" - - def test_absolute_offsets_keep_wall_clock_values(tmp_path): content = document().replace( ' duration: "02:00:00"\n record_step: "00:05:00"', From 67de8b640c7626a93be9507d54771c74d40ff38a Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 13:56:36 -0600 Subject: [PATCH 76/85] always display elapsed_seconds after status in the manifest. Remove the source digests --- src/itzi/manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py index 253c5b31..70d9c8cf 100644 --- a/src/itzi/manifest.py +++ b/src/itzi/manifest.py @@ -91,6 +91,7 @@ def _initial_member_states( "simulation_id": simulation.simulation_id, "selected": selected, "status": "planned" if selected else "not_selected", + "elapsed_seconds": None, "coordinates": _manifest_coordinates(simulation.coordinates), "artifacts": { "rasters": dict(simulation.artifacts.output_map_names), @@ -106,6 +107,7 @@ def _initial_member_states( "simulation_id": None, "selected": not has_selectors, "status": "not_selected" if has_selectors else "validation_failed", + "elapsed_seconds": None, "coordinates": _manifest_coordinates(failure.coordinates), "failure": {"phase": failure.phase, "detail": failure.detail}, } @@ -131,8 +133,6 @@ def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> d "source": { "path": str(ensemble.source.path), "document_index": ensemble.source.document_index, - "file_digest": ensemble.source.file_digest, - "document_digest": ensemble.source.document_digest, }, "members": list(states.values()), } From 8335eb9c7ea119e47efc5b915ddb70fc68ec6230 Mon Sep 17 00:00:00 2001 From: Laurent Courty Date: Wed, 30 Sep 2026 14:28:41 -0600 Subject: [PATCH 77/85] fold DomainConfig and GrassContextConfig under GrassConfig --- docs/conf_file.rst | 19 ++++++++++++------- docs/yaml-ensemble-report.html | 4 ++-- src/itzi/ensemble/models.py | 14 +++++--------- src/itzi/ensemble/resolution.py | 22 +++++++++++----------- src/itzi/ensemble/schema.py | 4 ++-- src/itzi/ensemble/yaml.py | 2 +- src/itzi/run_plan.py | 21 ++++++++++----------- tests/cli/test_cli.py | 2 +- tests/cli/test_ensemble_config.py | 20 +++++++++++++++----- tests/grass/test_ensemble.py | 19 +++++++++---------- 10 files changed, 68 insertions(+), 59 deletions(-) diff --git a/docs/conf_file.rst b/docs/conf_file.rst index 5a09a61a..6964a571 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -27,12 +27,11 @@ Basic YAML example ensemble: id: "central-city" description: "Central city rainfall study" - domain: - grass: - database: "/srv/grassdata" - project: "central_city" - mapset: "itzi" - executable: "grass" + grass: + database: "/srv/grassdata" + project: "central_city" + mapset: "itzi" + executable: "grass" region: "central_city_5m" mask: "central_city_mask" time: @@ -56,7 +55,7 @@ Basic YAML example The example expands to four simulations: two rainfall maps multiplied by two ``cfl`` values. Scalars are fixed across an ensemble, while non-empty lists in -``input``, ``parameters``, or ``drainage`` are sweep dimensions. ``domain`` and +``input``, ``parameters``, or ``drainage`` are sweep dimensions. ``grass`` and ``time`` are always scalar. Output variable lists are selections, not sweeps. The optional ``ensemble.description`` is limited to 256 characters. An ensemble may contain at most 100 candidate simulations; a full command may @@ -70,6 +69,12 @@ Hours may exceed 23; minutes and seconds must be between 0 and 59. Timezone offsets are retained as source metadata but execute as naive GRASS wall-clock times without conversion. +Use ``grass: {}`` to select the active GRASS session and its current region and +mask. Set ``grass.region`` to a named region or ``grass.mask`` to a raster map +to override either one. To open a GRASS session explicitly, provide +``grass.database``, ``grass.project``, and ``grass.mapset`` together; +``grass.executable`` may be set with them. + The ensemble manifest is written as YAML to ``results/.manifest.yaml`` relative to the configuration file. Set ``outputs.manifest.file`` to override this path. diff --git a/docs/yaml-ensemble-report.html b/docs/yaml-ensemble-report.html index 831de0eb..ef896a52 100644 --- a/docs/yaml-ensemble-report.html +++ b/docs/yaml-ensemble-report.html @@ -76,7 +76,7 @@

A four-member study

schema_version: 1
 ensemble:
   id: "study"
-domain: {}
+grass: {}
 time:
   duration: "02:00:00"
   record_step: "00:05:00"
@@ -97,7 +97,7 @@ 
A four-member study

         YAML fieldPlain-English meaning
         
           schema_version, ensembleFormat version 1 and a unique study ID. An optional name is descriptive.
-          domainOne GRASS context for the ensemble. Leave grass out to use the active session; optionally specify grass.database, project, mapset together, and executable. An optional region and mask apply to all members.
+          grassOne GRASS configuration for the ensemble. Use grass: {} for the active session; optionally specify database, project, and mapset together, plus executable. Optional region and mask names apply to all members.
           timeOne clock for every member: duration alone, start + duration, or start + end, always with record_step. Durations are positive HH:MM:SS, including hours over 23.
           inputTerrain (ground_elevation) and friction are required. Other raster/space-time raster inputs are optional. A scalar fixes a value; a nonempty list sweeps it.
           parameters, drainageNumerical settings use core defaults unless supplied; drainage is optional and requires the SWMM input and three exchange coefficients. Their lists can also be swept.
diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py
index 09bb3d66..1b274a90 100644
--- a/src/itzi/ensemble/models.py
+++ b/src/itzi/ensemble/models.py
@@ -92,26 +92,22 @@ class StrictModel(BaseModel):
     model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
 
 
-class GrassContextConfig(StrictModel):
+class GrassConfig(StrictModel):
     database: StrictStr | None = None
     project: StrictStr | None = None
     mapset: StrictStr | None = None
     executable: StrictStr | None = None
+    region: StrictStr | None = None
+    mask: StrictStr | None = None
 
     @model_validator(mode="after")
-    def validate_context(self) -> GrassContextConfig:
+    def validate_context(self) -> GrassConfig:
         GrassParams(grassdata=self.database, location=self.project, mapset=self.mapset)
         if self.executable is not None and self.database is None:
             raise ValueError("executable requires database, project, and mapset")
         return self
 
 
-class DomainConfig(StrictModel):
-    grass: GrassContextConfig | None = None
-    region: StrictStr | None = None
-    mask: StrictStr | None = None
-
-
 @dataclass(frozen=True)
 class NormalizedTime:
     """Time semantics retained until the final core configuration boundary."""
@@ -145,7 +141,7 @@ class ExpandedSimulation:
     source: SourceDocument
     ensemble_id: str
     coordinates: tuple[tuple[str, JsonValue], ...]
-    domain: DomainConfig
+    grass: GrassConfig
     time: NormalizedTime
     input_maps: tuple[tuple[str, str], ...]
     infiltration: NormalizedInfiltration
diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py
index 12022eca..1acce064 100644
--- a/src/itzi/ensemble/resolution.py
+++ b/src/itzi/ensemble/resolution.py
@@ -50,8 +50,8 @@ def resolve_ensemble(
         from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask
 
         actual_params = active_grass_params(requested_params)
-        domain = read_domain(simulations[0].domain.region)
-        effective_mask = resolve_effective_mask(simulations[0].domain.mask)
+        domain = read_domain(simulations[0].grass.region)
+        effective_mask = resolve_effective_mask(simulations[0].grass.mask)
         input_cache: dict[str, tuple[str, Literal["raster", "strds"]]] = {}
         swmm_cache: dict[Path, str] = {}
         results: list[ResolvedSimulation | ValidationFailure] = []
@@ -86,16 +86,16 @@ def resolve_ensemble(
 
 
 def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams:
-    context = expanded.domain.grass
-    if context is None or context.database is None:
-        return GrassParams(region=expanded.domain.region, mask=expanded.domain.mask)
+    config = expanded.grass
+    if config.database is None:
+        return GrassParams(region=config.region, mask=config.mask)
     return GrassParams(
-        grassdata=str(Path(context.database).expanduser().resolve()),
-        location=context.project,
-        mapset=context.mapset,
-        region=expanded.domain.region,
-        mask=expanded.domain.mask,
-        grass_bin=context.executable or "grass",
+        grassdata=str(Path(config.database).expanduser().resolve()),
+        location=config.project,
+        mapset=config.mapset,
+        region=config.region,
+        mask=config.mask,
+        grass_bin=config.executable or "grass",
     )
 
 
diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py
index ed2ad9b5..2e945f0a 100644
--- a/src/itzi/ensemble/schema.py
+++ b/src/itzi/ensemble/schema.py
@@ -25,7 +25,7 @@
     field_validator,
     model_validator,
 )
-from itzi.ensemble.models import DomainConfig, StrictModel
+from itzi.ensemble.models import GrassConfig, StrictModel
 
 
 def _validate_sweep[T](values: list[T]) -> list[T]:
@@ -169,7 +169,7 @@ class OutputConfig(StrictModel):
 class YamlEnsembleDocumentV1(StrictModel):
     schema_version: Literal[1]
     ensemble: EnsembleMetadata
-    domain: DomainConfig
+    grass: GrassConfig
     time: TimeConfig
     input: InputSweepConfig
     parameters: ParameterSweepConfig
diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py
index 66110efe..c9bf43cb 100644
--- a/src/itzi/ensemble/yaml.py
+++ b/src/itzi/ensemble/yaml.py
@@ -361,7 +361,7 @@ def expand_yaml_document(
                 source=source,
                 ensemble_id=document.ensemble.id,
                 coordinates=coordinates,
-                domain=document.domain,
+                grass=document.grass,
                 time=normalized_time,
                 input_maps=tuple(sorted(input_maps.items())),
                 infiltration=infiltration,
diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py
index 24971bc5..35aa1614 100644
--- a/src/itzi/run_plan.py
+++ b/src/itzi/run_plan.py
@@ -26,11 +26,10 @@
     DIRECT_INPUT_KEYS,
     GREEN_AMPT_KEYS,
     DocumentFailure,
-    DomainConfig,
     EnsembleError,
     ExpandedEnsemble,
     ExpandedSimulation,
-    GrassContextConfig,
+    GrassConfig,
     NormalizedInfiltration,
     NormalizedTime,
     OutputTemplates,
@@ -88,14 +87,14 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble:
         document_digest=hashlib.blake2b(b"legacy-v1\0" + source_bytes, digest_size=32).hexdigest(),
     )
     ensemble_id = f"legacy-{hashlib.blake2b(str(source_path).encode(), digest_size=8).hexdigest()}"
-    context = None
-    if grass.grassdata is not None:
-        context = GrassContextConfig(
-            database=grass.grassdata,
-            project=grass.location,
-            mapset=grass.mapset,
-            executable=grass.grass_bin,
-        )
+    grass_config = GrassConfig(
+        database=grass.grassdata,
+        project=grass.location,
+        mapset=grass.mapset,
+        executable=grass.grass_bin if grass.grassdata is not None else None,
+        region=grass.region,
+        mask=grass.mask,
+    )
     input_names = config.input_map_names
     infiltration_maps = {
         key: input_names[key] for key in ("infiltration", *GREEN_AMPT_KEYS) if key in input_names
@@ -129,7 +128,7 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble:
         source=source,
         ensemble_id=ensemble_id,
         coordinates=(),
-        domain=DomainConfig(grass=context, region=grass.region, mask=grass.mask),
+        grass=grass_config,
         time=normalized_time,
         input_maps=tuple(sorted(direct_inputs.items())),
         infiltration=NormalizedInfiltration(
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index a235d778..4ae5f05e 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -355,7 +355,7 @@ def test_main_reports_duplicate_ensemble_ids(tmp_path, itzi_stderr, same_file):
 schema_version: 1
 ensemble:
   id: repeated
-domain: {}
+grass: {}
 time:
   duration: "01:00:00"
   record_step: "00:05:00"
diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py
index 27eec93d..a2950739 100644
--- a/tests/cli/test_ensemble_config.py
+++ b/tests/cli/test_ensemble_config.py
@@ -17,7 +17,7 @@
 from itzi.ensemble.models import (
     MAX_ENSEMBLE_MEMBERS,
     ArtifactSummary,
-    GrassContextConfig,
+    GrassConfig,
     ResolvedSimulation,
     format_duration,
     parse_duration,
@@ -28,6 +28,7 @@
     _build_simulation_config,
     _last_record_index,
     _render_artifacts,
+    _requested_grass_params,
     _resolve_inputs,
     validate_resolved_ensemble,
 )
@@ -58,7 +59,7 @@ def document(*, ensemble_id: str = "study", extra: str = "") -> str:
 schema_version: 1
 ensemble:
   id: {ensemble_id}
-domain: {{}}
+grass: {{}}
 time:
   duration: "02:00:00"
   record_step: "00:05:00"
@@ -140,7 +141,16 @@ def test_time_config_accepts_only_supported_combinations():
 
 def test_grass_context_requires_complete_context():
     with pytest.raises(ValidationError, match="mapset must be supplied together"):
-        GrassContextConfig(database="/grassdata", project="project")
+        GrassConfig(database="/grassdata", project="project")
+
+
+def test_grass_region_and_mask_work_with_active_session(tmp_path):
+    content = document().replace("grass: {}", "grass:\n  region: local_region\n  mask: mask_map")
+    stream = load_yaml_stream(write_yaml(tmp_path, content))
+
+    assert stream.failures == ()
+    params = _requested_grass_params(stream.ensembles[0].simulations[0])
+    assert (params.grassdata, params.region, params.mask) == (None, "local_region", "mask_map")
 
 
 def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path):
@@ -178,7 +188,7 @@ def test_loader_continues_after_failed_explicit_document(tmp_path):
 schema_version: 1
 ensemble:
   id: first
-domain: {}
+grass: {}
 time:
   duration: "01:00:00"
   record_step: "00:05:00"
@@ -222,7 +232,7 @@ def test_loader_rejects_empty_documents_and_merge_keys(tmp_path):
 schema_version: 1
 ensemble:
   id: invalid-merge
-domain: {}
+grass: {}
 time:
   duration: "01:00:00"
   record_step: "00:05:00"
diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py
index a4f3154f..fbce1259 100644
--- a/tests/grass/test_ensemble.py
+++ b/tests/grass/test_ensemble.py
@@ -27,7 +27,7 @@ def write_study(tmp_path: Path, **changes: dict) -> Path:
     document = {
         "schema_version": 1,
         "ensemble": {"id": "stage-one"},
-        "domain": {},
+        "grass": {},
         "time": {"duration": "00:01:00", "record_step": "00:00:30"},
         "input": {"ground_elevation": "z", "friction": "n"},
         "parameters": {"dtmax": 0.3},
@@ -163,7 +163,7 @@ def test_worker_rejects_changed_mask_region_and_inputs_after_resolution(tmp_path
     gscript.mapcalc("stage_mask_values=if(col()==1, null(), if(col()==2, 0, 1))")
     gscript.run_command("g.copy", raster="stage_mask_values,MASK")
     try:
-        path = write_study(tmp_path, domain={"mask": "stage_mask_values"})
+        path = write_study(tmp_path, grass={"mask": "stage_mask_values"})
         expanded = load_batch([str(path)])[0][0].simulations
         resolved = _resolve_ensemble_in_subprocess(expanded)[0]
         assert isinstance(resolved, ResolvedSimulation)
@@ -199,7 +199,7 @@ def test_worker_rejects_changed_mask_region_and_inputs_after_resolution(tmp_path
         gscript.run_command("r.mask", flags="r")
         assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None
 
-        active_path = write_study(tmp_path, domain={})
+        active_path = write_study(tmp_path, grass={})
         active_resolved = _resolve_ensemble_in_subprocess(
             load_batch([str(active_path)])[0][0].simulations
         )[0]
@@ -285,7 +285,7 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path):
 schema_version: 1
 ensemble:
   id: basic
-domain: {}
+grass: {}
 time:
   duration: "00:01:00"
   record_step: "00:00:30"
@@ -346,12 +346,11 @@ def test_explicit_grass_context_runs_two_members_without_parent_session(test_dat
 schema_version: 1
 ensemble:
   id: explicit
-domain:
-  grass:
-    database: "{context["GISDBASE"]}"
-    project: "{context["LOCATION_NAME"]}"
-    mapset: "{context["MAPSET"]}"
-    executable: "grass"
+grass:
+  database: "{context["GISDBASE"]}"
+  project: "{context["LOCATION_NAME"]}"
+  mapset: "{context["MAPSET"]}"
+  executable: "grass"
 time:
   duration: "00:01:00"
   record_step: "00:00:30"

From e5895a372e08c2cf6c70fabb8906e9b65c8d8d2d Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 15:58:34 -0600
Subject: [PATCH 78/85] remove unused digest calculations

---
 src/itzi/ensemble/models.py     | 28 +++-------------------------
 src/itzi/ensemble/resolution.py | 18 +++++++-----------
 src/itzi/ensemble/yaml.py       | 14 --------------
 src/itzi/run_plan.py            |  3 ---
 4 files changed, 10 insertions(+), 53 deletions(-)

diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py
index 1b274a90..dfca6e49 100644
--- a/src/itzi/ensemble/models.py
+++ b/src/itzi/ensemble/models.py
@@ -14,7 +14,6 @@
 
 from __future__ import annotations
 
-import json
 import string
 from dataclasses import dataclass
 from datetime import datetime, timedelta
@@ -29,7 +28,6 @@
     TemporalType,
 )
 from pydantic import BaseModel, ConfigDict, StrictStr, model_validator
-from pydantic import JsonValue as PydanticJsonValue
 
 from itzi.grass.session import GrassParams
 
@@ -64,8 +62,6 @@ class SourceDocument:
 
     path: Path
     document_index: int
-    file_digest: str
-    document_digest: str
 
 
 @dataclass(frozen=True)
@@ -190,7 +186,7 @@ class ResolvedSimulation:
     input_kinds: tuple[tuple[str, Literal["raster", "strds"]], ...]
     simulation_config: SimulationConfig
     artifacts: ArtifactSummary
-    normalized_payload: str
+    normalized_payload: str = ""
 
 
 @dataclass(frozen=True)
@@ -250,29 +246,12 @@ def format_duration(value: timedelta) -> str:
     return rendered
 
 
-def _canonical_json(value: PydanticJsonValue) -> str:
-    """Serialize a JSON-compatible value in a stable form."""
-    try:
-        return json.dumps(
-            value,
-            allow_nan=False,
-            ensure_ascii=True,
-            separators=(",", ":"),
-            sort_keys=True,
-        )
-    except ValueError as error:
-        raise EnsembleError("NaN and infinity are not supported") from error
-
-
 def render_template(template: str, *, ensemble: str, simulation: str | None) -> str:
     """Render the intentionally small output-template language."""
     formatter = string.Formatter()
     values = {"ensemble": ensemble, "simulation": simulation}
     try:
-        fragments = formatter.parse(template)
-        rendered: list[str] = []
-        for literal, field_name, format_spec, conversion in fragments:
-            rendered.append(literal)
+        for _, field_name, format_spec, conversion in formatter.parse(template):
             if field_name is None:
                 continue
             if field_name not in values:
@@ -283,10 +262,9 @@ def render_template(template: str, *, ensemble: str, simulation: str | None) ->
                 raise EnsembleError(
                     "template format specifications and conversions are not supported"
                 )
-            rendered.append(str(values[field_name]))
+        return template.format_map(values)
     except ValueError as error:
         raise EnsembleError(f"invalid output template: {error}") from error
-    return "".join(rendered)
 
 
 def check_batch_limits(ensembles: tuple[ExpandedEnsemble, ...]) -> None:
diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py
index 1acce064..2d233fb2 100644
--- a/src/itzi/ensemble/resolution.py
+++ b/src/itzi/ensemble/resolution.py
@@ -110,7 +110,7 @@ def _build_resolved_simulation(
     swmm_digest: str | None,
 ) -> ResolvedSimulation:
     assert actual_params.mapset is not None
-    simulation_id, normalized_payload = _simulation_identity(
+    simulation_id = _simulation_identity(
         expanded,
         actual_params,
         domain,
@@ -134,7 +134,6 @@ def _build_resolved_simulation(
         input_kinds=tuple(sorted(input_kinds.items())),
         simulation_config=simulation_config,
         artifacts=artifacts,
-        normalized_payload=normalized_payload,
     )
 
 
@@ -183,7 +182,7 @@ def _simulation_identity(
     effective_mask: EffectiveMask,
     input_names: dict[str, str],
     swmm_digest: str | None,
-) -> tuple[str, str]:
+) -> str:
     """Build the versioned identity payload after all source normalization."""
     drainage = dict(expanded.drainage) if expanded.drainage is not None else None
     if drainage is not None:
@@ -216,7 +215,7 @@ def _simulation_identity(
     }
     canonical = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True)
     digest = hashlib.blake2b(canonical.encode(), digest_size=4).hexdigest()
-    return f"sim-{digest}", canonical
+    return f"sim-{digest}"
 
 
 def _render_artifacts(
@@ -364,7 +363,7 @@ def validate_resolved_ensemble(
     simulations: tuple[ResolvedSimulation, ...], protected_inputs: tuple[Path, ...] = ()
 ) -> None:
     """Reject identity and artifact collisions among resolved ensemble members."""
-    identities: dict[str, str] = {}
+    identities: set[str] = set()
     artifacts: dict[str, str] = {}
     ensemble_sources: set[str] = set()
     protected_paths = {path.resolve() for path in protected_inputs}
@@ -375,12 +374,9 @@ def validate_resolved_ensemble(
         if simulation.simulation_config.swmm_inp is not None:
             protected_paths.add(simulation.simulation_config.swmm_inp.resolve())
     for simulation in simulations:
-        existing_payload = identities.get(simulation.simulation_id)
-        if existing_payload is not None:
-            if existing_payload == simulation.normalized_payload:
-                raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}")
-            raise EnsembleError(f"simulation ID digest collision {simulation.simulation_id}")
-        identities[simulation.simulation_id] = simulation.normalized_payload
+        if simulation.simulation_id in identities:
+            raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}")
+        identities.add(simulation.simulation_id)
 
         last_record_index = _last_record_index(
             simulation.simulation_config.end_time - simulation.simulation_config.start_time,
diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py
index c9bf43cb..7d7dbe2b 100644
--- a/src/itzi/ensemble/yaml.py
+++ b/src/itzi/ensemble/yaml.py
@@ -14,7 +14,6 @@
 
 from __future__ import annotations
 
-import hashlib
 import itertools
 import math
 from datetime import date, datetime, timedelta
@@ -43,7 +42,6 @@
     NormalizedTime,
     OutputTemplates,
     SourceDocument,
-    _canonical_json,
     parse_duration,
     render_template,
 )
@@ -175,14 +173,6 @@ def _parse_datetime(value: str) -> datetime:
         raise EnsembleError(f"invalid ISO 8601 timestamp {value!r}") from error
 
 
-def _document_digest(document: YamlDocument) -> str:
-    payload: dict[str, PydanticJsonValue] = {
-        "schema": "itzi-yaml-document-v1",
-        "document": document,
-    }
-    return hashlib.blake2b(_canonical_json(payload).encode(), digest_size=32).hexdigest()
-
-
 def _is_document_marker(line: str, *, terminated: bool) -> bool:
     """Return whether a physical line is a supported YAML document marker."""
     if terminated and line.endswith("\r"):
@@ -244,13 +234,11 @@ def load_yaml_stream(path: str | Path) -> LoadedYamlStream:
     except UnicodeDecodeError as error:
         raise EnsembleError(f"{source_path}: YAML must be UTF-8") from error
 
-    file_digest = hashlib.blake2b(content, digest_size=32).hexdigest()
     ensembles: list[ExpandedEnsemble] = []
     failures: list[DocumentFailure] = []
     for document_index, (start_line, _offset, segment) in enumerate(_segments(text)):
         try:
             raw_document = _parse_segment(segment)
-            document_digest = _document_digest(raw_document)
         except yaml.YAMLError as error:
             marker = getattr(error, "problem_mark", None)
             failures.append(
@@ -280,8 +268,6 @@ def load_yaml_stream(path: str | Path) -> LoadedYamlStream:
         source = SourceDocument(
             path=source_path,
             document_index=document_index,
-            file_digest=file_digest,
-            document_digest=document_digest,
         )
         try:
             document = YamlEnsembleDocumentV1.model_validate(raw_document)
diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py
index 35aa1614..175ea4f4 100644
--- a/src/itzi/run_plan.py
+++ b/src/itzi/run_plan.py
@@ -79,12 +79,9 @@ def _legacy_ensemble(path: Path) -> ExpandedEnsemble:
         raise EnsembleError("hotstart output requires Stage 2 checkpoint support in mixed batches")
     grass = reader.grass_params
     source_path = path.expanduser().resolve()
-    source_bytes = source_path.read_bytes()
     source = SourceDocument(
         path=source_path,
         document_index=0,
-        file_digest=hashlib.blake2b(source_bytes, digest_size=32).hexdigest(),
-        document_digest=hashlib.blake2b(b"legacy-v1\0" + source_bytes, digest_size=32).hexdigest(),
     )
     ensemble_id = f"legacy-{hashlib.blake2b(str(source_path).encode(), digest_size=8).hexdigest()}"
     grass_config = GrassConfig(

From ee10cdbe7e6b0014d32852abb28e61bc5b5352c0 Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 16:32:15 -0600
Subject: [PATCH 79/85] in case of simulation failing, finalize() si called
 within the grass env

---
 src/itzi/itzi.py             | 24 ++++++++++-----
 tests/cli/test_cli.py        | 59 ++++++++++++++++++++++++++++++++++++
 tests/grass/test_ensemble.py | 44 +++++++++++++++++++++++++++
 3 files changed, 119 insertions(+), 8 deletions(-)

diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py
index bb48e371..03dcc24b 100644
--- a/src/itzi/itzi.py
+++ b/src/itzi/itzi.py
@@ -110,7 +110,6 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str
     """Run one resolved member and return its status."""
     msgr.raise_on_error = True
     msgr._itzi_logger.set_verbosity(msgr.verbosity())
-    runner: SimulationRunner | None = None
     try:
         with GrassSessionManager(simulation.grass_params):
             runner = SimulationRunner(
@@ -122,15 +121,24 @@ def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str
                 effective_mask=simulation.effective_mask,
                 input_kinds=dict(simulation.input_kinds),
             )
-            runner.initialize().run().finalize()
-        return "completed", None
-    except Exception as error:
-        if runner is not None:
+            runner.initialize()
             try:
-                runner.finalize()
+                runner.run()
             except Exception:
-                pass
-        return "execution_failed", f"{type(error).__name__}: {error}"
+                try:
+                    runner.finalize()
+                except Exception as finalize_error:
+                    msgr.warning(
+                        f"Could not finalize partial outputs: "
+                        f"{type(finalize_error).__name__}: {finalize_error}"
+                    )
+                raise
+            runner.finalize()
+        return "completed", None
+    except Exception as error:
+        detail = f"{type(error).__name__}: {error}"
+        msgr.warning(f"Simulation failed: {detail}")
+        return "execution_failed", detail
 
 
 def resolver_worker(
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 4ae5f05e..f4a6762b 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -29,6 +29,7 @@
     itzi_run,
     itzi_run_one,
     main,
+    resolved_sim_runner_worker,
     run_validation_worker,
     reconcile_hotstart_commands,
     sim_runner_worker,
@@ -326,6 +327,64 @@ def validate_run_requirements(simulation):
     assert calls == ["open", "good", "bad", "another", "close"]
 
 
+@pytest.mark.parametrize("finalize_fails", [False, True])
+def test_failed_ensemble_member_finalizes_in_session_and_reports_error(
+    monkeypatch, itzi_stderr, finalize_fails: bool
+) -> None:
+    calls = []
+
+    class FakeGrassSessionManager:
+        def __init__(self, _params):
+            pass
+
+        def __enter__(self):
+            calls.append("open")
+
+        def __exit__(self, *_):
+            calls.append("close")
+
+    class FakeSimulationRunner:
+        def __init__(self, *_args, **_kwargs):
+            pass
+
+        def initialize(self):
+            return self
+
+        def run(self):
+            calls.append("run")
+            raise ValueError("tiny computed dt")
+
+        def finalize(self):
+            calls.append("finalize")
+            if finalize_fails:
+                raise RuntimeError("registration failed")
+
+    monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager)
+    monkeypatch.setattr("itzi.itzi.SimulationRunner", FakeSimulationRunner)
+    simulation = cast(
+        ResolvedSimulation,
+        SimpleNamespace(
+            grass_params=None,
+            simulation_config=None,
+            artifacts=SimpleNamespace(statistics_file=None),
+            effective_mask=None,
+            input_kinds=(),
+        ),
+    )
+
+    assert resolved_sim_runner_worker(simulation) == (
+        "execution_failed",
+        "ValueError: tiny computed dt",
+    )
+    assert calls == ["open", "run", "finalize", "close"]
+    stderr = itzi_stderr.getvalue()
+    assert stderr.count("WARNING: Simulation failed: ValueError: tiny computed dt") == 1
+    assert ("Could not finalize partial outputs: RuntimeError: registration failed" in stderr) == (
+        finalize_fails
+    )
+    assert "Traceback" not in stderr
+
+
 def test_run_parser_rejects_v_and_q_together():
     with pytest.raises(SystemExit):
         build_parser().parse_args(["run", "a.ini", "-v", "-q"])
diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py
index fbce1259..7ea1994d 100644
--- a/tests/grass/test_ensemble.py
+++ b/tests/grass/test_ensemble.py
@@ -17,6 +17,7 @@
 
 from itzi.cli_parser import build_parser
 from itzi.ensemble.models import ResolvedSimulation, ValidationFailure
+from itzi.grass.names import derived_record_name
 from itzi.grass.session import GrassSessionManager
 from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, run_validation_worker
 from itzi.run_plan import load_batch
@@ -336,6 +337,49 @@ def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path):
     )
 
 
+@pytest.mark.forked
+@pytest.mark.usefixtures("grass_5by5")
+def test_failed_ensemble_member_registers_partial_maps(tmp_path: Path) -> None:
+    import grass.temporal as tgis
+
+    context = gscript.gisenv()
+    depth_map = f"stage_unstable_{uuid4().hex[:8]}"
+    gscript.mapcalc(f"{depth_map}=1000000000000000.0")
+    path = write_study(
+        tmp_path,
+        grass={
+            "database": context["GISDBASE"],
+            "project": context["LOCATION_NAME"],
+            "mapset": context["MAPSET"],
+            "executable": "grass",
+        },
+        input={"water_depth": depth_map},
+        outputs={
+            "rasters": {
+                "prefix": "stage_partial_{simulation}",
+                "variables": ["max_water_depth"],
+            }
+        },
+    )
+
+    gisrc = os.environ.pop("GISRC")
+    try:
+        with pytest.raises(RuntimeError, match="failed validation or execution"):
+            itzi_run(build_parser().parse_args(["run", str(path), "-o"]))
+    finally:
+        os.environ["GISRC"] = gisrc
+
+    member = read_manifest(tmp_path)["members"][0]
+    assert member["status"] == "execution_failed"
+    assert member["failure"]["detail"].startswith("DtError:")
+    stds_id = member["artifacts"]["rasters"]["max_water_depth"]
+    record_id = derived_record_name(stds_id, 0)
+    assert gscript.find_file(name=record_id, element="cell").get("file")
+    GrassSessionManager.ensure_temporal_initialized()
+    stds = tgis.open_stds.open_old_stds(stds_id, "strds")
+    assert [row["id"] for row in stds.get_registered_maps(columns="id")] == [record_id]
+
+
 @pytest.mark.forked
 @pytest.mark.usefixtures("grass_5by5")
 def test_explicit_grass_context_runs_two_members_without_parent_session(test_data_temp_path):

From 18a8a8f8082a079f3c2b928a2748fcb796478d0f Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 18:12:52 -0600
Subject: [PATCH 80/85] update documentation

---
 docs/conf_file.rst | 105 ++--------------
 docs/index.rst     |   2 +
 docs/manifest.rst  | 102 ++++++++++++++++
 docs/yaml_file.rst | 291 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 402 insertions(+), 98 deletions(-)
 create mode 100644 docs/manifest.rst
 create mode 100644 docs/yaml_file.rst

diff --git a/docs/conf_file.rst b/docs/conf_file.rst
index 6964a571..4454e241 100644
--- a/docs/conf_file.rst
+++ b/docs/conf_file.rst
@@ -1,102 +1,11 @@
 
-Configuration file
-==================
-
-The public configuration format is a versioned YAML stream. Each non-empty
-YAML document defines one ensemble, and a document may contain sweep lists for
-Cartesian member expansion. A command may receive several YAML streams and
-legacy INI files in order.
-
-YAML files must use the ``.yaml`` extension. Other extensions are interpreted
-as deprecated legacy INI configurations during the migration release. A YAML
-syntax or schema error never falls back to INI parsing.
-
-The supported YAML subset rejects duplicate mapping keys, merge keys, custom
-tags, empty documents, and unknown schema fields. Quote timestamps, durations,
-map identifiers, paths, templates, and ambiguous strings such as ``yes`` or
-``on``. Documents after an explicit ``---`` marker are independently loaded,
-so an invalid document does not prevent a later explicitly delimited document
-from being reported.
-
-Basic YAML example
-------------------
-
-.. code-block:: yaml
-
-   schema_version: 1
-   ensemble:
-     id: "central-city"
-     description: "Central city rainfall study"
-   grass:
-     database: "/srv/grassdata"
-     project: "central_city"
-     mapset: "itzi"
-     executable: "grass"
-     region: "central_city_5m"
-     mask: "central_city_mask"
-   time:
-     start: "2026-09-01T00:00:00"
-     duration: "02:00:00"
-     record_step: "00:05:00"
-   input:
-     ground_elevation: "elevation_5m@PERMANENT"
-     friction: "manning_n@PERMANENT"
-     rainfall_rate: ["rain_10yr@PERMANENT", "rain_100yr@PERMANENT"]
-     infiltration:
-       type: "none"
-   parameters:
-     cfl: [0.5, 0.7]
-   outputs:
-     rasters:
-       prefix: "central_city_{simulation}"
-       variables: [water_depth, flow_speed]
-     statistics:
-       file: "results/{simulation}.csv"
-
-The example expands to four simulations: two rainfall maps multiplied by two
-``cfl`` values. Scalars are fixed across an ensemble, while non-empty lists in
-``input``, ``parameters``, or ``drainage`` are sweep dimensions. ``grass`` and
-``time`` are always scalar. Output variable lists are selections, not sweeps.
-The optional ``ensemble.description`` is limited to 256 characters.
-An ensemble may contain at most 100 candidate simulations; a full command may
-contain at most 200.
-
-All simulations in an ensemble use one geographical domain, time form, and
-``record_step``. Time accepts exactly one of a relative ``duration``, absolute
-``start`` plus ``duration``, or absolute ``start`` plus ``end``. Durations use
-the legacy ``HH:MM:SS`` form, such as ``"00:05:00"`` and ``"26:00:00"``.
-Hours may exceed 23; minutes and seconds must be between 0 and 59.
-Timezone offsets are retained as source metadata but execute as naive GRASS
-wall-clock times without conversion.
-
-Use ``grass: {}`` to select the active GRASS session and its current region and
-mask. Set ``grass.region`` to a named region or ``grass.mask`` to a raster map
-to override either one. To open a GRASS session explicitly, provide
-``grass.database``, ``grass.project``, and ``grass.mapset`` together;
-``grass.executable`` may be set with them.
-
-The ensemble manifest is written as YAML to
-``results/.manifest.yaml`` relative to the configuration file.
-Set ``outputs.manifest.file`` to override this path.
-
-YAML infiltration alternatives are explicit under ``input.infiltration``:
-``type: "none"``, ``type: "constant"`` with a ``rate`` map, or
-``type: "green-ampt"`` with ``effective_porosity``, ``capillary_pressure``,
-and ``hydraulic_conductivity`` maps.
-A list of complete alternatives is one sweep dimension.
-
-``{ensemble}`` and ``{simulation}`` are the only output-template substitutions.
-Output maps are always bare names in the active execution mapset; qualified
-output names are rejected. YAML filesystem paths are relative to the YAML
-source file. Use ``itzi run --dry-run study.yaml`` to resolve and validate members
-without creating output artifacts.
-
-Legacy INI reference
---------------------
-
-The remaining sections document the deprecated single-simulation INI format.
-It remains supported for one announced migration release. New configurations
-should use YAML.
+INI configuration file
+======================
+
+The parameters of a simulation are given through a configuration file in
+a format similar to Microsoft Windows INI files.
+An example is given in the tutorial above.
+The file is separated in sections described below.
 
 [time]
 ------
diff --git a/docs/index.rst b/docs/index.rst
index bd5b0043..0575c016 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -17,5 +17,7 @@ Please see the Itzï's `home page `__.
    cli
    tutorial
    conf_file
+   yaml_file
+   manifest
    faq
    prog_manual
diff --git a/docs/manifest.rst b/docs/manifest.rst
new file mode 100644
index 00000000..c7e1ece4
--- /dev/null
+++ b/docs/manifest.rst
@@ -0,0 +1,102 @@
+Ensemble manifest
+=================
+
+Each YAML ensemble run writes a YAML manifest describing its resolved members,
+their planned outputs, and their run status. By default, it is saved
+as ``results/.manifest.yaml`` beside the source YAML file. Set
+``outputs.manifest.file`` in the ensemble configuration to choose another file:
+
+.. code-block:: yaml
+
+   outputs:
+     manifest:
+       file: "results/{ensemble}.yaml"
+
+The manifest path may be absolute or relative to the source YAML file; only
+``{ensemble}`` is allowed as a template placeholder (not ``{simulation}``).
+An existing manifest requires ``itzi run -o`` to be replaced; if the manifest
+destination fails validation, no manifest is written. A dry run validates the
+destination but does not write a manifest. An ensemble with no selected resolved
+members does not write one, unless it contains unresolved candidates in a run
+without ``--member``.
+
+Format
+------
+
+The manifest is a YAML mapping with these top-level fields:
+
+* ``manifest_version``: manifest format version, currently ``1``. This is
+  distinct from the configuration's ``schema_version``.
+* ``last_updated_at``: last write time as an ISO 8601 string with a local
+  timezone offset. It changes when the manifest is updated.
+* ``ensemble``: ``id`` and ``description`` from the configuration;
+  ``description`` is ``null`` when omitted.
+* ``source``: absolute ``path`` to the source YAML file and zero-based
+  ``document_index`` within that YAML stream.
+* ``members``: resolved members in sweep expansion order, followed by
+  unresolved candidates in sweep expansion order.
+
+For example, a completed member might appear as follows (names and timings
+are illustrative):
+
+.. code-block:: yaml
+
+   manifest_version: 1
+   last_updated_at: '2026-09-30T10:20:30+02:00'
+   ensemble:
+     id: central-city
+     description: Central city rainfall study
+   source:
+     path: /srv/studies/city.yaml
+     document_index: 0
+   members:
+   - simulation_id: sim-1234abcd
+     selected: true
+     status: completed
+     elapsed_seconds: 12.5
+     coordinates:
+       parameters.cfl: 0.5
+     artifacts:
+       rasters:
+         water_depth: central_city_sim-1234abcd_water_depth@itzi
+       drainage: null
+       statistics: /srv/studies/results/sim-1234abcd.csv
+
+Member fields
+-------------
+
+``simulation_id`` is the resolved member ID (``sim-...``), or ``null`` if
+input resolution failed before an ID could be assigned. ``selected`` indicates
+whether this member was selected to run. ``coordinates`` maps swept configuration
+paths (for example, ``parameters.cfl`` or ``input.rainfall_rate``) to their
+chosen values; it is ``{}`` without sweeps. An ``input.infiltration`` sweep
+value is a mapping containing its ``type`` and applicable fields.
+
+Resolved members also have ``artifacts``: ``rasters`` maps requested output
+variables to GRASS space-time raster dataset IDs (``name@mapset``),
+``drainage`` is a GRASS space-time vector dataset ID or ``null``, and
+``statistics`` is an absolute CSV file path or ``null``. ``rasters`` is ``{}``
+when no raster outputs are requested. These are intended destinations; a
+failed or unfinished member may not have produced them. Unresolved candidates
+have no ``artifacts`` field.
+
+``elapsed_seconds`` is ``null`` until execution finishes, then records the
+elapsed wall-clock time in seconds, including when execution fails. The
+``status`` field can be:
+
+* ``not_selected``: excluded by ``--member``.
+* ``planned``: selected, but not yet started.
+* ``running``: execution started; the manifest is updated before and after
+  each member runs.
+* ``completed``: execution finished successfully.
+* ``validation_failed``: the member could not be run.
+* ``execution_failed``: execution raised an error.
+
+Failures also have a ``failure`` mapping with ``phase`` and ``detail``.
+Possible phases are ``input_resolution`` (no resolved ID or artifacts),
+``artifact_validation`` (conflicting or invalid outputs),
+``run_validation`` (pre-run checks), and ``execution``. Unresolved candidates
+are marked ``validation_failed`` when running without ``--member``; with
+selectors they appear as ``not_selected`` but still carry their failure
+detail. A failed YAML document (parse, schema, or expansion error) has no
+manifest entry because no ensemble was expanded from it.
diff --git a/docs/yaml_file.rst b/docs/yaml_file.rst
new file mode 100644
index 00000000..816d6c7a
--- /dev/null
+++ b/docs/yaml_file.rst
@@ -0,0 +1,291 @@
+YAML configuration file
+=======================
+
+A ``.yaml`` file is a stream of ensemble documents.
+One single ``.yaml`` file can contain multiple documents separated with ``---``.
+Each non-empty document defines one ensemble.
+Multiple YAML files and :doc:`INI configurations ` can be passed to ``itzi run`` in
+one command.
+Other filename extensions are treated as INI.
+YAML ensemble resume from hotstart is not currently supported.
+
+The file must be encoded in UTF-8.
+The supported subset rejects empty documents, duplicate
+keys, merge keys, YAML directives, custom tags, non-JSON values, and unknown
+options at every level.
+Put quotes around values that should be text, such as timestamps, durations,
+map names, paths, and output templates.
+Without quotes, YAML may read values like ``yes`` as a boolean or ``2026-09-01`` as a date instead of text.
+Leave numeric parameters unquoted (for example, ``cfl: 0.7``).
+An invalid document does not prevent later explicitly delimited documents from being checked.
+
+Example
+-------
+
+.. code-block:: yaml
+
+   schema_version: 1
+   ensemble:
+     id: "central-city"
+     description: "Central city rainfall study"
+   grass:
+     database: "/srv/grassdata"
+     project: "central_city"
+     mapset: "itzi"
+     region: "central_city_5m"
+   time:
+     start: "2026-09-01T00:00:00"
+     duration: "02:00:00"
+     record_step: "00:05:00"
+   input:
+     ground_elevation: "elevation_5m@PERMANENT"
+     friction: "manning_n@PERMANENT"
+     rainfall_rate: ["rain_10yr@PERMANENT", "rain_100yr@PERMANENT"]
+     infiltration:
+       type: "none"
+   parameters:
+     cfl: [0.5, 0.7]
+   outputs:
+     rasters:
+       prefix: "central_city_{simulation}"
+       variables: [water_depth, flow_speed]
+     statistics:
+       file: "results/{simulation}.csv"
+
+This produces four candidate simulations (two rainfall maps × two ``cfl`` values).
+Use ``itzi run --dry-run study.yaml`` to expand and validate members
+without writing a manifest or model outputs.
+
+Document structure and sweeps
+-----------------------------
+
+The top-level sections are ``schema_version``, ``ensemble``, ``grass``,
+``time``, ``input``, ``parameters``, ``drainage``, and ``outputs``. All except
+``drainage`` are required.
+Section names and option names are case-sensitive.
+
+Each ``input`` map, ``parameters`` value, and ``drainage`` option can be a
+scalar or a non-empty list of alternatives.
+Lists are independent Cartesian dimensions, or sweep.
+Duplicate values within a list are rejected.
+``input.infiltration`` can instead be a non-empty list of whole model alternatives.
+``grass``, ``time``, and output options are fixed across the ensemble.
+``outputs.rasters.variables`` is a selection list.
+At most 100 candidate simulations may come from one document and 200 from the whole command.
+Ensemble IDs must be unique across a batch.
+
+``schema_version``
+------------------
+
+Required integer, currently only ``1``. This is distinct from the
+:doc:`manifest version `.
+
+``ensemble``
+------------
+
+* ``id`` (required): 1–80 characters, starting with an ASCII letter or digit;
+  remaining characters may also be ``.``, ``_``, or ``-``. This identifies the
+  ensemble and names its default manifest. It must be unique within a batch.
+* ``description`` (optional): text up to 256 characters, included in the
+  manifest.
+
+``grass``
+---------
+
+Use ``grass: {}`` to use the active GRASS session, computational region, and mask.
+An explicit session requires all three of the following options:
+
+* ``database``: path to the GIS database.
+* ``project``: GRASS project (location) name.
+* ``mapset``: mapset name.
+
+``executable`` optionally selects the GRASS executable for an explicit
+session (defaults to ``grass``); it cannot be used without those three options.
+``region`` optionally names a saved region; ``mask`` optionally names a raster mask.
+Both can also be used to override the active region or mask for the run.
+When a session is already active, an explicit database, project, and mapset must match it.
+All members of an ensemble share this GRASS context and domain.
+Only a projected location / project is accepted, not latitude/longitude.
+The region must have at least 3 rows and 3 columns, and a mask cannot exclude the entire simulation domain.
+
+``time``
+--------
+
+* ``record_step`` (required): interval between recorded outputs.
+* ``duration``: length of the simulation; alone, it produces relative time.
+* ``start``: ISO 8601 date-time with ``T`` between date and time; with
+  ``duration`` or ``end``, it produces absolute time.
+* ``end``: ISO 8601 date-time after ``start``.
+
+Exactly one combination is allowed: ``duration`` alone, ``start`` with
+``duration``, or ``start`` with ``end``. ``record_step`` and ``duration`` must
+be positive ``HH:MM:SS`` strings (for example ``"26:00:00"``); hours may
+exceed 23, minutes and seconds must be 0–59. ``end`` must follow ``start``.
+For ``start``/``end``, either both timestamps carry timezone offsets or neither does.
+Offsets are ignored by GRASS: execution uses the supplied wall-clock values without timezone conversion.
+All members share the same time settings.
+
+``input``
+---------
+
+These options name a GRASS raster map or space-time raster dataset (STRDS), possibly qualified as ``name@mapset``.
+If the name resolves to both a raster and an STRDS, it is ambiguous and rejected.
+Maps in another mapset must be on the GRASS search path.
+``ground_elevation`` and ``friction`` must contain finite values in the active domain.
+Each of the following accepts one string or a non-empty list of map-name strings:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Option
+     - Meaning (unit)
+   * - ``ground_elevation`` (required)
+     - Terrain elevation (m).
+   * - ``friction`` (required)
+     - Manning's roughness ``n`` (s m⁻¹ᐟ³).
+   * - ``water_depth``
+     - Initial water depth (m).
+   * - ``water_surface_elevation``
+     - Initial surface elevation, ground elevation plus depth (m).
+   * - ``rainfall_rate``
+     - Rainfall (mm/h).
+   * - ``inflow``
+     - Point inflow (m/s).
+   * - ``losses``
+     - Loss rate (mm/h).
+   * - ``boundary_type``
+     - Boundary condition type (integer map values).
+   * - ``boundary_value``
+     - Boundary condition value (m).
+
+``water_depth`` and ``water_surface_elevation`` cannot both be specified.
+Boundary types are 0/1 (closed), 2 (open), and 4 (specified water depth);
+type 3 is not implemented. Open and closed types only apply at the
+computational region border. See :doc:`conf_file` for detailed input and
+boundary behavior, including caveats about time-varying input datasets.
+
+``input.infiltration`` is a single mapping (defaults to ``{type: none}``) or
+a list of complete mappings, each with one of these shapes:
+
+.. code-block:: yaml
+
+   infiltration:
+     - type: "none"
+     - type: "constant"
+       rate: "infiltration_rate@PERMANENT"
+     - type: "green-ampt"
+       effective_porosity: "porosity@PERMANENT"
+       capillary_pressure: "suction@PERMANENT"
+       hydraulic_conductivity: "conductivity@PERMANENT"
+       soil_water_content: "initial_moisture@PERMANENT"
+
+``constant.rate`` is an infiltration rate map (mm/h).
+The Green-Ampt maps ``effective_porosity`` (fraction),
+``capillary_pressure`` (mm),
+and ``hydraulic_conductivity`` (mm/h) are required together.
+``soil_water_content`` (fraction) is optional.
+The model types are mutually exclusive, and fields inside an infiltration alternative cannot be sweeps.
+
+``parameters``
+--------------
+
+All options are optional.
+Each takes a finite YAML number (integers are accepted) or
+a non-empty list of finite numbers.
+Quoted numbers are not accepted.
+Values must also meet these model limits:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Option
+     - Meaning
+     - Default
+     - Range
+   * - ``hmin``
+     - Minimum water depth (m)
+     - 0.005
+     - ≥ 0
+   * - ``cfl``
+     - Time-step coefficient
+     - 0.7
+     - 0.01–1
+   * - ``theta``
+     - Inertia weighting
+     - 0.9
+     - 0–1
+   * - ``g``
+     - Gravity (m/s²)
+     - 9.80665
+     - ≥ 0
+   * - ``dtmax``
+     - Maximum surface-flow time-step (s)
+     - 5.0
+     - > 0
+   * - ``dtinf``
+     - Infiltration/losses time-step (s)
+     - 60.0
+     - > 0
+   * - ``slope_threshold``
+     - GMS slope threshold (m/m)
+     - 0.8
+     - ≥ 0
+   * - ``max_slope``
+     - Maximum GMS slope (m/m)
+     - 0.8
+     - ≥ ``slope_threshold``
+   * - ``max_error``
+     - Maximum relative volume error
+     - 0.05
+     - > 0
+
+``drainage``
+------------
+
+Optional; enables experimental SWMM coupling. If present, **all four** options
+are required (each can be a scalar or a non-empty sweep list):
+
+* ``swmm_input``: path to an EPA SWMM ``.inp`` file. Relative paths are
+  resolved from the YAML file's directory, not the current working directory.
+* ``orifice_coeff``: flow-exchange coefficient, float from 0 to 1.
+* ``free_weir_coeff``: flow-exchange coefficient, float from 0 to 1.
+* ``submerged_weir_coeff``: flow-exchange coefficient, float from 0 to 1.
+
+Unlike INI, YAML requires the coefficients explicitly when drainage is enabled;
+the corresponding INI defaults are 0.167, 0.54, and 0.056. SWMM input files
+must exist at resolution time. ``outputs.drainage`` is optional even when
+coupling is enabled; without it, no drainage vector dataset is written. See
+:doc:`conf_file` for the contents of drainage vector output.
+
+``outputs``
+-----------
+
+This section is required, but any of its four subsections may be omitted:
+
+* ``rasters``: requires ``prefix`` (GRASS name prefix template) and
+  ``variables`` (non-empty list of distinct output names). For each variable,
+  Itzi writes a STRDS named ``_``. Supported variables are
+  ``water_depth``, ``max_water_depth``, ``water_surface_elevation``,
+  ``flow_speed``, ``max_flow_speed``, ``flow_velocity_direction``, ``froude``,
+  ``flow_rate_x``, ``flow_rate_y``, ``mean_boundary_flow``,
+  ``mean_infiltration``, ``mean_rainfall``, ``mean_inflow``, ``mean_losses``,
+  ``mean_drainage_flow``, and ``created_volume``. See :doc:`conf_file` for
+  meanings, units, and output caveats. Legacy output aliases are not accepted.
+* ``statistics``: requires ``file`` (non-empty CSV file path template); the
+  file is updated at each ``record_step``. See :doc:`conf_file` for its columns.
+* ``drainage``: requires ``vector_dataset`` (GRASS vector dataset name
+  template); requires the top-level ``drainage`` section. Drainage vectors
+  form a space-time vector dataset (STVDS).
+* ``manifest``: requires ``file`` (non-empty YAML path template). By default,
+  the manifest goes to ``results/.manifest.yaml`` relative to the
+  YAML source. See :doc:`manifest` for its contents.
+
+``prefix``, ``statistics.file``, and ``drainage.vector_dataset`` can use
+``{ensemble}`` and ``{simulation}`` placeholders. ``manifest.file`` can use
+only ``{ensemble}``. Formatting specifications and conversions are unsupported;
+use ``{{`` and ``}}`` for literal braces. Filesystem output paths may be
+absolute or relative to the YAML source file; ``~`` is expanded. GRASS output
+names must be unqualified, valid names in the execution mapset (not ``MASK``).
+Derived record names must also be valid. Colliding output names, existing
+outputs without ``-o``, and file outputs that would overwrite the YAML source,
+SWMM input, or each other are rejected during validation.

From d871d4eaa03186c3a7cf0710c4b36028ba40048d Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 18:29:10 -0600
Subject: [PATCH 81/85] drainage coefficients are optional

---
 docs/yaml_file.rst                | 29 +++++++++----------
 src/itzi/ensemble/schema.py       |  8 +++---
 src/itzi/ensemble/yaml.py         |  6 ++--
 tests/cli/test_ensemble_config.py | 46 +++++++++++++++++++++++++++++++
 4 files changed, 69 insertions(+), 20 deletions(-)

diff --git a/docs/yaml_file.rst b/docs/yaml_file.rst
index 816d6c7a..c668a0be 100644
--- a/docs/yaml_file.rst
+++ b/docs/yaml_file.rst
@@ -242,20 +242,21 @@ Values must also meet these model limits:
 ``drainage``
 ------------
 
-Optional; enables experimental SWMM coupling. If present, **all four** options
-are required (each can be a scalar or a non-empty sweep list):
-
-* ``swmm_input``: path to an EPA SWMM ``.inp`` file. Relative paths are
-  resolved from the YAML file's directory, not the current working directory.
-* ``orifice_coeff``: flow-exchange coefficient, float from 0 to 1.
-* ``free_weir_coeff``: flow-exchange coefficient, float from 0 to 1.
-* ``submerged_weir_coeff``: flow-exchange coefficient, float from 0 to 1.
-
-Unlike INI, YAML requires the coefficients explicitly when drainage is enabled;
-the corresponding INI defaults are 0.167, 0.54, and 0.056. SWMM input files
-must exist at resolution time. ``outputs.drainage`` is optional even when
-coupling is enabled; without it, no drainage vector dataset is written. See
-:doc:`conf_file` for the contents of drainage vector output.
+Optional; enables SWMM coupling.
+``swmm_input`` is required when the section is present.
+Each supplied option can be a scalar or a non-empty sweep list:
+
+* ``swmm_input``: path to an EPA SWMM ``.inp`` file.
+  Relative paths are resolved from the YAML file's directory, not the current working directory.
+* ``orifice_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.167).
+* ``free_weir_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.54).
+* ``submerged_weir_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.056).
+
+Omitted coefficients use the defaults.
+SWMM input files must exist at resolution time.
+``outputs.drainage`` is optional even when coupling is enabled
+Without it, no drainage vector dataset is written.
+See :doc:`conf_file` for the contents of drainage vector output.
 
 ``outputs``
 -----------
diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py
index 2e945f0a..547aa97c 100644
--- a/src/itzi/ensemble/schema.py
+++ b/src/itzi/ensemble/schema.py
@@ -127,9 +127,9 @@ class ParameterSweepConfig(StrictModel):
 
 class DrainageSweepConfig(StrictModel):
     swmm_input: SweepString
-    orifice_coeff: SweepFloat
-    free_weir_coeff: SweepFloat
-    submerged_weir_coeff: SweepFloat
+    orifice_coeff: SweepFloat | None = None
+    free_weir_coeff: SweepFloat | None = None
+    submerged_weir_coeff: SweepFloat | None = None
 
 
 class RasterOutputs(StrictModel):
@@ -179,5 +179,5 @@ class YamlEnsembleDocumentV1(StrictModel):
     @model_validator(mode="after")
     def validate_drainage_output(self) -> YamlEnsembleDocumentV1:
         if self.outputs.drainage is not None and self.drainage is None:
-            raise ValueError("outputs.drainage requires a complete drainage configuration")
+            raise ValueError("outputs.drainage requires a drainage configuration")
         return self
diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py
index 7d7dbe2b..ed85373a 100644
--- a/src/itzi/ensemble/yaml.py
+++ b/src/itzi/ensemble/yaml.py
@@ -389,7 +389,7 @@ def _collect_dimensions(
             dimensions.append((f"parameters.{key}", cast(tuple[SweepScalar, ...], tuple(value))))
     if document.drainage is not None:
         for key in type(document.drainage).model_fields:
-            value = cast(SweepString | SweepFloat, getattr(document.drainage, key))
+            value = cast(SweepString | SweepFloat | None, getattr(document.drainage, key))
             if isinstance(value, list):
                 dimensions.append((f"drainage.{key}", cast(tuple[SweepScalar, ...], tuple(value))))
     return tuple(sorted(dimensions, key=lambda item: item[0]))
@@ -448,8 +448,10 @@ def _selected_drainage(
     values: dict[str, str | float] = {}
     for key in type(drainage).model_fields:
         value = selected.get(
-            f"drainage.{key}", cast(SweepString | SweepFloat, getattr(drainage, key))
+            f"drainage.{key}", cast(SweepString | SweepFloat | None, getattr(drainage, key))
         )
+        if value is None:
+            continue
         assert isinstance(value, (str, float)), "drainage must be scalar after expansion"
         values[key] = value
     return values
diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py
index a2950739..695809ec 100644
--- a/tests/cli/test_ensemble_config.py
+++ b/tests/cli/test_ensemble_config.py
@@ -118,6 +118,52 @@ def test_drainage_coefficients_are_forwarded(tmp_path):
     ) == (0.2, 0.3, 0.4)
 
 
+def test_drainage_coefficients_can_be_omitted(tmp_path):
+    content = document().replace(
+        "outputs:\n", 'drainage:\n  swmm_input: "network.inp"\noutputs:\n', 1
+    )
+    stream = load_yaml_stream(write_yaml(tmp_path, content))
+
+    assert stream.failures == ()
+    expanded = stream.ensembles[0].simulations[0]
+    assert dict(expanded.drainage) == {"swmm_input": "network.inp"}
+    config = _build_simulation_config(
+        expanded,
+        dict(expanded.input_maps),
+        ArtifactSummary((), None, None),
+        tmp_path / "network.inp",
+    )
+    assert (
+        config.orifice_coeff,
+        config.free_weir_coeff,
+        config.submerged_weir_coeff,
+    ) == (
+        DefaultValues.ORIFICE_COEFF,
+        DefaultValues.FREE_WEIR_COEFF,
+        DefaultValues.SUBMERGED_WEIR_COEFF,
+    )
+
+
+def test_drainage_coefficients_allow_partial_sweep(tmp_path):
+    content = document().replace(
+        "outputs:\n",
+        'drainage:\n  swmm_input: "network.inp"\n  orifice_coeff: 0.2\n'
+        "  free_weir_coeff: [0.3, 0.4]\noutputs:\n",
+        1,
+    )
+    stream = load_yaml_stream(write_yaml(tmp_path, content))
+
+    assert stream.failures == ()
+    simulations = stream.ensembles[0].simulations
+    assert [dict(simulation.drainage) for simulation in simulations] == [
+        {"swmm_input": "network.inp", "orifice_coeff": 0.2, "free_weir_coeff": coefficient}
+        for coefficient in (0.3, 0.4)
+    ]
+    assert [dict(simulation.coordinates) for simulation in simulations] == [
+        {"drainage.free_weir_coeff": coefficient} for coefficient in (0.3, 0.4)
+    ]
+
+
 def test_time_config_accepts_only_supported_combinations():
     start = "2026-09-01T00:00:00"
     end = "2026-09-01T02:00:00"

From 35d490bb99bc799f1635d11caf5d0d96927923ee Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 18:45:16 -0600
Subject: [PATCH 82/85] AGETNS: do no duplicate pydantic declarative validation

---
 AGENTS.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/AGENTS.md b/AGENTS.md
index e4feb3fb..075b3d42 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -11,8 +11,9 @@
 - Do not use the `Any` type, and avoid the use of the general `object` type. Define the actual type instead.
 - Since the arguments types and return types are already documented by the annotations, there's no need to duplicate this information in the docstrings.
 - Use pydantic BaseModel when validation is needed, and dataclass for internal data exchange.
+- No need to write tests to check if declarative Pydantic validation is working.
 - Place imports at the top of the file. Only break this rule to prevent heavy imports in a rarely used function (for example, CLI options).
-- When passing data structures between fucntions, prefer immutable objects instead of equivalent mutable ones, unless immutability is desired.
+- When passing data structures between functions, prefer immutable objects instead of equivalent mutable ones, unless immutability is desired.
 - For GRASS interactions, prefer C functions calls through `pygrass` when available, instead of sub-process tool calls through `grass.script`
 
 ## General comments

From e2600db2920672cb02516300e4fa6d4ca5e137b0 Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 18:45:34 -0600
Subject: [PATCH 83/85] remove exogenous doc

---
 docs/yaml-ensemble-report.html | 195 ---------------------------------
 1 file changed, 195 deletions(-)
 delete mode 100644 docs/yaml-ensemble-report.html

diff --git a/docs/yaml-ensemble-report.html b/docs/yaml-ensemble-report.html
deleted file mode 100644
index ef896a52..00000000
--- a/docs/yaml-ensemble-report.html
+++ /dev/null
@@ -1,195 +0,0 @@
-
-
-
-  
-  
-  YAML ensembles | Itzï branch report
-  
-
-
-  

-    
Itzï · branch comparison: ensemble vs main

-    
YAML configuration & ensemble runs

-    
One YAML document now describes a study rather than just one run. Lists of inputs or settings turn it into several simulations, which Itzï checks and runs one at a time.

-  

-  

-    

-      
What changed

-      

-        
Before
Each INI file described one simulation. A batch ran several INI files, with the legacy hotstart and resume workflow.

-        
On this branch
.yaml files contain one or more documents separated by ---. Each document defines an ensemble: a shared domain and clock, plus one or more combinations of input maps and numerical parameters. A command can process several YAML files and legacy INI files in order.

-      

-      
The .yaml suffix selects YAML parsing. Other suffixes use the deprecated INI reader; invalid YAML is never silently treated as INI.

-    

-
-    

-      
A four-member study

-      
This example assumes the GRASS session can see the named rasters. Save it as study.yaml:

-      
schema_version: 1
-ensemble:
-  id: "study"
-grass: {}
-time:
-  duration: "02:00:00"
-  record_step: "00:05:00"
-input:
-  ground_elevation: "terrain"
-  friction: "manning"
-  rainfall_rate: ["rain_10yr", "rain_100yr"]
-parameters:
-  cfl: [0.5, 0.7]
-outputs:
-  rasters:
-    prefix: "study_{simulation}"
-    variables: [water_depth, flow_speed]
-  statistics:
-    file: "results/{simulation}.csv"

-      
Two rainfall choices × two cfl choices = four simulations. Each has the same terrain, friction, region, duration and recording interval. variables chooses what to save; it does not multiply the number of simulations.

-      

-        
-        
-          
-          
-          
-          
-          
-          
-        
-      
YAML fieldPlain-English meaning
schema_version, ensembleFormat version 1 and a unique study ID. An optional name is descriptive.
grassOne GRASS configuration for the ensemble. Use grass: {} for the active session; optionally specify database, project, and mapset together, plus executable. Optional region and mask names apply to all members.
timeOne clock for every member: duration alone, start + duration, or start + end, always with record_step. Durations are positive HH:MM:SS, including hours over 23.
inputTerrain (ground_elevation) and friction are required. Other raster/space-time raster inputs are optional. A scalar fixes a value; a nonempty list sweeps it.
parameters, drainageNumerical settings use core defaults unless supplied; drainage is optional and requires the SWMM input and three exchange coefficients. Their lists can also be swept.
outputsOptional raster, CSV statistics, drainage-vector and manifest destinations. The section itself is required, even if empty (outputs: {}).

-      
input.infiltration is either {type: "none"} (the default), {type: "constant", rate: "map"}, or type: "green-ampt" with maps for effective porosity, capillary pressure and hydraulic conductivity (optionally soil water content). A list of complete infiltration alternatives counts as one sweep dimension.

-    

-
-    

-      
From choices to runnable members

-      
    
-        
  1. Read and expand. YAML is safely parsed, checked against a strict versioned schema, and every combination of list values is produced. Unknown fields, duplicate or merge keys, custom tags, empty documents and duplicate sweep values are rejected. An explicitly separated later document can still load after an earlier document fails.
    2. 
-        
  2. Resolve against GRASS. A fresh process opens the requested or active GRASS context, finds the region and mask, resolves each input as a raster or space-time raster dataset, and builds a simulation configuration. Ambiguous or missing inputs fail for that member. Optional SWMM files are resolved relative to the YAML file.
    3. 
-        
  3. Name and check outputs. Each successfully resolved member gets a sim-xxxxxxxx ID from a digest of its normalized domain, time, inputs, settings, mask and drainage information. This ID is based on the resolved run, not its position in the sweep. Itzï renders output templates, then checks for duplicate IDs, collisions between members, and output names that would overwrite inputs or the mask.
    4. 
-        
  4. Validate run requirements and run. Selected members are checked for usable input data, temporal coverage, existing destinations and a still-matching GRASS environment. The real run writes a manifest, then runs eligible members sequentially in separate processes. After the first member, each next member's run requirements are validated again because earlier output may have changed what exists.
    5. 
-      

-      
The limits apply before member selection: at most 100 candidate simulations per document and 200 across the batch.

-    

-
-    

-      
Execution flow

-      
The CLI branches into the purple legacy INI path or the YAML path. Teal YAML cards run in spawned GRASS processes; YAML then branches into dry-run and real-run outcomes. Labels use Python module and function or class method names.

-      

-        
itzi.itzi.main() → itzi.itzi.itzi_run()Parse “itzi run” and choose a route

-        
↓ choose a route ↓

-        

-          

-            
Legacy INI-only

-            

-              
itzi.itzi._uses_legacy_execution_path()Only INI files; no --member or --dry-run

-              
itzi.itzi._run_legacy_batch()Warn that INI is deprecated; process files in order

-              
itzi.itzi.reconcile_hotstart_commands()Map optional --resume-from checkpoints, or start fresh

-              
itzi.itzi.itzi_run_one()Launch one subprocess for each INI file

-              
itzi.itzi.sim_runner_worker()ConfigReader.__init__() reads this INI file

-              
GrassSessionManager.open()Enter the requested or active GRASS session

-              
SimulationRunner.initialize()Set up providers; optionally load a hotstart

-              
SimulationRunner.run() → SimulationRunner.step()Advance the model until the end time

-              
SimulationRunner.finalize()Clean up; the worker exits with a status

-              
itzi.itzi._run_legacy_batch()Repeat for the next INI; report times and failures

-            

-          

-          

-            
YAML ensemble

-            

-              
itzi.itzi._run_ensemble_batch()Coordinate ordered documents and collect errors

-              
itzi.run_plan.load_batch() → itzi.ensemble.yaml.load_yaml_stream()Read each .yaml stream (or adapt an INI file)

-              
itzi.ensemble.schema.YamlEnsembleDocumentV1.model_validate()itzi.ensemble.yaml.expand_yaml_document() builds the Cartesian members

-              
itzi.itzi._resolve_ensemble_in_subprocess()itzi.itzi.resolver_worker() · one spawned GRASS process per ensemble

-              
itzi.ensemble.resolution.resolve_ensemble()itzi.ensemble.resolution._build_resolved_simulation() creates member IDs

-              
itzi.ensemble.resolution.validate_resolved_ensemble()Reject artifact collisions; itzi.itzi._select_members() applies --member

-               
itzi.itzi._validate_ensemble_run_requirements()itzi.itzi._validate_run_requirements_in_subprocess() · one process per ensemble

-               
itzi.itzi.run_validation_worker()itzi.ensemble.run_validation.validate_run_requirements() checks each selected member

-            

-            
↓ choose an outcome ↓

-            

-              

-                
--dry-run

-                
itzi.itzi._display_dry_plan()Print IDs, destinations and errors; write nothing

-              

-              

-                
Real run

-                

-                  
itzi.itzi._run_one_ensemble()itzi.manifest._create_manifest() records all members

-                   
itzi.itzi._run_one_ensemble()Later members: itzi.itzi._validate_run_requirements_in_subprocess(); itzi.manifest._update_manifest() marks members running

-                  
itzi.itzi._run_simulation_in_subprocess() → itzi.itzi.resolved_sim_runner_worker()New GRASS process for this member

-                  
SimulationRunner.initialize() → SimulationRunner.run() → SimulationRunner.step() → SimulationRunner.finalize()Build providers, advance the model, clean up

-                  
itzi.manifest._update_manifest()Record completion or execution failure

-                

-              

-            

-          

-        

-      

-       
The INI path runs each file in a separate subprocess and supports mapped --resume-from checkpoints. The YAML real-run loop repeats the per-member run-requirements validation and execution boxes for each selected, eligible member. Each worker opens a GRASS session with GrassSessionManager.open(); separate processes prevent GRASS temporal state leaking across mapsets.

-    

-
-    

-      
Using and reading the results

-      
itzi run study.yaml --dry-run
-itzi run study.yaml --member 'study#sim-xxxxxxxx'
-itzi run study.yaml

-      
First inspect the dry-run output for the actual sim-… ID; replace xxxxxxxx above with those eight hex digits. Repeat --member to choose several. If the command contains exactly one ensemble, the simulation ID may be unqualified. -o permits overwriting existing destinations.

-      
    
-        
  • Raster variables become GRASS space-time raster outputs named from the prefix plus _variable; each record has its own map. Statistics go to CSV, and optional SWMM drainage results can go to a space-time vector dataset.
    • 
-        
  • {ensemble} and {simulation} are the only template placeholders. The manifest path may use {ensemble}, but not {simulation}. Filesystem paths in YAML are relative to its file; output GRASS map names must be unqualified names in the execution mapset.
    • 
-        
  • A real run writes results/study.manifest.yaml beside this example file by default. outputs.manifest.file overrides that path. The YAML manifest records source digests, member choices, destinations, statuses (planned, running, completed, validation_failed, execution_failed, not_selected) and failures/elapsed time when applicable.
    • 
-        
  • Dry runs perform validation without creating manifests or model outputs. Document and member failures are reported; valid members in other documents can still be processed. The command ends with an error status when relevant validation or execution fails.
    • 
-      

-      
Compatibility: A plain INI-only run keeps its old execution and hotstart-resume path. INI files in the new batch planner (for example, mixed with YAML or using --dry-run) are adapted as single-member ensembles. YAML resume and checkpoint creation are not yet supported; --resume-from cannot be combined with the new ensemble path.

-    

-  

-  
-
-

From 13f3e8aab679418210c554dcb2071f963bdb7abd Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 19:20:44 -0600
Subject: [PATCH 84/85] reorganize imports, and fold write_raster_map_blocking
 into write_raster_map

---
 src/itzi/ensemble/resolution.py        |  2 +-
 src/itzi/ensemble/schema.py            |  1 +
 src/itzi/grass/interface.py            | 49 +++++++++++++++++++++-----
 src/itzi/grass/utils.py                | 46 ------------------------
 tests/cli/test_cli.py                  |  4 +--
 tests/cli/test_ensemble_config.py      |  4 +--
 tests/grass/test_ensemble.py           |  2 +-
 tests/grass/test_grass_input.py        |  2 +-
 tests/grass/test_temporal_overwrite.py |  4 +--
 9 files changed, 50 insertions(+), 64 deletions(-)

diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py
index 2d233fb2..355514ef 100644
--- a/src/itzi/ensemble/resolution.py
+++ b/src/itzi/ensemble/resolution.py
@@ -34,8 +34,8 @@
     format_duration,
     render_template,
 )
-from itzi.grass.session import GrassParams, GrassSessionManager
 from itzi.grass.names import derived_drainage_table_names, derived_record_name
+from itzi.grass.session import GrassParams, GrassSessionManager
 
 
 def resolve_ensemble(
diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py
index 547aa97c..d3288264 100644
--- a/src/itzi/ensemble/schema.py
+++ b/src/itzi/ensemble/schema.py
@@ -25,6 +25,7 @@
     field_validator,
     model_validator,
 )
+
 from itzi.ensemble.models import GrassConfig, StrictModel
 
 
diff --git a/src/itzi/grass/interface.py b/src/itzi/grass/interface.py
index f506519c..94c7fb56 100644
--- a/src/itzi/grass/interface.py
+++ b/src/itzi/grass/interface.py
@@ -16,6 +16,7 @@
 import os
 from collections.abc import Mapping
 from datetime import datetime, timedelta
+from pathlib import Path
 from types import MappingProxyType
 from typing import ClassVar, NamedTuple
 
@@ -23,6 +24,7 @@
 import grass.script as gscript
 import grass.temporal as tgis
 import numpy as np
+from grass.pygrass import raster
 from grass.pygrass.gis.region import Region
 from grass.pygrass.vector import VectorTopo
 from grass.pygrass.vector.geometry import Line, Point
@@ -41,15 +43,33 @@
     MapData,
     RasterMapType,
     STDSType,
+    get_current_mapset,
     name_is_map,
     output_name,
     read_domain,
     replace_cell_null_sentinel,
     resolve_effective_mask,
+    set_null,
     split_identifier,
-    write_raster_map_blocking,
 )
 
+# color rules
+_DIR = Path(__file__).parent / "colortable"
+RULE_H = _DIR / "depth.txt"
+RULE_V = _DIR / "velocity.txt"
+RULE_VDIR = _DIR / "vdir.txt"
+RULE_FR = _DIR / "froude.txt"
+colors_rules_dict = {
+    "water_depth": str(RULE_H),
+    "max_water_depth": str(RULE_H),
+    "flow_speed": str(RULE_V),
+    "max_flow_speed": str(RULE_V),
+    "flow_velocity_direction": str(RULE_VDIR),
+    "froude": str(RULE_FR),
+}
+for f in colors_rules_dict.values():
+    assert Path(f).is_file()
+
 
 class DBLinkDescription(NamedTuple):
     layer: int
@@ -359,14 +379,25 @@ def read_raster_map(self, raster_id: str) -> np.ndarray:
 
     def write_raster_map(self, arr: np.ndarray, raster_id: str, mkey: str, hmin: float) -> None:
         """Take a numpy array and write it to GRASS DB"""
-        write_raster_map_blocking(
-            arr,
-            output_name(raster_id),
-            self.grass_dtype(str(arr.dtype)),
-            mkey,
-            hmin,
-            self.overwrite,
-        )
+        map_type = self.grass_dtype(str(arr.dtype))
+        with raster.RasterRow(
+            raster_id,
+            mapset=get_current_mapset(),
+            mode="w",
+            mtype=map_type,
+            overwrite=self.overwrite,
+        ) as newraster:
+            newrow = raster.Buffer((arr.shape[1],), mtype=map_type)
+            for row in arr:
+                newrow[:] = row[:]
+                newraster.put_row(newrow)
+        # apply color table
+        colors_rules = colors_rules_dict.get(mkey)
+        if colors_rules is not None:
+            gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=raster_id)
+        # set null values
+        if mkey == "water_depth" and hmin > 0:
+            set_null(raster_id, hmin)
 
     def create_db_links(
         self, vect_map: VectorTopo, linking_elem: dict[str, DBLayerDescription]
diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py
index 1a571734..5da57dfb 100644
--- a/src/itzi/grass/utils.py
+++ b/src/itzi/grass/utils.py
@@ -23,7 +23,6 @@
 import grass.script as gscript
 import grass.temporal as tgis
 import numpy as np
-from grass.pygrass import raster
 from grass.pygrass.gis import Mapset
 from grass.pygrass.gis.region import Region
 from itzi_core import INPUT_ARRAY_KEYS, DomainData
@@ -37,23 +36,6 @@
 
 MIN_GRASS_VERSION = (8, 4)
 
-# color rules
-_DIR = Path(__file__).parent / "colortable"
-RULE_H = _DIR / "depth.txt"
-RULE_V = _DIR / "velocity.txt"
-RULE_VDIR = _DIR / "vdir.txt"
-RULE_FR = _DIR / "froude.txt"
-colors_rules_dict = {
-    "water_depth": str(RULE_H),
-    "max_water_depth": str(RULE_H),
-    "flow_speed": str(RULE_V),
-    "max_flow_speed": str(RULE_V),
-    "flow_velocity_direction": str(RULE_VDIR),
-    "froude": str(RULE_FR),
-}
-for f in colors_rules_dict.values():
-    assert Path(f).is_file()
-
 
 type STDSType = Literal["strds", "stvds"]
 type RasterMapType = Literal["CELL", "FCELL", "DCELL"]
@@ -88,34 +70,6 @@ def check_output_files(file_list: list[str]) -> None:
             msgr.fatal(f"File {map_name} exists and will not be overwritten")
 
 
-def write_raster_map_blocking(
-    array: np.ndarray,
-    raster_name: str,
-    map_type: RasterMapType,
-    map_key: str,
-    hmin: float,
-    overwrite: bool,
-) -> None:
-    with raster.RasterRow(
-        raster_name,
-        mapset=get_current_mapset(),
-        mode="w",
-        mtype=map_type,
-        overwrite=overwrite,
-    ) as newraster:
-        newrow = raster.Buffer((array.shape[1],), mtype=map_type)
-        for row in array:
-            newrow[:] = row[:]
-            newraster.put_row(newrow)
-    # apply color table
-    colors_rules = colors_rules_dict.get(map_key)
-    if colors_rules is not None:
-        gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=raster_name)
-    # set null values
-    if map_key == "water_depth" and hmin > 0:
-        set_null(raster_name, hmin)
-
-
 def resolve_effective_mask(mask: str | None) -> EffectiveMask:
     """Select the effective mask descriptor without modifying the mapset MASK."""
     current_mapset = gutils.getenv("MAPSET")
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index f4a6762b..625fd25b 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -22,16 +22,16 @@
 from itzi.grass.session import GrassParams
 from itzi.itzi import (
     VerbosityLevel,
-    _validate_ensemble_run_requirements,
     _run_ensemble_batch,
     _run_one_ensemble,
     _select_members,
+    _validate_ensemble_run_requirements,
     itzi_run,
     itzi_run_one,
     main,
+    reconcile_hotstart_commands,
     resolved_sim_runner_worker,
     run_validation_worker,
-    reconcile_hotstart_commands,
     sim_runner_worker,
 )
 
diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py
index 695809ec..bb72c5b1 100644
--- a/tests/cli/test_ensemble_config.py
+++ b/tests/cli/test_ensemble_config.py
@@ -2,9 +2,9 @@
 
 from __future__ import annotations
 
+import sys
 from dataclasses import replace
 from datetime import datetime, timedelta
-import sys
 from types import SimpleNamespace
 from typing import cast
 
@@ -23,7 +23,6 @@
     parse_duration,
     render_template,
 )
-from itzi.ensemble.schema import TimeConfig
 from itzi.ensemble.resolution import (
     _build_simulation_config,
     _last_record_index,
@@ -32,6 +31,7 @@
     _resolve_inputs,
     validate_resolved_ensemble,
 )
+from itzi.ensemble.schema import TimeConfig
 from itzi.manifest import _manifest_document
 
 
diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py
index 7ea1994d..1b91291c 100644
--- a/tests/grass/test_ensemble.py
+++ b/tests/grass/test_ensemble.py
@@ -546,9 +546,9 @@ def test_dry_run_checks_every_generated_output_record(test_data_temp_path):
 @pytest.mark.forked
 @pytest.mark.usefixtures("grass_xy_session")
 def test_run_validation_accepts_unoccupied_drainage_id():
-    from itzi.grass.utils import get_current_mapset
     from itzi.ensemble.run_validation import _validate_grass_outputs
     from itzi.grass.interface import GrassInterface
+    from itzi.grass.utils import get_current_mapset
 
     GrassSessionManager.ensure_temporal_initialized()
     name = f"drainage_collision_{uuid4().hex[:8]}"
diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py
index 2c4adc28..cf58c99f 100644
--- a/tests/grass/test_grass_input.py
+++ b/tests/grass/test_grass_input.py
@@ -38,8 +38,8 @@ def test_resolved_input_kinds_share_raster_and_strds_construction(
 ) -> None:
     from itzi.grass import utils
     from itzi.grass.interface import GrassInterface
-    from itzi.providers.grass_input import GrassRasterInputProvider
     from itzi.grass.utils import MapData
+    from itzi.providers.grass_input import GrassRasterInputProvider
 
     monkeypatch.setattr(utils, "name_is_stds", lambda map_id: map_id == "series@test")
     monkeypatch.setattr(utils, "name_is_map", lambda map_id: map_id == "raster@test")
diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py
index 97b47e53..8fac527b 100644
--- a/tests/grass/test_temporal_overwrite.py
+++ b/tests/grass/test_temporal_overwrite.py
@@ -1,16 +1,16 @@
 from __future__ import annotations
 
-from configparser import ConfigParser
 import os
+from configparser import ConfigParser
 from uuid import uuid4
 
 import grass.script as gscript
 import pytest
+from itzi_core.const import TemporalType
 
 from itzi import SimulationRunner
 from itzi.configreader import ConfigReader
 from itzi.grass.session import GrassSessionManager
-from itzi_core.const import TemporalType
 
 
 def _build_runner(

From faf162703f43b0cda553ee362acb715e8f50bd69 Mon Sep 17 00:00:00 2001
From: Laurent Courty 
Date: Wed, 30 Sep 2026 19:50:57 -0600
Subject: [PATCH 85/85] fix test after removing write_raster_map_blocking

---
 tests/grass/test_grass_input.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py
index cf58c99f..092fcd51 100644
--- a/tests/grass/test_grass_input.py
+++ b/tests/grass/test_grass_input.py
@@ -65,15 +65,12 @@ def test_resolved_input_kinds_share_raster_and_strds_construction(
 def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> None:
     import grass.script as gscript
 
-    from itzi.grass.utils import (
-        output_name,
-        qualify_output_id,
-        write_raster_map_blocking,
-    )
+    from itzi.grass.interface import GrassInterface
+    from itzi.grass.utils import output_name, qualify_output_id
     from itzi.providers.grass_output import derived_record_name
 
     gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1)
-    gscript.mapcalc("visible_output=1")
+    gscript.mapcalc("visible_output=2")
     gscript.run_command("g.mapset", mapset="child", flags="c")
     output_id = qualify_output_id("visible_output", "child")
     assert output_id == "visible_output@child"
@@ -83,8 +80,14 @@ def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> No
         qualify_output_id("visible_output@PERMANENT", "child")
 
     gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1)
-    write_raster_map_blocking(
-        np.ones((3, 3), dtype=np.int32), "visible_output", "CELL", "ground_elevation", 0, False
+    start = datetime(2020, 1, 1)
+    with GrassInterface(start, start + timedelta(seconds=1), np.int32, None, None) as interface:
+        interface.write_raster_map(
+            np.ones((3, 3), dtype=np.int32), output_id, "ground_elevation", 0
+        )
+    assert gscript.parse_command("r.univar", map=output_id, flags="g")["min"] == "1"
+    assert (
+        gscript.parse_command("r.univar", map="visible_output@PERMANENT", flags="g")["min"] == "2"
     )