diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 30f6040f..0949fb9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: test: strategy: matrix: - python-version: ["3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] runs-on: ubuntu-latest container: osgeo/grass-gis:releasebranch_8_4-ubuntu @@ -31,11 +31,12 @@ jobs: - name: Pytest run: | - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-cli-results.xml --cov-report=xml --cov-report=html tests/cli tests/test_grass_session.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-itzi-results.xml --cov-report=xml --cov-report=html tests/grass/test_itzi.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-bmi-results.xml --cov-report=xml --cov-report=html tests/grass/test_bmi.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-tutorial-results.xml --cov-report=xml --cov-report=html tests/grass/test_tutorial.py - $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --junitxml=junit/test-temporal-overwrite-results.xml --cov-report=xml --cov-report=html tests/grass/test_temporal_overwrite.py + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --cov-report= --junitxml=junit/test-cli-results.xml tests/cli tests/test_grass_session.py + for test_file in tests/grass/test_*.py; do + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} pytest --cov=src --cov-append --cov-report= --junitxml="junit/$(basename "$test_file" .py)-results.xml" "$test_file" + done + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} coverage xml + $HOME/.local/bin/uv run --all-extras --python ${{ matrix.python-version }} coverage html - name: Upload coverage uses: actions/upload-artifact@v7 diff --git a/AGENTS.md b/AGENTS.md index f1865059..075b3d42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,15 +2,19 @@ ## Common commands - Run a single test: `uv run pytest tests/my_test.py` -- Due to a bug in GRASS, tests will fails if the mapset is changed between tests. Run each test file independently to prevent this. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. -- Enforce code formatting: `uvx ruff format .` +- Due to a bug in GRASS<8.5, a change of mapset cannot occur within the same process when using the temporal framework. Each test file must therefore be run independently. Tests that need to run in a separate process are marked with `@pytest.mark.forked`. +- Enforce code formatting: `uv run ruff format .` ## Code style -- Use python type hints. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. -- Do not quote class names in hints. Use `from __future__ import annotations` when necessary. -- Since the arguments types and return types are already documented by the hints, there's no need to duplicate this information in the docstrings. -- Apart from particular cases, use pydantic BaseModel instead of dataclass +- Use python type annotation. When a function that does not yet use hints is substantially edited, take the opportunity to add type hints. +- Do not quote class names in hints. Use `from __futue__ import annotations` when necessary. +- Do not use the `Any` type, and avoid the use of the general `object` type. Define the actual type instead. +- Since the arguments types and return types are already documented by the annotations, there's no need to duplicate this information in the docstrings. +- Use pydantic BaseModel when validation is needed, and dataclass for internal data exchange. +- No need to write tests to check if declarative Pydantic validation is working. - Place imports at the top of the file. Only break this rule to prevent heavy imports in a rarely used function (for example, CLI options). +- When passing data structures between functions, prefer immutable objects instead of equivalent mutable ones, unless immutability is desired. +- For GRASS interactions, prefer C functions calls through `pygrass` when available, instead of sub-process tool calls through `grass.script` ## General comments - The project uses `uv`. To run a command in the correct environment, use `uv run` diff --git a/docs/cli.rst b/docs/cli.rst index 5b891bb6..26c8e8a3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -11,6 +11,28 @@ Run a simulation :path: run :nodefault: +YAML ensembles +~~~~~~~~~~~~~~ + +Each YAML document is one ensemble. Use ``--dry-run`` (or ``-d``) to perform parsing, +expansion, GRASS input resolution, member-ID calculation, and artifact +validation without writing manifests or model outputs: + +.. code-block:: bash + + itzi run --dry-run studies.yaml + +Select resolved members with a qualified ensemble and simulation ID. The +unqualified ID form is accepted only when the batch contains one ensemble. + +.. code-block:: bash + + itzi run studies.yaml --member central-city#sim-c1f52e9a + +Stage 1 provides basic YAML ensemble execution. YAML hotstart resume mapping +and checkpoint provenance are introduced in Stage 2; the retained INI resume +syntax below continues to apply to legacy INI-only invocations. + Hotstart usage ~~~~~~~~~~~~~~ diff --git a/docs/conf_file.rst b/docs/conf_file.rst index 61e2210a..4454e241 100644 --- a/docs/conf_file.rst +++ b/docs/conf_file.rst @@ -1,6 +1,6 @@ -Configuration file -================== +INI configuration file +====================== The parameters of a simulation are given through a configuration file in a format similar to Microsoft Windows INI files. diff --git a/docs/index.rst b/docs/index.rst index bd5b0043..0575c016 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,5 +17,7 @@ Please see the Itzï's `home page `__. cli tutorial conf_file + yaml_file + manifest faq prog_manual diff --git a/docs/manifest.rst b/docs/manifest.rst new file mode 100644 index 00000000..c7e1ece4 --- /dev/null +++ b/docs/manifest.rst @@ -0,0 +1,102 @@ +Ensemble manifest +================= + +Each YAML ensemble run writes a YAML manifest describing its resolved members, +their planned outputs, and their run status. By default, it is saved +as ``results/.manifest.yaml`` beside the source YAML file. Set +``outputs.manifest.file`` in the ensemble configuration to choose another file: + +.. code-block:: yaml + + outputs: + manifest: + file: "results/{ensemble}.yaml" + +The manifest path may be absolute or relative to the source YAML file; only +``{ensemble}`` is allowed as a template placeholder (not ``{simulation}``). +An existing manifest requires ``itzi run -o`` to be replaced; if the manifest +destination fails validation, no manifest is written. A dry run validates the +destination but does not write a manifest. An ensemble with no selected resolved +members does not write one, unless it contains unresolved candidates in a run +without ``--member``. + +Format +------ + +The manifest is a YAML mapping with these top-level fields: + +* ``manifest_version``: manifest format version, currently ``1``. This is + distinct from the configuration's ``schema_version``. +* ``last_updated_at``: last write time as an ISO 8601 string with a local + timezone offset. It changes when the manifest is updated. +* ``ensemble``: ``id`` and ``description`` from the configuration; + ``description`` is ``null`` when omitted. +* ``source``: absolute ``path`` to the source YAML file and zero-based + ``document_index`` within that YAML stream. +* ``members``: resolved members in sweep expansion order, followed by + unresolved candidates in sweep expansion order. + +For example, a completed member might appear as follows (names and timings +are illustrative): + +.. code-block:: yaml + + manifest_version: 1 + last_updated_at: '2026-09-30T10:20:30+02:00' + ensemble: + id: central-city + description: Central city rainfall study + source: + path: /srv/studies/city.yaml + document_index: 0 + members: + - simulation_id: sim-1234abcd + selected: true + status: completed + elapsed_seconds: 12.5 + coordinates: + parameters.cfl: 0.5 + artifacts: + rasters: + water_depth: central_city_sim-1234abcd_water_depth@itzi + drainage: null + statistics: /srv/studies/results/sim-1234abcd.csv + +Member fields +------------- + +``simulation_id`` is the resolved member ID (``sim-...``), or ``null`` if +input resolution failed before an ID could be assigned. ``selected`` indicates +whether this member was selected to run. ``coordinates`` maps swept configuration +paths (for example, ``parameters.cfl`` or ``input.rainfall_rate``) to their +chosen values; it is ``{}`` without sweeps. An ``input.infiltration`` sweep +value is a mapping containing its ``type`` and applicable fields. + +Resolved members also have ``artifacts``: ``rasters`` maps requested output +variables to GRASS space-time raster dataset IDs (``name@mapset``), +``drainage`` is a GRASS space-time vector dataset ID or ``null``, and +``statistics`` is an absolute CSV file path or ``null``. ``rasters`` is ``{}`` +when no raster outputs are requested. These are intended destinations; a +failed or unfinished member may not have produced them. Unresolved candidates +have no ``artifacts`` field. + +``elapsed_seconds`` is ``null`` until execution finishes, then records the +elapsed wall-clock time in seconds, including when execution fails. The +``status`` field can be: + +* ``not_selected``: excluded by ``--member``. +* ``planned``: selected, but not yet started. +* ``running``: execution started; the manifest is updated before and after + each member runs. +* ``completed``: execution finished successfully. +* ``validation_failed``: the member could not be run. +* ``execution_failed``: execution raised an error. + +Failures also have a ``failure`` mapping with ``phase`` and ``detail``. +Possible phases are ``input_resolution`` (no resolved ID or artifacts), +``artifact_validation`` (conflicting or invalid outputs), +``run_validation`` (pre-run checks), and ``execution``. Unresolved candidates +are marked ``validation_failed`` when running without ``--member``; with +selectors they appear as ``not_selected`` but still carry their failure +detail. A failed YAML document (parse, schema, or expansion error) has no +manifest entry because no ensemble was expanded from it. diff --git a/docs/yaml_file.rst b/docs/yaml_file.rst new file mode 100644 index 00000000..c668a0be --- /dev/null +++ b/docs/yaml_file.rst @@ -0,0 +1,292 @@ +YAML configuration file +======================= + +A ``.yaml`` file is a stream of ensemble documents. +One single ``.yaml`` file can contain multiple documents separated with ``---``. +Each non-empty document defines one ensemble. +Multiple YAML files and :doc:`INI configurations ` can be passed to ``itzi run`` in +one command. +Other filename extensions are treated as INI. +YAML ensemble resume from hotstart is not currently supported. + +The file must be encoded in UTF-8. +The supported subset rejects empty documents, duplicate +keys, merge keys, YAML directives, custom tags, non-JSON values, and unknown +options at every level. +Put quotes around values that should be text, such as timestamps, durations, +map names, paths, and output templates. +Without quotes, YAML may read values like ``yes`` as a boolean or ``2026-09-01`` as a date instead of text. +Leave numeric parameters unquoted (for example, ``cfl: 0.7``). +An invalid document does not prevent later explicitly delimited documents from being checked. + +Example +------- + +.. code-block:: yaml + + schema_version: 1 + ensemble: + id: "central-city" + description: "Central city rainfall study" + grass: + database: "/srv/grassdata" + project: "central_city" + mapset: "itzi" + region: "central_city_5m" + time: + start: "2026-09-01T00:00:00" + duration: "02:00:00" + record_step: "00:05:00" + input: + ground_elevation: "elevation_5m@PERMANENT" + friction: "manning_n@PERMANENT" + rainfall_rate: ["rain_10yr@PERMANENT", "rain_100yr@PERMANENT"] + infiltration: + type: "none" + parameters: + cfl: [0.5, 0.7] + outputs: + rasters: + prefix: "central_city_{simulation}" + variables: [water_depth, flow_speed] + statistics: + file: "results/{simulation}.csv" + +This produces four candidate simulations (two rainfall maps × two ``cfl`` values). +Use ``itzi run --dry-run study.yaml`` to expand and validate members +without writing a manifest or model outputs. + +Document structure and sweeps +----------------------------- + +The top-level sections are ``schema_version``, ``ensemble``, ``grass``, +``time``, ``input``, ``parameters``, ``drainage``, and ``outputs``. All except +``drainage`` are required. +Section names and option names are case-sensitive. + +Each ``input`` map, ``parameters`` value, and ``drainage`` option can be a +scalar or a non-empty list of alternatives. +Lists are independent Cartesian dimensions, or sweep. +Duplicate values within a list are rejected. +``input.infiltration`` can instead be a non-empty list of whole model alternatives. +``grass``, ``time``, and output options are fixed across the ensemble. +``outputs.rasters.variables`` is a selection list. +At most 100 candidate simulations may come from one document and 200 from the whole command. +Ensemble IDs must be unique across a batch. + +``schema_version`` +------------------ + +Required integer, currently only ``1``. This is distinct from the +:doc:`manifest version `. + +``ensemble`` +------------ + +* ``id`` (required): 1–80 characters, starting with an ASCII letter or digit; + remaining characters may also be ``.``, ``_``, or ``-``. This identifies the + ensemble and names its default manifest. It must be unique within a batch. +* ``description`` (optional): text up to 256 characters, included in the + manifest. + +``grass`` +--------- + +Use ``grass: {}`` to use the active GRASS session, computational region, and mask. +An explicit session requires all three of the following options: + +* ``database``: path to the GIS database. +* ``project``: GRASS project (location) name. +* ``mapset``: mapset name. + +``executable`` optionally selects the GRASS executable for an explicit +session (defaults to ``grass``); it cannot be used without those three options. +``region`` optionally names a saved region; ``mask`` optionally names a raster mask. +Both can also be used to override the active region or mask for the run. +When a session is already active, an explicit database, project, and mapset must match it. +All members of an ensemble share this GRASS context and domain. +Only a projected location / project is accepted, not latitude/longitude. +The region must have at least 3 rows and 3 columns, and a mask cannot exclude the entire simulation domain. + +``time`` +-------- + +* ``record_step`` (required): interval between recorded outputs. +* ``duration``: length of the simulation; alone, it produces relative time. +* ``start``: ISO 8601 date-time with ``T`` between date and time; with + ``duration`` or ``end``, it produces absolute time. +* ``end``: ISO 8601 date-time after ``start``. + +Exactly one combination is allowed: ``duration`` alone, ``start`` with +``duration``, or ``start`` with ``end``. ``record_step`` and ``duration`` must +be positive ``HH:MM:SS`` strings (for example ``"26:00:00"``); hours may +exceed 23, minutes and seconds must be 0–59. ``end`` must follow ``start``. +For ``start``/``end``, either both timestamps carry timezone offsets or neither does. +Offsets are ignored by GRASS: execution uses the supplied wall-clock values without timezone conversion. +All members share the same time settings. + +``input`` +--------- + +These options name a GRASS raster map or space-time raster dataset (STRDS), possibly qualified as ``name@mapset``. +If the name resolves to both a raster and an STRDS, it is ambiguous and rejected. +Maps in another mapset must be on the GRASS search path. +``ground_elevation`` and ``friction`` must contain finite values in the active domain. +Each of the following accepts one string or a non-empty list of map-name strings: + +.. list-table:: + :header-rows: 1 + + * - Option + - Meaning (unit) + * - ``ground_elevation`` (required) + - Terrain elevation (m). + * - ``friction`` (required) + - Manning's roughness ``n`` (s m⁻¹ᐟ³). + * - ``water_depth`` + - Initial water depth (m). + * - ``water_surface_elevation`` + - Initial surface elevation, ground elevation plus depth (m). + * - ``rainfall_rate`` + - Rainfall (mm/h). + * - ``inflow`` + - Point inflow (m/s). + * - ``losses`` + - Loss rate (mm/h). + * - ``boundary_type`` + - Boundary condition type (integer map values). + * - ``boundary_value`` + - Boundary condition value (m). + +``water_depth`` and ``water_surface_elevation`` cannot both be specified. +Boundary types are 0/1 (closed), 2 (open), and 4 (specified water depth); +type 3 is not implemented. Open and closed types only apply at the +computational region border. See :doc:`conf_file` for detailed input and +boundary behavior, including caveats about time-varying input datasets. + +``input.infiltration`` is a single mapping (defaults to ``{type: none}``) or +a list of complete mappings, each with one of these shapes: + +.. code-block:: yaml + + infiltration: + - type: "none" + - type: "constant" + rate: "infiltration_rate@PERMANENT" + - type: "green-ampt" + effective_porosity: "porosity@PERMANENT" + capillary_pressure: "suction@PERMANENT" + hydraulic_conductivity: "conductivity@PERMANENT" + soil_water_content: "initial_moisture@PERMANENT" + +``constant.rate`` is an infiltration rate map (mm/h). +The Green-Ampt maps ``effective_porosity`` (fraction), +``capillary_pressure`` (mm), +and ``hydraulic_conductivity`` (mm/h) are required together. +``soil_water_content`` (fraction) is optional. +The model types are mutually exclusive, and fields inside an infiltration alternative cannot be sweeps. + +``parameters`` +-------------- + +All options are optional. +Each takes a finite YAML number (integers are accepted) or +a non-empty list of finite numbers. +Quoted numbers are not accepted. +Values must also meet these model limits: + +.. list-table:: + :header-rows: 1 + + * - Option + - Meaning + - Default + - Range + * - ``hmin`` + - Minimum water depth (m) + - 0.005 + - ≥ 0 + * - ``cfl`` + - Time-step coefficient + - 0.7 + - 0.01–1 + * - ``theta`` + - Inertia weighting + - 0.9 + - 0–1 + * - ``g`` + - Gravity (m/s²) + - 9.80665 + - ≥ 0 + * - ``dtmax`` + - Maximum surface-flow time-step (s) + - 5.0 + - > 0 + * - ``dtinf`` + - Infiltration/losses time-step (s) + - 60.0 + - > 0 + * - ``slope_threshold`` + - GMS slope threshold (m/m) + - 0.8 + - ≥ 0 + * - ``max_slope`` + - Maximum GMS slope (m/m) + - 0.8 + - ≥ ``slope_threshold`` + * - ``max_error`` + - Maximum relative volume error + - 0.05 + - > 0 + +``drainage`` +------------ + +Optional; enables SWMM coupling. +``swmm_input`` is required when the section is present. +Each supplied option can be a scalar or a non-empty sweep list: + +* ``swmm_input``: path to an EPA SWMM ``.inp`` file. + Relative paths are resolved from the YAML file's directory, not the current working directory. +* ``orifice_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.167). +* ``free_weir_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.54). +* ``submerged_weir_coeff``: flow-exchange coefficient, float from 0 to 1 (default 0.056). + +Omitted coefficients use the defaults. +SWMM input files must exist at resolution time. +``outputs.drainage`` is optional even when coupling is enabled +Without it, no drainage vector dataset is written. +See :doc:`conf_file` for the contents of drainage vector output. + +``outputs`` +----------- + +This section is required, but any of its four subsections may be omitted: + +* ``rasters``: requires ``prefix`` (GRASS name prefix template) and + ``variables`` (non-empty list of distinct output names). For each variable, + Itzi writes a STRDS named ``_``. Supported variables are + ``water_depth``, ``max_water_depth``, ``water_surface_elevation``, + ``flow_speed``, ``max_flow_speed``, ``flow_velocity_direction``, ``froude``, + ``flow_rate_x``, ``flow_rate_y``, ``mean_boundary_flow``, + ``mean_infiltration``, ``mean_rainfall``, ``mean_inflow``, ``mean_losses``, + ``mean_drainage_flow``, and ``created_volume``. See :doc:`conf_file` for + meanings, units, and output caveats. Legacy output aliases are not accepted. +* ``statistics``: requires ``file`` (non-empty CSV file path template); the + file is updated at each ``record_step``. See :doc:`conf_file` for its columns. +* ``drainage``: requires ``vector_dataset`` (GRASS vector dataset name + template); requires the top-level ``drainage`` section. Drainage vectors + form a space-time vector dataset (STVDS). +* ``manifest``: requires ``file`` (non-empty YAML path template). By default, + the manifest goes to ``results/.manifest.yaml`` relative to the + YAML source. See :doc:`manifest` for its contents. + +``prefix``, ``statistics.file``, and ``drainage.vector_dataset`` can use +``{ensemble}`` and ``{simulation}`` placeholders. ``manifest.file`` can use +only ``{ensemble}``. Formatting specifications and conversions are unsupported; +use ``{{`` and ``}}`` for literal braces. Filesystem output paths may be +absolute or relative to the YAML source file; ``~`` is expanded. GRASS output +names must be unqualified, valid names in the execution mapset (not ``MASK``). +Derived record names must also be valid. Colliding output names, existing +outputs without ``-o``, and file outputs that would overwrite the YAML source, +SWMM input, or each other are rejected during validation. diff --git a/pyproject.toml b/pyproject.toml index 66c7e8bf..1dc0d865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,8 @@ dependencies = [ "numpy>=2.5", "itzi-core==0.8.0", "bmipy>=2.0.1", - "pydantic>=2.10" + "pydantic>=2.10", + "pyyaml>=6.0" ] # min 3.12 for typing. requires-python = ">=3.12" @@ -42,7 +43,7 @@ dev = [ "pandas>=3.0.3", "pre-commit>=4.6", "ruff>=0.15", - "ty>=0.0.55", + "ty>=0.0.84", "sphinx>=9.1.0", "sphinx-argparse>=0.5.2", "sphinx-rtd-theme>=3.1.0", @@ -57,8 +58,7 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -"itzi.data" = ["*.ini"] -"itzi.data.colortable" = ["*.txt"] +"itzi.grass" = ["colortable/*.txt"] [tool.pytest.ini_options] pythonpath = ["src", "."] diff --git a/src/itzi/__init__.py b/src/itzi/__init__.py index 34598c83..5b349d75 100644 --- a/src/itzi/__init__.py +++ b/src/itzi/__init__.py @@ -1,2 +1,2 @@ -from itzi.itzi import SimulationRunner as SimulationRunner from itzi.bmi_itzi import BmiItzi as BmiItzi +from itzi.itzi import SimulationRunner as SimulationRunner diff --git a/src/itzi/bmi_itzi.py b/src/itzi/bmi_itzi.py index 4e3e973a..2059972c 100644 --- a/src/itzi/bmi_itzi.py +++ b/src/itzi/bmi_itzi.py @@ -15,14 +15,15 @@ """ from datetime import timedelta +from types import MappingProxyType import numpy as np from bmipy import Bmi +from itzi_core import ARRAY_DEFINITIONS, ArrayCategory -from itzi import SimulationRunner from itzi.configreader import ConfigReader -from itzi.grass_session import GrassSessionManager -from itzi_core.array_definitions import ARRAY_DEFINITIONS, ArrayCategory +from itzi.grass.session import GrassSessionManager +from itzi.simulation_runner import SimulationRunner class BmiItzi(Bmi): @@ -30,29 +31,38 @@ class BmiItzi(Bmi): _name = "Itzï" # Mappings of CSDMS name: internal key for input and output arrays - _input_var_names = { - arr_def.csdms_name: arr_def.key - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in arr_def.category - } - _output_var_names = { - arr_def.csdms_name: arr_def.key - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.OUTPUT in arr_def.category - } - + _input_var_names = MappingProxyType( + { + arr_def.csdms_name: arr_def.key + for arr_def in ARRAY_DEFINITIONS + if ArrayCategory.INPUT in arr_def.category + } + ) + _output_var_names = MappingProxyType( + { + arr_def.csdms_name: arr_def.key + for arr_def in ARRAY_DEFINITIONS + if ArrayCategory.OUTPUT in arr_def.category + } + ) # A list of array definition for both input and output, without overlaps - input_output_array_definition = [ + input_output_array_definition = tuple( arr_def for arr_def in ARRAY_DEFINITIONS if any(cat in arr_def.category for cat in [ArrayCategory.OUTPUT, ArrayCategory.INPUT]) - ] + ) # Mapping of csdms_name: internal key for both input and output - _var_names = {arr_def.csdms_name: arr_def.key for arr_def in input_output_array_definition} + _var_names = MappingProxyType( + {arr_def.csdms_name: arr_def.key for arr_def in input_output_array_definition} + ) # Mapping of csdms_name: unit - _var_units = {arr_def.csdms_name: arr_def.unit for arr_def in input_output_array_definition} + _var_units = MappingProxyType( + {arr_def.csdms_name: arr_def.unit for arr_def in input_output_array_definition} + ) # Mapping of csdms_name: var_loc - _var_loc = {arr_def.csdms_name: arr_def.var_loc for arr_def in input_output_array_definition} + _var_loc = MappingProxyType( + {arr_def.csdms_name: arr_def.var_loc for arr_def in input_output_array_definition} + ) def __init__(self): """Create a BmiItzi model that is ready for initialization.""" @@ -61,7 +71,7 @@ def __init__(self): # Model control functions # - def initialize(self, filename=None) -> None: + def initialize(self, config_file=None) -> None: """Initialize the Itzï model. Parameters @@ -69,17 +79,17 @@ def initialize(self, filename=None) -> None: filename : str, optional Path to name of input file. """ - conf_data = ConfigReader(filename) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + conf_data = ConfigReader(config_file) + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params self.grass_session_manager = GrassSessionManager(grass_params) self.grass_session_manager.open() self.itzi = SimulationRunner( sim_params, grass_params, - stats_file=conf_data.get_stats_file(), - ) + stats_file=conf_data.stats_file, + ).initialize() def update(self): """Advance model by one time step.""" diff --git a/src/itzi/cli_parser.py b/src/itzi/cli_parser.py index 1b42dd67..e384d093 100644 --- a/src/itzi/cli_parser.py +++ b/src/itzi/cli_parser.py @@ -4,7 +4,6 @@ import argparse - DESCR = "A dynamic, fully distributed hydraulic and hydrologic model." @@ -25,11 +24,11 @@ def build_parser() -> argparse.ArgumentParser: subparsers = arg_parser.add_subparsers(dest="command", required=True) # run a simulation - run_parser = subparsers.add_parser("run", help="Run a simulation.") + run_parser = subparsers.add_parser("run", help="Run a configuration file or YAML stream.") run_parser.add_argument( "config_file", nargs="+", - help=("An Itzï configuration file (if several given, run in batch mode.)"), + help="An Itzï configuration file. ini and yaml are allowed, several files form one batch.", ) run_parser.add_argument("-o", action="store_true", help="Overwrite files if exist.") verbosity_parser = run_parser.add_mutually_exclusive_group() @@ -44,9 +43,23 @@ def build_parser() -> argparse.ArgumentParser: help=( "Resume a simulation from a hotstart file. " "If only the path to a hotstart file is given, batch is not allowed. " - "For batch processing, use the 'CONFIG_PATH=HOTSTART_PATH' construct." + "For batch processing, use CONFIG_PATH=HOTSTART_PATH." ), ) + run_parser.add_argument( + "--member", + action="append", + default=[], + metavar="[ENSEMBLE#]SIMULATION", + help="Run only one ensemble member. Repeat to select multiple members.", + ) + run_parser.add_argument( + "-d", + "--dry-run", + dest="dry", + action="store_true", + help="Resolve and validate selected members without writing user artifacts.", + ) # display version subparsers.add_parser("version", help="Display software version number.") diff --git a/src/itzi/configreader.py b/src/itzi/configreader.py index b4e367b9..8a127dd9 100644 --- a/src/itzi/configreader.py +++ b/src/itzi/configreader.py @@ -14,25 +14,25 @@ from __future__ import annotations -from collections.abc import Callable from configparser import ConfigParser from datetime import datetime, timedelta from pathlib import Path -from typing import Any, NoReturn +from typing import NoReturn from itzi_core import ( - ARRAY_DEFINITIONS, - ArrayCategory, + INPUT_ARRAY_KEYS, + OUTPUT_ARRAY_KEYS, HotstartRunConfig, InfiltrationModelType, SimulationConfig, SurfaceFlowParameters, TemporalType, ) +from itzi_core.const import DefaultValues from pydantic import BaseModel, ConfigDict, ValidationError import itzi.messenger as msgr -from itzi.grass_session import GrassParams +from itzi.grass.session import GrassParams DEPRECATED_INPUT_ALIASES: list[tuple[str, str]] = [ # (old, new) @@ -74,24 +74,13 @@ ) TIME_OPTION_KEYS = ("start_time", "end_time", "duration", "record_step") -HOTSTART_OPTION_KEYS = ("wallclock_step", "save_file") GREEN_AMPT_KEYS = ( "effective_porosity", "capillary_pressure", "hydraulic_conductivity", ) -GRASS_MANDATORY_KEYS = ("grassdata", "location", "mapset") -GRASS_OPTION_KEYS = (*GRASS_MANDATORY_KEYS, "region", "mask", "grass_bin") -INPUT_MAP_KEYS = tuple( - arr_def.key for arr_def in ARRAY_DEFINITIONS if ArrayCategory.INPUT in arr_def.category -) -OUTPUT_MAP_KEYS = tuple( - arr_def.key for arr_def in ARRAY_DEFINITIONS if ArrayCategory.OUTPUT in arr_def.category -) +GRASS_OPTION_KEYS = ("grassdata", "location", "mapset", "region", "mask", "grass_bin") SURFACE_FLOW_OPTION_KEYS = tuple(SurfaceFlowParameters.model_fields) -SIMULATION_OPTION_KEYS = ("dtinf",) -DRAINAGE_STRING_KEYS = ("swmm_inp", "output") -DRAINAGE_FLOAT_KEYS = ("orifice_coeff", "free_weir_coeff", "submerged_weir_coeff") def _read_parser(filename: str) -> ConfigParser: @@ -102,25 +91,13 @@ def _read_parser(filename: str) -> ConfigParser: return params -def _read_optional_value( - params: ConfigParser, - section: str, - option: str, - reader: Callable[[str, str], Any], -) -> Any | None: - """Read one config value when the option exists.""" - if not params.has_option(section, option): - return None - return reader(section, option) - - def _read_string_options( params: ConfigParser, section: str, option_names: tuple[str, ...] ) -> dict[str, str]: """Collect the string-valued options present in a section.""" values: dict[str, str] = {} for option_name in option_names: - value = _read_optional_value(params, section, option_name, params.get) + value = params.get(section, option_name, fallback=None) if value is not None: values[option_name] = value return values @@ -132,7 +109,7 @@ def _read_float_options( """Collect the float-valued options present in a section.""" values: dict[str, float] = {} for option_name in option_names: - value = _read_optional_value(params, section, option_name, params.getfloat) + value = params.getfloat(section, option_name, fallback=None) if value is not None: values[option_name] = value return values @@ -169,28 +146,28 @@ def _read_hotstart_values(params: ConfigParser) -> HotstartRunConfig | None: return None hotstart_values = { - "wallclock_step": _read_optional_value(params, "hotstart", "wallclock_step", params.get), - "save_file_name": _read_optional_value(params, "hotstart", "save_file", params.get), + "wallclock_step": params.get("hotstart", "wallclock_step", fallback=None), + "save_file_name": params.get("hotstart", "save_file", fallback=None), } if all(value is None for value in hotstart_values.values()): return None try: - return HotstartRunConfig(**hotstart_values) + return HotstartRunConfig.model_validate(hotstart_values) except ValidationError as error: _fatal_validation_error(error) def _read_input_map_names(params: ConfigParser) -> dict[str, str | None]: """Read input map names and normalize deprecated aliases.""" - map_names: dict[str, str | None] = dict.fromkeys(INPUT_MAP_KEYS) + map_names: dict[str, str | None] = dict.fromkeys(INPUT_ARRAY_KEYS) for old_input_name, new_input_name in DEPRECATED_INPUT_ALIASES: if params.has_option("input", old_input_name): _warn_about_deprecated_alias("Input", old_input_name, new_input_name) map_names[new_input_name] = params.get("input", old_input_name) - for input_name in INPUT_MAP_KEYS: + for input_name in INPUT_ARRAY_KEYS: if params.has_option("input", input_name): map_names[input_name] = params.get("input", input_name) @@ -218,7 +195,7 @@ def _normalize_output_values(raw_values: str | None) -> list[str]: def _generate_output_map_names(prefix: str, output_values: list[str]) -> dict[str, str | None]: """Build the output map dictionary from the selected outputs.""" - output_map_names: dict[str, str | None] = dict.fromkeys(OUTPUT_MAP_KEYS) + output_map_names: dict[str, str | None] = dict.fromkeys(OUTPUT_ARRAY_KEYS) for value in output_values: if value in output_map_names: output_map_names[value] = f"{prefix}_{value}" @@ -227,12 +204,10 @@ def _generate_output_map_names(prefix: str, output_values: list[str]) -> dict[st def _read_output_config(params: ConfigParser) -> tuple[str, list[str], dict[str, str | None]]: """Read output settings and derive output map names.""" - prefix = _read_optional_value(params, "output", "prefix", params.get) + prefix = params.get("output", "prefix", fallback=None) if prefix is None: prefix = f"itzi_results_{datetime.now().strftime('%Y%m%dT%H%M%S')}" - output_values = _normalize_output_values( - _read_optional_value(params, "output", "values", params.get) - ) + output_values = _normalize_output_values(params.get("output", "values", fallback=None)) output_map_names = _generate_output_map_names(prefix, output_values) return prefix, output_values, output_map_names @@ -246,28 +221,6 @@ def _read_surface_flow_parameters(params: ConfigParser) -> SurfaceFlowParameters _fatal_validation_error(error) -def _read_simulation_option_values(params: ConfigParser) -> dict[str, float]: - """Read simulation options that live outside the surface flow model.""" - return _read_float_options(params, "options", SIMULATION_OPTION_KEYS) - - -def _read_simulation_drainage_values(params: ConfigParser) -> dict[str, str | float]: - """Read drainage settings using SimulationConfig field names.""" - drainage_values: dict[str, str | float] = {} - - for option_name in DRAINAGE_STRING_KEYS: - value = _read_optional_value(params, "drainage", option_name, params.get) - if value is None: - continue - if option_name == "output": - drainage_values["drainage_output"] = value - else: - drainage_values[option_name] = value - - drainage_values.update(_read_float_options(params, "drainage", DRAINAGE_FLOAT_KEYS)) - return drainage_values - - def _resolve_swmm_input_path(swmm_inp: str, config_file: str) -> Path: """Resolve a SWMM input path, searching cwd first and then the config directory.""" configured_path = Path(swmm_inp) @@ -294,15 +247,11 @@ def _resolve_swmm_input_path(swmm_inp: str, config_file: str) -> Path: def _read_grass_params(params: ConfigParser) -> GrassParams: """Build GRASS session parameters from the config file.""" - return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) - - -def _build_simulation_config(**kwargs: Any) -> SimulationConfig: - """Build a validated simulation config from normalized values.""" try: - return SimulationConfig(**kwargs) - except ValidationError as error: - _fatal_validation_error(error) + return GrassParams(**_read_string_options(params, "grass", GRASS_OPTION_KEYS)) + except ValueError as error: + msgr.fatal(str(error)) + raise AssertionError("unreachable") from error class SimulationTimes(BaseModel): @@ -312,7 +261,6 @@ class SimulationTimes(BaseModel): start: datetime end: datetime - duration: timedelta record_step: timedelta | None temporal_type: TemporalType @@ -332,13 +280,10 @@ def from_raw_values(cls, raw_values: dict[str, str | None]) -> SimulationTimes: end: datetime = start + duration if start >= end: msgr.fatal("Simulation duration must be positive") - if duration is None: - duration: timedelta = end - start return cls( start=start, end=end, - duration=duration, record_step=record_step, temporal_type=temporal_type, ) @@ -404,65 +349,68 @@ def __init__(self, filename: str | None) -> None: msgr.fatal("Not a valid configuration file") self.config_file = filename - self.ga_list = list(GREEN_AMPT_KEYS) - self.grass_mandatory = list(GRASS_MANDATORY_KEYS) params = _read_parser(filename) self.hotstart_config = _read_hotstart_values(params) - self.raw_input_times = _read_time_values(params) + raw_time_values = _read_time_values(params) self.input_map_names = _read_input_map_names(params) self.out_prefix, self.out_values, self.output_map_names = _read_output_config(params) - self.stats_file = ( - _read_optional_value(params, "statistics", "stats_file", params.get) or None - ) - self.sim_times = SimulationTimes.from_raw_values(self.raw_input_times) + self.stats_file = params.get("statistics", "stats_file", fallback=None) or None + self.sim_times = SimulationTimes.from_raw_values(raw_time_values) self._check_general_input(self.input_map_names) infiltration_model = self._resolve_infiltration_model(self.input_map_names) self.grass_params = _read_grass_params(params) - self._check_grass_params(self.grass_params) surface_flow_parameters = _read_surface_flow_parameters(params) assert self.sim_times.record_step is not None - simulation_kwargs: dict[str, Any] = { - "start_time": self.sim_times.start, - "end_time": self.sim_times.end, - "record_step": self.sim_times.record_step, - "temporal_type": self.sim_times.temporal_type, - "input_map_names": self.input_map_names, - "output_map_names": self.output_map_names, - "surface_flow_parameters": surface_flow_parameters, - "infiltration_model": infiltration_model, - } - - if self.hotstart_config is not None: - simulation_kwargs["hotstart_config"] = self.hotstart_config - - simulation_kwargs.update(_read_simulation_option_values(params)) - simulation_kwargs.update(_read_simulation_drainage_values(params)) - if "swmm_inp" in simulation_kwargs: - simulation_kwargs["swmm_inp"] = _resolve_swmm_input_path( - simulation_kwargs["swmm_inp"], self.config_file - ) - - self.sim_config = _build_simulation_config(**simulation_kwargs) + swmm_inp = params.get("drainage", "swmm_inp", fallback=None) + if swmm_inp is not None: + resolved_swmm_inp = _resolve_swmm_input_path(swmm_inp, self.config_file) + else: + resolved_swmm_inp = None - def _check_grass_params(self, grass_params: GrassParams) -> None: - """Ensure mandatory GRASS settings are provided together.""" - grass_values = grass_params.model_dump() - grass_any = any(grass_values[key] for key in self.grass_mandatory) - grass_all = all(grass_values[key] for key in self.grass_mandatory) - if grass_any and not grass_all: - msgr.fatal(f"{self.grass_mandatory} are mutualy inclusive") + try: + self.sim_config = SimulationConfig( + start_time=self.sim_times.start, + end_time=self.sim_times.end, + record_step=self.sim_times.record_step, + temporal_type=self.sim_times.temporal_type, + hotstart_config=self.hotstart_config, + input_map_names={ + key: value for key, value in self.input_map_names.items() if value is not None + }, + output_map_names={ + key: value for key, value in self.output_map_names.items() if value is not None + }, + surface_flow_parameters=surface_flow_parameters, + dtinf=params.getfloat("options", "dtinf", fallback=DefaultValues.DTINF), + infiltration_model=infiltration_model, + swmm_inp=resolved_swmm_inp, + drainage_output=params.get("drainage", "output", fallback=None), + orifice_coeff=params.getfloat( + "drainage", "orifice_coeff", fallback=DefaultValues.ORIFICE_COEFF + ), + free_weir_coeff=params.getfloat( + "drainage", "free_weir_coeff", fallback=DefaultValues.FREE_WEIR_COEFF + ), + submerged_weir_coeff=params.getfloat( + "drainage", + "submerged_weir_coeff", + fallback=DefaultValues.SUBMERGED_WEIR_COEFF, + ), + ) + except ValidationError as error: + _fatal_validation_error(error) def _resolve_infiltration_model( self, input_map_names: dict[str, str | None] ) -> InfiltrationModelType: """Infer the infiltration model from the configured input maps.""" infiltration_input = input_map_names["infiltration"] - ga_any = any(input_map_names[key] for key in self.ga_list) - ga_all = all(input_map_names[key] for key in self.ga_list) + ga_any = any(input_map_names[key] for key in GREEN_AMPT_KEYS) + ga_all = all(input_map_names[key] for key in GREEN_AMPT_KEYS) if not infiltration_input and not ga_any: return InfiltrationModelType.NULL @@ -471,7 +419,7 @@ def _resolve_infiltration_model( if infiltration_input and ga_any: msgr.fatal("Infiltration model incompatible with user-defined rate") if ga_any and not ga_all: - msgr.fatal(f"{self.ga_list} are mutualy inclusive") + msgr.fatal(f"{GREEN_AMPT_KEYS} are mutualy inclusive") return InfiltrationModelType.GREEN_AMPT def _check_general_input(self, input_map_names: dict[str, str | None]) -> None: @@ -488,15 +436,3 @@ def _check_general_input(self, input_map_names: dict[str, str | None]) -> None: msgr.fatal( "inputs and are mutually exclusive." ) - - def get_sim_params(self) -> SimulationConfig: - """Return validated simulation parameters.""" - return self.sim_config - - def get_grass_params(self) -> GrassParams: - """Return validated GRASS GIS session parameters.""" - return self.grass_params - - def get_stats_file(self) -> str | None: - """Return the CSV statistics output file name, if configured.""" - return self.stats_file diff --git a/src/itzi/data/colortable/default.txt b/src/itzi/data/colortable/default.txt deleted file mode 100644 index cdd70fb9..00000000 --- a/src/itzi/data/colortable/default.txt +++ /dev/null @@ -1,11 +0,0 @@ -0% black --10 blue --1 aqua --.1 cyan -0 white -.1 yellow -1 orange -10 red -100% magenta -nv white -default white diff --git a/src/itzi/data/example.ini b/src/itzi/data/example.ini deleted file mode 100644 index 52754f68..00000000 --- a/src/itzi/data/example.ini +++ /dev/null @@ -1,97 +0,0 @@ -[time] -# Start of the simulation. Format yyyy-mm-dd HH:MM -start_time = -# End of the simulation. Format yyyy-mm-dd HH:MM -end_time = -# Duration of the simulation. Format HH:MM:SS -duration = -# Duration between two records. Format HH:MM:SS -record_step = - -[hotstart] -# Wallclock duration between records. Format HH:MM:SS -wallclock_step = 00:15:00 -# File name for the hotstart file -save_file = - -[input] -# ground elevation (raster map/stds) -ground_elevation = -# friction coefficient (raster map/stds) -friction = -# starting water depth (raster map/stds) -water_depth = -# starting water surface elevation (raster map/stds) -water_surface_elevation = - -# rainfall in mm/h (raster map/stds) -rainfall_rate = -# Input user flow in m/s (raster map/stds) -inflow = -# Input boundary conditions type (raster map/stds) -boundary_type = -# Input boundary conditions values (raster map/stds) -boundary_value = - -# Fixed input infiltration rate in mm/h (raster map/stds) -infiltration = -# Effective porosity in mm/mm (raster map/stds) -effective_porosity = -# Wetting front capillary pressure head in mm(raster map/stds) -capillary_pressure = -# Hydraulic conductivity in mm/h (raster map/stds) -hydraulic_conductivity = -# Relative soil water content in mm (raster map/stds) -soil_water_content = - -# user-input losses in mm/h (raster map/strds) -losses = - -[output] -# prefix of output strds -prefix = -# strds to be saved, separated by comma -values = water_depth, max_water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, froude, flow_rate_x, flow_rate_y, created_volume, mean_boundary_flow, mean_infiltration, mean_rainfall, mean_inflow, mean_losses, mean_drainage_flow - -[statistics] -# statistic file -stats_file = - -[options] -# Water depth threshold in metres -hmin = 0.005 -# Slope threshold in m/m -slope_threshold = 0.8 -# Max slope in m/m -max_slope = 0.8 -# CFL coefficient used to calculate time-step -cfl = 0.7 -# Inertia weighting coefficient -theta = 0.9 -# Maximum surface flow time-step in seconds -dtmax = 5 -# Infiltration time-step in seconds -dtinf = 60 -# Maximum created-volume ratio. -max_error = 0.05 - -[drainage] -swmm_inp = -output = -orifice_coeff = 0.167 -free_weir_coeff = 0.54 -submerged_weir_coeff = 0.056 - -[grass] -# Path to the grass binary -grass_bin = -# Path to the GIS DataBase -grassdata = -# name of the location -location = -# name of the mapset -mapset = -# name of the region definition -region = -# name of raster map to be used as a mask -mask = diff --git a/src/itzi/data/tutorial.ini b/src/itzi/data/tutorial.ini deleted file mode 100644 index f3ae8c85..00000000 --- a/src/itzi/data/tutorial.ini +++ /dev/null @@ -1,20 +0,0 @@ -[time] -duration = 02:00:00 -record_step = 00:05:00 - -[input] -ground_elevation = elev_lid792_5m@PERMANENT -friction = n@PERMANENT -rainfall_rate = rain@PERMANENT -boundary_type = bctype@PERMANENT -boundary_value = bcvalue@PERMANENT - -[output] -prefix = nc_itzi_tutorial -values = water_depth, max_water_depth, water_surface_elevation, flow_speed, flow_velocity_direction, mean_boundary_flow - -[statistics] -stats_file = nc_itzi_tutorial.csv - -[options] -cfl = 0.5 diff --git a/src/itzi/ensemble/__init__.py b/src/itzi/ensemble/__init__.py new file mode 100644 index 00000000..9774eeb3 --- /dev/null +++ b/src/itzi/ensemble/__init__.py @@ -0,0 +1,3 @@ +"""Configuration and execution plans for YAML ensembles.""" + +from itzi.ensemble.yaml import load_yaml_stream as load_yaml_stream diff --git a/src/itzi/ensemble/models.py b/src/itzi/ensemble/models.py new file mode 100644 index 00000000..dfca6e49 --- /dev/null +++ b/src/itzi/ensemble/models.py @@ -0,0 +1,285 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import string +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path +from typing import Literal + +from itzi_core import ( + INPUT_ARRAY_KEYS, + DomainData, + InfiltrationModelType, + SimulationConfig, + TemporalType, +) +from pydantic import BaseModel, ConfigDict, StrictStr, model_validator + +from itzi.grass.session import GrassParams + +GREEN_AMPT_KEYS = frozenset( + { + "effective_porosity", + "capillary_pressure", + "hydraulic_conductivity", + "soil_water_content", + } +) +assert GREEN_AMPT_KEYS <= INPUT_ARRAY_KEYS + +DIRECT_INPUT_KEYS = tuple( + key for key in INPUT_ARRAY_KEYS if key not in {"infiltration", *GREEN_AMPT_KEYS} +) +MAX_ENSEMBLE_MEMBERS = 100 +MAX_BATCH_MEMBERS = 200 + +type JsonValue = ( + None | bool | int | float | str | tuple[JsonValue, ...] | tuple[tuple[str, JsonValue], ...] +) + + +class EnsembleError(ValueError): + """Configuration error associated with a single YAML document.""" + + +@dataclass(frozen=True) +class SourceDocument: + """Stable source metadata for one document in a YAML stream.""" + + path: Path + document_index: int + + +@dataclass(frozen=True) +class DocumentFailure: + """A document-local error that does not prevent later documents loading.""" + + path: Path + document_index: int + line: int | None + column: int | None + phase: Literal["parse", "schema", "expansion"] + detail: str + + def format(self) -> str: + location = f"{self.path} document {self.document_index}" + if self.line is not None and self.column is not None: + location += f" line {self.line}, column {self.column}" + return f"{location}: {self.phase}: {self.detail}" + + +class StrictModel(BaseModel): + """Base model for the public YAML schema.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class GrassConfig(StrictModel): + database: StrictStr | None = None + project: StrictStr | None = None + mapset: StrictStr | None = None + executable: StrictStr | None = None + region: StrictStr | None = None + mask: StrictStr | None = None + + @model_validator(mode="after") + def validate_context(self) -> GrassConfig: + GrassParams(grassdata=self.database, location=self.project, mapset=self.mapset) + if self.executable is not None and self.database is None: + raise ValueError("executable requires database, project, and mapset") + return self + + +@dataclass(frozen=True) +class NormalizedTime: + """Time semantics retained until the final core configuration boundary.""" + + temporal_type: TemporalType + start: datetime | None + end: datetime | None + duration: timedelta + record_step: timedelta + had_timezone_offset: bool + + +@dataclass(frozen=True) +class NormalizedInfiltration: + model: InfiltrationModelType + input_maps: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class OutputTemplates: + raster_prefix: str | None + raster_variables: tuple[str, ...] + statistics_file: str | None + drainage_dataset: str | None + + +@dataclass(frozen=True) +class ExpandedSimulation: + """One fully scalar pre-GRASS application configuration.""" + + source: SourceDocument + ensemble_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + grass: GrassConfig + time: NormalizedTime + input_maps: tuple[tuple[str, str], ...] + infiltration: NormalizedInfiltration + parameters: tuple[tuple[str, float], ...] + drainage: tuple[tuple[str, str | float], ...] | None + outputs: OutputTemplates + + +@dataclass(frozen=True) +class ExpandedEnsemble: + """One YAML document after Cartesian expansion.""" + + source: SourceDocument + ensemble_id: str + ensemble_description: str | None + manifest_template: str | None + simulations: tuple[ExpandedSimulation, ...] + + +@dataclass(frozen=True) +class EffectiveMask: + """The immutable mask source selected during GRASS resolution.""" + + mode: Literal["explicit", "active", "none"] + source: str | None + + +@dataclass(frozen=True) +class ArtifactSummary: + """Rendered, application-owned output destinations for one member.""" + + output_map_names: tuple[tuple[str, str], ...] + drainage_output: str | None + statistics_file: Path | None + + +@dataclass(frozen=True) +class ResolvedSimulation: + """Spawn-serializable execution payload with one final core configuration.""" + + simulation_id: str + coordinates: tuple[tuple[str, JsonValue], ...] + grass_params: GrassParams + domain_data: DomainData + effective_mask: EffectiveMask + input_kinds: tuple[tuple[str, Literal["raster", "strds"]], ...] + simulation_config: SimulationConfig + artifacts: ArtifactSummary + normalized_payload: str = "" + + +@dataclass(frozen=True) +class ValidationFailure: + """A coordinate that could not become an executable resolved simulation.""" + + coordinates: tuple[tuple[str, JsonValue], ...] + phase: str + detail: str + + +@dataclass(frozen=True) +class ResolvedEnsemble: + """One expanded ensemble with its resolved members and validation results.""" + + ensemble: ExpandedEnsemble + simulations: tuple[ResolvedSimulation, ...] + failures: tuple[ValidationFailure, ...] + artifact_failure: str | None = None + + +@dataclass(frozen=True) +class LoadedYamlStream: + """Valid ensembles and document-local failures from one YAML file.""" + + ensembles: tuple[ExpandedEnsemble, ...] + failures: tuple[DocumentFailure, ...] + + +def parse_duration(value: str) -> timedelta: + """Parse a strictly positive ``HH:MM:SS`` duration.""" + try: + hours_str, minutes_str, seconds_str = value.split(":") + hours = int(hours_str) + minutes = int(minutes_str) + seconds = int(seconds_str) + except ValueError as error: + raise ValueError("must use HH:MM:SS") from error + if hours < 0 or not 0 <= minutes <= 59 or not 0 <= seconds <= 59: + raise ValueError("must use HH:MM:SS") + duration = timedelta(hours=hours, minutes=minutes, seconds=seconds) + if duration <= timedelta(): + raise ValueError("must be strictly positive") + return duration + + +def format_duration(value: timedelta) -> str: + """Render a positive duration for canonical internal data.""" + if value <= timedelta(): + raise ValueError("duration must be strictly positive") + total_seconds = value.days * 86400 + value.seconds + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + rendered = f"{hours:02}:{minutes:02}:{seconds:02}" + if value.microseconds: + return f"{rendered}.{value.microseconds:06}".rstrip("0") + return rendered + + +def render_template(template: str, *, ensemble: str, simulation: str | None) -> str: + """Render the intentionally small output-template language.""" + formatter = string.Formatter() + values = {"ensemble": ensemble, "simulation": simulation} + try: + for _, field_name, format_spec, conversion in formatter.parse(template): + if field_name is None: + continue + if field_name not in values: + raise EnsembleError(f"unsupported template placeholder {{{field_name}}}") + if values[field_name] is None: + raise EnsembleError(f"{{{field_name}}} is not permitted in this template") + if format_spec or conversion: + raise EnsembleError( + "template format specifications and conversions are not supported" + ) + return template.format_map(values) + except ValueError as error: + raise EnsembleError(f"invalid output template: {error}") from error + + +def check_batch_limits(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a valid batch that is too large before any resolver is started.""" + count = sum(len(ensemble.simulations) for ensemble in ensembles) + if count > MAX_BATCH_MEMBERS: + raise EnsembleError( + f"batch expands to {count} simulations, exceeding the {MAX_BATCH_MEMBERS} limit" + ) + + +def check_unique_ensemble_ids(ensembles: tuple[ExpandedEnsemble, ...]) -> None: + """Reject a batch-global duplicate ensemble identifier.""" + seen: set[str] = set() + for ensemble in ensembles: + if ensemble.ensemble_id in seen: + raise EnsembleError(f"duplicate ensemble ID {ensemble.ensemble_id!r}") + seen.add(ensemble.ensemble_id) diff --git a/src/itzi/ensemble/resolution.py b/src/itzi/ensemble/resolution.py new file mode 100644 index 00000000..355514ef --- /dev/null +++ b/src/itzi/ensemble/resolution.py @@ -0,0 +1,449 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable +from datetime import datetime, timedelta +from pathlib import Path +from typing import Literal + +from itzi_core import DomainData, SimulationConfig, SurfaceFlowParameters +from pydantic import ValidationError + +from itzi.ensemble.models import ( + ArtifactSummary, + EffectiveMask, + EnsembleError, + ExpandedSimulation, + ResolvedSimulation, + ValidationFailure, + format_duration, + render_template, +) +from itzi.grass.names import derived_drainage_table_names, derived_record_name +from itzi.grass.session import GrassParams, GrassSessionManager + + +def resolve_ensemble( + simulations: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: + """Resolve one ensemble in one GRASS session, retaining member-local failures.""" + if not simulations: + return () + requested_params = _requested_grass_params(simulations[0]) + + with GrassSessionManager(requested_params): + from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask + + actual_params = active_grass_params(requested_params) + domain = read_domain(simulations[0].grass.region) + effective_mask = resolve_effective_mask(simulations[0].grass.mask) + input_cache: dict[str, tuple[str, Literal["raster", "strds"]]] = {} + swmm_cache: dict[Path, str] = {} + results: list[ResolvedSimulation | ValidationFailure] = [] + for expanded in simulations: + try: + input_names, input_kinds = _resolve_inputs( + dict(expanded.input_maps) | dict(expanded.infiltration.input_maps), + cache=input_cache, + ) + swmm_path, swmm_digest = _resolve_swmm_input(expanded, swmm_cache) + results.append( + _build_resolved_simulation( + expanded, + actual_params, + domain, + effective_mask, + input_names, + input_kinds, + swmm_path, + swmm_digest, + ) + ) + except Exception as error: + results.append( + ValidationFailure( + coordinates=expanded.coordinates, + phase="input_resolution", + detail=f"{type(error).__name__}: {error}", + ) + ) + return tuple(results) + + +def _requested_grass_params(expanded: ExpandedSimulation) -> GrassParams: + config = expanded.grass + if config.database is None: + return GrassParams(region=config.region, mask=config.mask) + return GrassParams( + grassdata=str(Path(config.database).expanduser().resolve()), + location=config.project, + mapset=config.mapset, + region=config.region, + mask=config.mask, + grass_bin=config.executable or "grass", + ) + + +def _build_resolved_simulation( + expanded: ExpandedSimulation, + actual_params: GrassParams, + domain: DomainData, + effective_mask: EffectiveMask, + input_names: dict[str, str], + input_kinds: dict[str, Literal["raster", "strds"]], + swmm_path: Path | None, + swmm_digest: str | None, +) -> ResolvedSimulation: + assert actual_params.mapset is not None + simulation_id = _simulation_identity( + expanded, + actual_params, + domain, + effective_mask, + input_names, + swmm_digest, + ) + artifacts = _render_artifacts(expanded, simulation_id, actual_params.mapset) + simulation_config = _build_simulation_config( + expanded, + input_names, + artifacts, + swmm_path, + ) + return ResolvedSimulation( + simulation_id=simulation_id, + coordinates=expanded.coordinates, + grass_params=actual_params, + domain_data=domain, + effective_mask=effective_mask, + input_kinds=tuple(sorted(input_kinds.items())), + simulation_config=simulation_config, + artifacts=artifacts, + ) + + +def _resolve_inputs( + inputs: dict[str, str], + *, + cache: dict[str, tuple[str, Literal["raster", "strds"]]] | None = None, +) -> tuple[dict[str, str], dict[str, Literal["raster", "strds"]]]: + from itzi.grass.utils import resolve_input_identifier + + if cache is None: + cache = {} + resolved: dict[str, str] = {} + kinds: dict[str, Literal["raster", "strds"]] = {} + for key, identifier in inputs.items(): + if identifier not in cache: + cache[identifier] = resolve_input_identifier(identifier) + source, kind = cache[identifier] + resolved[key] = source + kinds[key] = kind + return resolved, kinds + + +def _resolve_swmm_input( + expanded: ExpandedSimulation, cache: dict[Path, str] +) -> tuple[Path | None, str | None]: + if expanded.drainage is None: + return None, None + drainage = dict(expanded.drainage) + configured = Path(str(drainage["swmm_input"])).expanduser() + path = configured if configured.is_absolute() else expanded.source.path.parent / configured + path = path.resolve() + if not path.is_file(): + raise EnsembleError(f"SWMM input file <{path}> not found") + if path in cache: + return path, cache[path] + digest = f"blake2b:{hashlib.blake2b(path.read_bytes(), digest_size=32).hexdigest()}" + cache[path] = digest + return path, digest + + +def _simulation_identity( + expanded: ExpandedSimulation, + grass_params: GrassParams, + domain: DomainData, + effective_mask: EffectiveMask, + input_names: dict[str, str], + swmm_digest: str | None, +) -> str: + """Build the versioned identity payload after all source normalization.""" + drainage = dict(expanded.drainage) if expanded.drainage is not None else None + if drainage is not None: + assert swmm_digest is not None + drainage["swmm_digest"] = swmm_digest + drainage["swmm_input"] = str( + (expanded.source.path.parent / str(drainage["swmm_input"])).resolve() + ) + time_payload: dict[str, str | None] = { + "temporal_type": str(expanded.time.temporal_type), + "start": expanded.time.start.isoformat() if expanded.time.start is not None else None, + "end": expanded.time.end.isoformat() if expanded.time.end is not None else None, + "duration": format_duration(expanded.time.duration), + "record_step": format_duration(expanded.time.record_step), + } + payload = { + "identity_version": 1, + "grass": { + "database": grass_params.grassdata, + "project": grass_params.location, + "mapset": grass_params.mapset, + }, + "domain": domain.model_dump(), + "mask": {"mode": effective_mask.mode, "source": effective_mask.source}, + "time": time_payload, + "inputs": input_names, + "infiltration": {"model": str(expanded.infiltration.model)}, + "parameters": dict(expanded.parameters), + "drainage": drainage, + } + canonical = json.dumps(payload, allow_nan=False, separators=(",", ":"), sort_keys=True) + digest = hashlib.blake2b(canonical.encode(), digest_size=4).hexdigest() + return f"sim-{digest}" + + +def _render_artifacts( + expanded: ExpandedSimulation, + simulation_id: str, + mapset: str, +) -> ArtifactSummary: + output_map_names: dict[str, str] = {} + if expanded.outputs.raster_prefix is not None: + prefix = render_template( + expanded.outputs.raster_prefix, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ) + output_map_names = { + variable: f"{prefix}_{variable}" for variable in expanded.outputs.raster_variables + } + drainage_output = ( + render_template( + expanded.outputs.drainage_dataset, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ) + if expanded.outputs.drainage_dataset is not None + else None + ) + stats_file = ( + _source_relative_path( + render_template( + expanded.outputs.statistics_file, + ensemble=expanded.ensemble_id, + simulation=simulation_id, + ), + expanded.source.path.parent, + ) + if expanded.outputs.statistics_file is not None + else None + ) + _validate_output_names( + output_map_names, + drainage_output, + _last_record_index(expanded.time.duration, expanded.time.record_step), + ) + _validate_protected_file_output(stats_file, (expanded.source.path, _drainage_path(expanded))) + return ArtifactSummary( + tuple(sorted((key, f"{name}@{mapset}") for key, name in output_map_names.items())), + f"{drainage_output}@{mapset}" if drainage_output is not None else None, + stats_file, + ) + + +def _source_relative_path(value: str, source_dir: Path) -> Path: + path = Path(value).expanduser() + return (path if path.is_absolute() else source_dir / path).resolve() + + +def _drainage_path(expanded: ExpandedSimulation) -> Path | None: + if expanded.drainage is None: + return None + return _source_relative_path( + str(dict(expanded.drainage)["swmm_input"]), expanded.source.path.parent + ) + + +def _validate_protected_file_output(output: Path | None, protected: Iterable[Path | None]) -> None: + if output is None: + return + for path in protected: + if path is None: + continue + source = path.resolve() + if output == source or (output.exists() and source.exists() and output.samefile(source)): + raise EnsembleError(f"output file <{output}> aliases protected input <{source}>") + + +def _last_record_index(duration: timedelta, record_step: timedelta) -> int: + """Return the final output index, including the initial record at index zero.""" + return -(-duration // record_step) + + +def _validate_output_names( + output_map_names: dict[str, str], drainage_output: str | None, last_record_index: int +) -> None: + from itzi.grass.utils import is_clean_name + + names = list(output_map_names.values()) + if drainage_output is not None: + names.append(drainage_output) + for name in names: + if "@" in name: + raise EnsembleError(f"GRASS output names must be unqualified: {name!r}") + if name == "MASK": + raise EnsembleError("MASK cannot be used as an output name") + if not is_clean_name(name): + raise EnsembleError(f"invalid GRASS output name {name!r}") + child = derived_record_name(name, last_record_index) + if child == "MASK" or not is_clean_name(child): + raise EnsembleError(f"invalid derived GRASS output name {child!r}") + if drainage_output is not None: + child = derived_record_name(drainage_output, last_record_index) + for table_name in derived_drainage_table_names(child): + if not is_clean_name(table_name): + raise EnsembleError(f"invalid derived drainage table name {table_name!r}") + + +def _build_simulation_config( + expanded: ExpandedSimulation, + input_names: dict[str, str], + artifacts: ArtifactSummary, + swmm_path: Path | None, +) -> SimulationConfig: + start = expanded.time.start + end = expanded.time.end + if start is None: + start = datetime.min # noqa: DTZ901 # Core's established relative-time adapter. + end = start + expanded.time.duration + assert end is not None + parameters = dict(expanded.parameters) + surface_parameters = {key: value for key, value in parameters.items() if key != "dtinf"} + # Create the dict first to satisfy the type checker + config_data = { + "start_time": start, + "end_time": end, + "record_step": expanded.time.record_step, + "temporal_type": expanded.time.temporal_type, + "hotstart_config": None, + "input_map_names": input_names, + "output_map_names": dict(artifacts.output_map_names), + "surface_flow_parameters": SurfaceFlowParameters(**surface_parameters), + "dtinf": parameters["dtinf"], + "infiltration_model": expanded.infiltration.model, + "swmm_inp": swmm_path, + "drainage_output": artifacts.drainage_output, + } + if expanded.drainage is not None: + config_data.update((key, value) for key, value in expanded.drainage if key != "swmm_input") + try: + return SimulationConfig.model_validate(config_data) + except ValidationError as error: + details = "; ".join(item["msg"] for item in error.errors(include_url=False)) + raise EnsembleError(details) from error + + +def validate_resolved_ensemble( + simulations: tuple[ResolvedSimulation, ...], protected_inputs: tuple[Path, ...] = () +) -> None: + """Reject identity and artifact collisions among resolved ensemble members.""" + identities: set[str] = set() + artifacts: dict[str, str] = {} + ensemble_sources: set[str] = set() + protected_paths = {path.resolve() for path in protected_inputs} + for simulation in simulations: + ensemble_sources.update(simulation.simulation_config.input_map_names.values()) + if simulation.effective_mask.source is not None: + ensemble_sources.add(simulation.effective_mask.source) + if simulation.simulation_config.swmm_inp is not None: + protected_paths.add(simulation.simulation_config.swmm_inp.resolve()) + for simulation in simulations: + if simulation.simulation_id in identities: + raise EnsembleError(f"duplicate resolved simulation {simulation.simulation_id}") + identities.add(simulation.simulation_id) + + last_record_index = _last_record_index( + simulation.simulation_config.end_time - simulation.simulation_config.start_time, + simulation.simulation_config.record_step, + ) + for _, name in simulation.artifacts.output_map_names: + _claim_artifact(artifacts, name, simulation.simulation_id) + _reject_output_source_alias(name, ensemble_sources) + for index in range(last_record_index + 1): + child = derived_record_name(name, index) + _claim_artifact(artifacts, child, simulation.simulation_id) + _reject_output_source_alias(child, ensemble_sources) + if simulation.artifacts.drainage_output is not None: + drainage_name = simulation.artifacts.drainage_output + _claim_artifact(artifacts, drainage_name, simulation.simulation_id) + _reject_output_source_alias(drainage_name, ensemble_sources) + for index in range(last_record_index + 1): + child = derived_record_name(drainage_name, index) + _claim_artifact(artifacts, child, simulation.simulation_id) + _reject_output_source_alias(child, ensemble_sources) + for table_name in derived_drainage_table_names(child): + _claim_artifact(artifacts, f"table:{table_name}", simulation.simulation_id) + if simulation.artifacts.statistics_file is not None: + _validate_protected_file_output(simulation.artifacts.statistics_file, protected_paths) + _claim_artifact( + artifacts, + str(simulation.artifacts.statistics_file), + simulation.simulation_id, + ) + + +def _claim_artifact(artifacts: dict[str, str], artifact: str, simulation_id: str) -> None: + owner = artifacts.get(artifact) + if owner is None: + artifacts[artifact] = simulation_id + return + if owner == simulation_id: + raise EnsembleError(f"output artifact {artifact!r} is used more than once") + raise EnsembleError( + f"output artifact {artifact!r} is shared by simulations {owner} and {simulation_id}" + ) + + +def _reject_output_source_alias(map_id: str, sources: set[str]) -> None: + if map_id in sources: + raise EnsembleError(f"output {map_id!r} aliases an ensemble input or mask") + + +def verify_resolved_simulation(simulation: ResolvedSimulation) -> None: + """Re-resolve worker facts and reject a changed GRASS environment.""" + from itzi.grass.utils import active_grass_params, read_domain, resolve_effective_mask + + actual = active_grass_params(simulation.grass_params) + if actual != simulation.grass_params: + raise EnsembleError("GRASS context changed after resolution") + domain = read_domain(simulation.grass_params.region) + if domain != simulation.domain_data: + raise EnsembleError("GRASS domain changed after resolution") + mask = resolve_effective_mask(simulation.grass_params.mask) + if mask != simulation.effective_mask: + raise EnsembleError("effective GRASS mask changed after resolution") + _verify_input_kinds(simulation.simulation_config.input_map_names, dict(simulation.input_kinds)) + + +def _verify_input_kinds( + inputs: dict[str, str], expected: dict[str, Literal["raster", "strds"]] +) -> None: + resolved, kinds = _resolve_inputs(inputs) + if resolved != inputs or kinds != expected: + raise EnsembleError("resolved GRASS input sources changed after resolution") diff --git a/src/itzi/ensemble/run_validation.py b/src/itzi/ensemble/run_validation.py new file mode 100644 index 00000000..85b8e4f2 --- /dev/null +++ b/src/itzi/ensemble/run_validation.py @@ -0,0 +1,136 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from itzi.ensemble.models import EnsembleError +from itzi.ensemble.resolution import _last_record_index, verify_resolved_simulation +from itzi.grass.names import derived_record_name +from itzi.manifest import validate_file_destination + +if TYPE_CHECKING: + from itzi.ensemble.models import ResolvedSimulation + from itzi.grass.interface import GrassInterface + + +def validate_run_requirements(simulation: ResolvedSimulation) -> None: + """Validate one resolved simulation without creating user artifacts.""" + from itzi.grass.interface import GrassInterface + from itzi.grass.utils import ensure_min_version + + verify_resolved_simulation(simulation) + ensure_min_version() + config = simulation.simulation_config + interface = GrassInterface( + start_time=config.start_time, + end_time=config.end_time, + dtype=np.float32, + region_id=simulation.grass_params.region, + raster_mask_id=simulation.grass_params.mask, + effective_mask=simulation.effective_mask, + ) + try: + _validate_inputs(simulation, interface) + _validate_grass_outputs(simulation, interface) + if simulation.artifacts.statistics_file is not None: + validate_file_destination( + simulation.artifacts.statistics_file, + overwrite=interface.overwrite, + description="statistics file", + ) + finally: + interface.cleanup() + + +def _validate_inputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + from itzi.grass.utils import resolve_input_map_lists + + config = simulation.simulation_config + mask = interface.get_npmask() + if np.all(mask): + raise EnsembleError("effective mask excludes the complete simulation domain") + + map_lists = resolve_input_map_lists( + interface, + config.input_map_names, + config.start_time, + config.end_time, + dict(simulation.input_kinds), + ) + for key in ("ground_elevation", "friction"): + for map_data in map_lists[key] or (): + array = interface.read_raster_map(map_data.id) + active = array[~mask] + if not np.any(np.isfinite(active)): + raise EnsembleError( + f"input map <{key}> contains only NULL/NaN cells inside the active domain" + ) + + +def _validate_grass_outputs(simulation: ResolvedSimulation, interface: GrassInterface) -> None: + from itzi.grass.utils import ( + get_current_mapset, + output_name, + raster_exists, + stds_exists, + vector_exists, + ) + + config = simulation.simulation_config + last_index = _last_record_index( + config.end_time - config.start_time, + config.record_step, + ) + + mapset = get_current_mapset() + + def reject_existing(name: str, exists: bool, kind: str) -> None: + if exists and not interface.overwrite: + raise EnsembleError(f"{kind} <{name}> already exists") + + for map_id in config.output_map_names.values(): + name = output_name(map_id) + if not interface.overwrite: + reject_existing( + map_id, + raster_exists(name, mapset) or stds_exists(map_id, "strds"), + "raster output", + ) + interface.validate_output_stds_temporal_type(map_id, "strds", config.temporal_type) + if not interface.overwrite: + for index in range(last_index + 1): + child = derived_record_name(map_id, index) + reject_existing(child, raster_exists(output_name(child), mapset), "raster output") + + if config.drainage_output is None: + return + map_id = config.drainage_output + name = output_name(map_id) + if not interface.overwrite: + reject_existing( + map_id, + vector_exists(name, mapset) or stds_exists(map_id, "stvds"), + "drainage output", + ) + interface.validate_output_stds_temporal_type(map_id, "stvds", config.temporal_type) + if interface.overwrite: + return + + for index in range(last_index + 1): + child = derived_record_name(map_id, index) + reject_existing(child, vector_exists(output_name(child), mapset), "drainage output") diff --git a/src/itzi/ensemble/schema.py b/src/itzi/ensemble/schema.py new file mode 100644 index 00000000..d3288264 --- /dev/null +++ b/src/itzi/ensemble/schema.py @@ -0,0 +1,184 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from itzi_core import OUTPUT_ARRAY_KEYS +from pydantic import ( + AfterValidator, + Field, + StrictFloat, + StrictStr, + field_validator, + model_validator, +) + +from itzi.ensemble.models import GrassConfig, StrictModel + + +def _validate_sweep[T](values: list[T]) -> list[T]: + """Reject duplicate values in a sweep after type validation.""" + if len(values) != len(set(values)): + raise ValueError("contains duplicate semantic values") + return values + + +type SweepString = ( + StrictStr | Annotated[list[StrictStr], Field(min_length=1), AfterValidator(_validate_sweep)] +) +type FiniteFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] +type SweepFloat = ( + FiniteFloat + | Annotated[list[FiniteFloat], Field(min_length=1), AfterValidator(_validate_sweep)] +) + + +class EnsembleMetadata(StrictModel): + id: StrictStr = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$", max_length=80) + description: StrictStr | None = Field(default=None, max_length=256) + + +class TimeConfig(StrictModel): + start: StrictStr | None = None + end: StrictStr | None = None + record_step: StrictStr + duration: StrictStr | None = None + + @model_validator(mode="after") + def validate_combination(self) -> TimeConfig: + if (self.start is None and self.end is None and self.duration is not None) or ( + self.start is not None and (self.end is None) != (self.duration is None) + ): + return self + raise ValueError("time requires duration alone, start and duration, or start and end") + + +class NoInfiltration(StrictModel): + type: Literal["none"] + + +class ConstantInfiltration(StrictModel): + type: Literal["constant"] + rate: StrictStr + + +class GreenAmptInfiltration(StrictModel): + type: Literal["green-ampt"] + effective_porosity: StrictStr + capillary_pressure: StrictStr + hydraulic_conductivity: StrictStr + soil_water_content: StrictStr | None = None + + +type InfiltrationAlternative = Annotated[ + NoInfiltration | ConstantInfiltration | GreenAmptInfiltration, + Field(discriminator="type"), +] +type SweepInfiltration = ( + InfiltrationAlternative + | Annotated[ + list[InfiltrationAlternative], Field(min_length=1), AfterValidator(_validate_sweep) + ] +) + + +class InputSweepConfig(StrictModel): + ground_elevation: SweepString + friction: SweepString + water_depth: SweepString | None = None + water_surface_elevation: SweepString | None = None + losses: SweepString | None = None + rainfall_rate: SweepString | None = None + inflow: SweepString | None = None + boundary_value: SweepString | None = None + boundary_type: SweepString | None = None + infiltration: SweepInfiltration = NoInfiltration(type="none") + + @model_validator(mode="after") + def validate_initial_conditions(self) -> InputSweepConfig: + if self.water_depth is not None and self.water_surface_elevation is not None: + raise ValueError("water_depth and water_surface_elevation are mutually exclusive") + return self + + +class ParameterSweepConfig(StrictModel): + hmin: SweepFloat | None = None + cfl: SweepFloat | None = None + theta: SweepFloat | None = None + g: SweepFloat | None = None + dtmax: SweepFloat | None = None + slope_threshold: SweepFloat | None = None + max_slope: SweepFloat | None = None + max_error: SweepFloat | None = None + dtinf: SweepFloat | None = None + + +class DrainageSweepConfig(StrictModel): + swmm_input: SweepString + orifice_coeff: SweepFloat | None = None + free_weir_coeff: SweepFloat | None = None + submerged_weir_coeff: SweepFloat | None = None + + +class RasterOutputs(StrictModel): + prefix: StrictStr + variables: list[StrictStr] = Field(min_length=1) + + @field_validator("variables") + @classmethod + def validate_variables(cls, value: list[str]) -> list[str]: + unknown = sorted(set(value) - set(OUTPUT_ARRAY_KEYS)) + if unknown: + raise ValueError(f"contains unsupported output variables: {', '.join(unknown)}") + if len(value) != len(set(value)): + raise ValueError("contains duplicate output variables") + return value + + +class StatisticsOutputs(StrictModel): + file: StrictStr = Field(min_length=1) + + +class DrainageOutputs(StrictModel): + vector_dataset: StrictStr + + +class ManifestOutputs(StrictModel): + file: StrictStr = Field(min_length=1) + + +class OutputConfig(StrictModel): + rasters: RasterOutputs | None = None + statistics: StatisticsOutputs | None = None + drainage: DrainageOutputs | None = None + manifest: ManifestOutputs | None = None + + +class YamlEnsembleDocumentV1(StrictModel): + schema_version: Literal[1] + ensemble: EnsembleMetadata + grass: GrassConfig + time: TimeConfig + input: InputSweepConfig + parameters: ParameterSweepConfig + drainage: DrainageSweepConfig | None = None + outputs: OutputConfig + + @model_validator(mode="after") + def validate_drainage_output(self) -> YamlEnsembleDocumentV1: + if self.outputs.drainage is not None and self.drainage is None: + raise ValueError("outputs.drainage requires a drainage configuration") + return self diff --git a/src/itzi/ensemble/yaml.py b/src/itzi/ensemble/yaml.py new file mode 100644 index 00000000..ed85373a --- /dev/null +++ b/src/itzi/ensemble/yaml.py @@ -0,0 +1,481 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import itertools +import math +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import cast + +import yaml +from itzi_core import InfiltrationModelType, SurfaceFlowParameters, TemporalType +from itzi_core.const import DefaultValues +from pydantic import JsonValue as PydanticJsonValue +from pydantic import TypeAdapter, ValidationError +from yaml.constructor import ConstructorError +from yaml.resolver import BaseResolver + +import itzi.messenger as msgr +from itzi.ensemble.models import ( + DIRECT_INPUT_KEYS, + MAX_ENSEMBLE_MEMBERS, + DocumentFailure, + EnsembleError, + ExpandedEnsemble, + ExpandedSimulation, + JsonValue, + LoadedYamlStream, + NormalizedInfiltration, + NormalizedTime, + OutputTemplates, + SourceDocument, + parse_duration, + render_template, +) +from itzi.ensemble.schema import ( + ConstantInfiltration, + DrainageSweepConfig, + GreenAmptInfiltration, + InfiltrationAlternative, + InputSweepConfig, + NoInfiltration, + ParameterSweepConfig, + SweepFloat, + SweepString, + TimeConfig, + YamlEnsembleDocumentV1, +) + +type RawYamlScalar = None | bool | int | float | str | bytes | date | datetime +type RawYamlKey = RawYamlScalar | tuple[RawYamlKey, ...] +type RawYamlValue = ( + RawYamlScalar + | list[RawYamlValue] + | tuple[RawYamlValue, ...] + | dict[RawYamlKey, RawYamlValue] + | set[RawYamlKey] +) +type RawYamlMapping = dict[RawYamlKey, RawYamlValue] +type YamlDocument = dict[str, PydanticJsonValue] +type SweepScalar = str | float | InfiltrationAlternative + +_YAML_DOCUMENT_ADAPTER = TypeAdapter(YamlDocument) + + +class _StrictSafeLoader(yaml.SafeLoader): + """SafeLoader variant that rejects duplicate and merge mapping keys.""" + + +def _construct_mapping( + loader: _StrictSafeLoader, node: yaml.MappingNode, deep: bool = False +) -> RawYamlMapping: + mapping: RawYamlMapping = {} + for key_node, value_node in node.value: + if key_node.tag == "tag:yaml.org,2002:merge": + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "YAML merge keys are not supported", + key_node.start_mark, + ) + key = cast(RawYamlKey, loader.construct_object(key_node, deep=deep)) + try: + duplicate = key in mapping + except TypeError as error: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "mapping keys must be hashable", + key_node.start_mark, + ) from error + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"duplicate mapping key {key!r}", + key_node.start_mark, + ) + mapping[key] = cast(RawYamlValue, loader.construct_object(value_node, deep=deep)) + return mapping + + +_StrictSafeLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, + _construct_mapping, +) + + +def normalize_time(value: TimeConfig) -> NormalizedTime: + """Validate a public time form without leaking core's relative sentinels.""" + duration = parse_duration(value.duration) if value.duration is not None else None + record_step = parse_duration(value.record_step) + start = _parse_datetime(value.start) if value.start is not None else None + end = _parse_datetime(value.end) if value.end is not None else None + aware = [timestamp for timestamp in (start, end) if timestamp is not None and timestamp.tzinfo] + naive = [ + timestamp for timestamp in (start, end) if timestamp is not None and not timestamp.tzinfo + ] + if aware and naive: + raise EnsembleError( + "absolute timestamps must either all have timezone offsets or all be naive" + ) + had_timezone_offset = bool(aware) + if aware: + start = start.replace(tzinfo=None) if start is not None else None + end = end.replace(tzinfo=None) if end is not None else None + + if start is None: + assert duration is not None + return NormalizedTime( + temporal_type=TemporalType.RELATIVE, + start=None, + end=None, + duration=duration, + record_step=record_step, + had_timezone_offset=False, + ) + if end is None: + assert duration is not None + end = start + duration + else: + duration = end - start + if duration <= timedelta(): + raise EnsembleError("simulation duration must be strictly positive") + return NormalizedTime( + temporal_type=TemporalType.ABSOLUTE, + start=start, + end=end, + duration=duration, + record_step=record_step, + had_timezone_offset=had_timezone_offset, + ) + + +def _parse_datetime(value: str) -> datetime: + if "T" not in value: + raise EnsembleError("timestamps must use ISO 8601 date and time notation") + try: + return datetime.fromisoformat(value) + except ValueError as error: + raise EnsembleError(f"invalid ISO 8601 timestamp {value!r}") from error + + +def _is_document_marker(line: str, *, terminated: bool) -> bool: + """Return whether a physical line is a supported YAML document marker.""" + if terminated and line.endswith("\r"): + line = line[:-1] + if not line.startswith("---"): + return False + suffix = line[3:] + if suffix and suffix[0] not in " \t": + return False + suffix = suffix.lstrip(" \t") + return not suffix or suffix.startswith("#") + + +def _segments(text: str) -> tuple[tuple[int, int, str], ...]: + """Split at supported explicit document markers without parsing adjacent docs.""" + markers: list[tuple[int, int]] = [] + offset = 0 + lines = text.split("\n") + for index, line in enumerate(lines): + terminated = index < len(lines) - 1 + if _is_document_marker(line, terminated=terminated): + markers.append((offset, index + 1)) + offset += len(line) + int(terminated) + if not markers: + return ((1, 0, text),) + segments: list[tuple[int, int, str]] = [] + first = text[: markers[0][0]] + if first.strip(): + segments.append((1, 0, first)) + for index, (marker_offset, start_line) in enumerate(markers): + end = markers[index + 1][0] if index + 1 < len(markers) else len(text) + segments.append((start_line, marker_offset, text[marker_offset:end])) + return tuple(segments) + + +def _parse_segment(segment: str) -> YamlDocument: + if any(line.startswith("%") for line in segment.split("\n")): + raise EnsembleError("YAML directives are not supported") + value = cast(RawYamlValue, yaml.load(segment, Loader=_StrictSafeLoader)) + if value is None: + raise EnsembleError("empty YAML documents are not supported") + if not isinstance(value, dict): + raise EnsembleError("document root must be a mapping") + try: + return _YAML_DOCUMENT_ADAPTER.validate_python(value, strict=True) + except ValidationError as error: + raise EnsembleError("document must contain JSON-compatible values") from error + + +def load_yaml_stream(path: str | Path) -> LoadedYamlStream: + """Load every explicitly delimited YAML document, retaining local failures.""" + source_path = Path(path).expanduser().resolve() + try: + content = source_path.read_bytes() + except OSError as error: + raise EnsembleError(f"cannot read {source_path}: {error}") from error + try: + text = content.decode("utf-8") + except UnicodeDecodeError as error: + raise EnsembleError(f"{source_path}: YAML must be UTF-8") from error + + ensembles: list[ExpandedEnsemble] = [] + failures: list[DocumentFailure] = [] + for document_index, (start_line, _offset, segment) in enumerate(_segments(text)): + try: + raw_document = _parse_segment(segment) + except yaml.YAMLError as error: + marker = getattr(error, "problem_mark", None) + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line + marker.line if marker is not None else None, + column=marker.column + 1 if marker is not None else None, + phase="parse", + detail=str(getattr(error, "problem", error)), + ) + ) + continue + except EnsembleError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="parse", + detail=str(error), + ) + ) + continue + + source = SourceDocument( + path=source_path, + document_index=document_index, + ) + try: + document = YamlEnsembleDocumentV1.model_validate(raw_document) + ensemble = expand_yaml_document(source, document) + if ensemble.simulations[0].time.had_timezone_offset: + msgr.warning( + f"{source.path} document {source.document_index}: timezone offsets are " + "informational and are ignored by GRASS" + ) + ensembles.append(ensemble) + except ValidationError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="schema", + detail=_format_validation_error(error), + ) + ) + except EnsembleError as error: + failures.append( + DocumentFailure( + path=source_path, + document_index=document_index, + line=start_line, + column=1, + phase="expansion", + detail=str(error), + ) + ) + return LoadedYamlStream(tuple(ensembles), tuple(failures)) + + +def _format_validation_error(error: ValidationError) -> str: + details = [] + for item in error.errors(include_url=False): + location = ".".join(str(part) for part in item["loc"]) + details.append(f"{location}: {item['msg']}" if location else item["msg"]) + return "; ".join(details) + + +def expand_yaml_document( + source: SourceDocument, document: YamlEnsembleDocumentV1 +) -> ExpandedEnsemble: + """Expand sweep dimensions in canonical path order into scalar simulations.""" + normalized_time = normalize_time(document.time) + dimensions = _collect_dimensions(document) + count = math.prod(len(values) for _, values in dimensions) + if count > MAX_ENSEMBLE_MEMBERS: + sizes = ", ".join(f"{path}={len(values)}" for path, values in dimensions) + raise EnsembleError( + f"ensemble expands to {count} simulations, exceeding the {MAX_ENSEMBLE_MEMBERS} limit ({sizes})" + ) + + simulations = [] + for selected_values in itertools.product(*(values for _, values in dimensions)): + selected: dict[str, SweepScalar] = { + path: value for (path, _), value in zip(dimensions, selected_values, strict=True) + } + coordinates = tuple( + (path, _freeze_json(value)) for path, value in sorted(selected.items()) + ) + input_maps = _selected_input_maps(document.input, selected) + infiltration_value = selected.get("input.infiltration", document.input.infiltration) + assert isinstance( + infiltration_value, + (NoInfiltration, ConstantInfiltration, GreenAmptInfiltration), + ) + infiltration = _normalize_infiltration(infiltration_value) + parameters = _selected_parameters(document.parameters, selected) + drainage = _selected_drainage(document.drainage, selected) + outputs = _output_templates(document) + simulations.append( + ExpandedSimulation( + source=source, + ensemble_id=document.ensemble.id, + coordinates=coordinates, + grass=document.grass, + time=normalized_time, + input_maps=tuple(sorted(input_maps.items())), + infiltration=infiltration, + parameters=tuple(sorted(parameters.items())), + drainage=tuple(sorted(drainage.items())) if drainage is not None else None, + outputs=outputs, + ) + ) + manifest_template = document.outputs.manifest.file if document.outputs.manifest else None + if manifest_template is not None: + render_template(manifest_template, ensemble=document.ensemble.id, simulation=None) + return ExpandedEnsemble( + source=source, + ensemble_id=document.ensemble.id, + ensemble_description=document.ensemble.description, + manifest_template=manifest_template, + simulations=tuple(simulations), + ) + + +def _collect_dimensions( + document: YamlEnsembleDocumentV1, +) -> tuple[tuple[str, tuple[SweepScalar, ...]], ...]: + dimensions: list[tuple[str, tuple[SweepScalar, ...]]] = [] + for key in DIRECT_INPUT_KEYS: + value = cast(SweepString | None, getattr(document.input, key)) + if isinstance(value, list): + dimensions.append((f"input.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) + if isinstance(document.input.infiltration, list): + dimensions.append( + ( + "input.infiltration", + cast(tuple[SweepScalar, ...], tuple(document.input.infiltration)), + ) + ) + for key in type(document.parameters).model_fields: + value = cast(SweepFloat | None, getattr(document.parameters, key)) + if isinstance(value, list): + dimensions.append((f"parameters.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) + if document.drainage is not None: + for key in type(document.drainage).model_fields: + value = cast(SweepString | SweepFloat | None, getattr(document.drainage, key)) + if isinstance(value, list): + dimensions.append((f"drainage.{key}", cast(tuple[SweepScalar, ...], tuple(value)))) + return tuple(sorted(dimensions, key=lambda item: item[0])) + + +def _selected_input_maps( + input_config: InputSweepConfig, selected: dict[str, SweepScalar] +) -> dict[str, str]: + values: dict[str, str] = {} + for key in DIRECT_INPUT_KEYS: + value = selected.get(f"input.{key}", cast(SweepString | None, getattr(input_config, key))) + if value is not None: + assert isinstance(value, str) + values[key] = value + return values + + +def _normalize_infiltration(value: InfiltrationAlternative) -> NormalizedInfiltration: + if isinstance(value, NoInfiltration): + return NormalizedInfiltration(InfiltrationModelType.NULL, ()) + if isinstance(value, ConstantInfiltration): + return NormalizedInfiltration( + InfiltrationModelType.CONSTANT, (("infiltration", value.rate),) + ) + assert isinstance(value, GreenAmptInfiltration) + maps = { + "effective_porosity": value.effective_porosity, + "capillary_pressure": value.capillary_pressure, + "hydraulic_conductivity": value.hydraulic_conductivity, + } + if value.soil_water_content is not None: + maps["soil_water_content"] = value.soil_water_content + return NormalizedInfiltration(InfiltrationModelType.GREEN_AMPT, tuple(sorted(maps.items()))) + + +def _selected_parameters( + parameters: ParameterSweepConfig, selected: dict[str, SweepScalar] +) -> dict[str, float]: + values: dict[str, float] = SurfaceFlowParameters().model_dump() + values["dtinf"] = DefaultValues.DTINF + for key in type(parameters).model_fields: + value = selected.get( + f"parameters.{key}", cast(SweepFloat | None, getattr(parameters, key)) + ) + if value is not None: + assert isinstance(value, float) + values[key] = value + return values + + +def _selected_drainage( + drainage: DrainageSweepConfig | None, selected: dict[str, SweepScalar] +) -> dict[str, str | float] | None: + if drainage is None: + return None + values: dict[str, str | float] = {} + for key in type(drainage).model_fields: + value = selected.get( + f"drainage.{key}", cast(SweepString | SweepFloat | None, getattr(drainage, key)) + ) + if value is None: + continue + assert isinstance(value, (str, float)), "drainage must be scalar after expansion" + values[key] = value + return values + + +def _output_templates(document: YamlEnsembleDocumentV1) -> OutputTemplates: + raster_prefix = document.outputs.rasters.prefix if document.outputs.rasters else None + variables = tuple(document.outputs.rasters.variables) if document.outputs.rasters else () + statistics_file = document.outputs.statistics.file if document.outputs.statistics else None + drainage_dataset = ( + document.outputs.drainage.vector_dataset if document.outputs.drainage else None + ) + for template in (raster_prefix, statistics_file, drainage_dataset): + if template is not None: + render_template(template, ensemble=document.ensemble.id, simulation="simulation") + return OutputTemplates( + raster_prefix=raster_prefix, + raster_variables=variables, + statistics_file=statistics_file, + drainage_dataset=drainage_dataset, + ) + + +def _freeze_json(value: SweepScalar) -> JsonValue: + if isinstance(value, (str, int, float)): + return value + return tuple(sorted(value.model_dump(exclude_none=True).items())) diff --git a/src/itzi/grass/__init__.py b/src/itzi/grass/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/itzi/data/colortable/depth.txt b/src/itzi/grass/colortable/depth.txt similarity index 100% rename from src/itzi/data/colortable/depth.txt rename to src/itzi/grass/colortable/depth.txt diff --git a/src/itzi/data/colortable/froude.txt b/src/itzi/grass/colortable/froude.txt similarity index 100% rename from src/itzi/data/colortable/froude.txt rename to src/itzi/grass/colortable/froude.txt diff --git a/src/itzi/data/colortable/vdir.txt b/src/itzi/grass/colortable/vdir.txt similarity index 100% rename from src/itzi/data/colortable/vdir.txt rename to src/itzi/grass/colortable/vdir.txt diff --git a/src/itzi/data/colortable/velocity.txt b/src/itzi/grass/colortable/velocity.txt similarity index 100% rename from src/itzi/data/colortable/velocity.txt rename to src/itzi/grass/colortable/velocity.txt diff --git a/src/itzi/providers/grass_interface.py b/src/itzi/grass/interface.py similarity index 59% rename from src/itzi/providers/grass_interface.py rename to src/itzi/grass/interface.py index f3610e86..94c7fb56 100644 --- a/src/itzi/providers/grass_interface.py +++ b/src/itzi/grass/interface.py @@ -17,10 +17,8 @@ from collections.abc import Mapping from datetime import datetime, timedelta from pathlib import Path -from queue import Queue -from threading import Lock, Thread from types import MappingProxyType -from typing import ClassVar, NamedTuple, Self +from typing import ClassVar, NamedTuple import grass.pygrass.utils as gutils import grass.script as gscript @@ -31,7 +29,7 @@ from grass.pygrass.vector import VectorTopo from grass.pygrass.vector.geometry import Line, Point from grass.pygrass.vector.table import Link, Table -from itzi_core import TemporalType +from itzi_core import DomainData, TemporalType from itzi_core.data_containers import ( DrainageLinkAttributes, DrainageNetworkAttributes, @@ -40,15 +38,27 @@ ) import itzi.messenger as msgr +from itzi.ensemble.models import EffectiveMask +from itzi.grass.utils import ( + MapData, + RasterMapType, + STDSType, + get_current_mapset, + name_is_map, + output_name, + read_domain, + replace_cell_null_sentinel, + resolve_effective_mask, + set_null, + split_identifier, +) # color rules -_ROOT = Path(__file__).parent.parent -_DIR = _ROOT / "data" / "colortable" +_DIR = Path(__file__).parent / "colortable" RULE_H = _DIR / "depth.txt" RULE_V = _DIR / "velocity.txt" RULE_VDIR = _DIR / "vdir.txt" RULE_FR = _DIR / "froude.txt" -RULE_DEF = _DIR / "default.txt" colors_rules_dict = { "water_depth": str(RULE_H), "max_water_depth": str(RULE_H), @@ -57,79 +67,10 @@ "flow_velocity_direction": str(RULE_VDIR), "froude": str(RULE_FR), } - - -def _init_temporal() -> None: - """Initialize GRASS temporal APIs without allowing them to exit Itzi.""" - gscript.set_raise_on_error(True) - tgis.init(raise_fatal_error=True) - tgis.set_raise_on_error(True) - - -# Check if color rule paths are OK for f in colors_rules_dict.values(): assert Path(f).is_file() -def file_exists(name): - """Return True if name is an existing map or stds, False otherwise""" - if not name: - return False - else: - _id = GrassInterface.format_id(name) - return GrassInterface.name_is_map(_id) or GrassInterface.name_is_stds(_id) - - -def check_output_files(file_list) -> None: - """Check if the output files exist""" - for map_name in file_list: - if file_exists(map_name) and not gscript.overwrite(): - msgr.fatal(f"File {map_name} exists and will not be overwritten") - - -def apply_color_table(map_name, mkey) -> None: - """Apply a color table determined by mkey to the given map""" - try: - colors_rules = colors_rules_dict[mkey] - except KeyError: - # in case no specific color table is given, use GRASS default. - pass - else: - gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=map_name) - - -def raster_writer(q, lock): - """Write a raster map in GRASS""" - while True: - # Get values from the queue - next_object = q.get() - if next_object is None: - break - arr, rast_name, mtype, mkey, hmin, overwrite = next_object - # Write raster - with lock: - with raster.RasterRow( - rast_name, mode="w", mtype=mtype, overwrite=overwrite - ) as newraster: - newrow = raster.Buffer((arr.shape[1],), mtype=mtype) - for row in arr: - newrow[:] = row[:] - newraster.put_row(newrow) - # Apply colour table - apply_color_table(rast_name, mkey) - # set null values - if mkey == "water_depth" and hmin > 0: - GrassInterface.set_null(rast_name, hmin) - # Signal end of task - q.task_done() - - -class MapData(NamedTuple): - id: str - start_time: datetime - end_time: datetime - - class DBLinkDescription(NamedTuple): layer: int table: Table @@ -182,20 +123,16 @@ def __init__( dtype, region_id: str | None, raster_mask_id: str | None, - non_blocking_write=True, + effective_mask: EffectiveMask | None = None, ) -> None: assert isinstance(start_time, datetime), "start_time not a datetime object!" assert isinstance(end_time, datetime), "end_time not a datetime object!" assert start_time <= end_time, "start_time > end_time!" self.region_id = region_id - self.raster_mask_id = raster_mask_id self.start_time = start_time self.end_time = end_time self.dtype = dtype - self.non_blocking_write = non_blocking_write - - self.old_mask_name = None # LatLon is not supported if gscript.locn_is_latlong(): @@ -210,61 +147,29 @@ def __init__( # Check if region is at least 3x3 if self.xr < 3 or self.yr < 3: msgr.fatal("GRASS Region should be at least 3 cells by 3 cells") - self.dx = self.region.ewres - self.dy = self.region.nsres - self.reg_bbox = { - "e": self.region.east, - "w": self.region.west, - "n": self.region.north, - "s": self.region.south, - } - # Set temporary mask - if self.raster_mask_id: - self.set_temp_mask() + selected_mask = effective_mask or resolve_effective_mask(raster_mask_id) + if selected_mask.mode != "none" and selected_mask.source is None: + msgr.fatal(f"Effective {selected_mask.mode} mask has no source") + self.mask_mode = selected_mask.mode + self.mask_source = selected_mask.source if selected_mask.mode != "none" else None self.overwrite = gscript.overwrite() - # init temporal module - _init_temporal() - # Create thread and queue for writing raster maps - if self.non_blocking_write: - self.raster_lock = Lock() - self.raster_writer_queue = Queue(maxsize=15) - worker_args = (self.raster_writer_queue, self.raster_lock) - self.raster_writer_thread = Thread( - name="RasterWriter", target=raster_writer, args=worker_args - ) - self.raster_writer_thread.start() def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): - self.finalize() self.cleanup() def __del__(self): - self.finalize() self.cleanup() - def finalize(self) -> None: - """Make sure that all maps are written.""" - if self.non_blocking_write: - msgr.debug("Writing last maps...") - self.raster_writer_queue.join() - def cleanup(self) -> None: - """Remove temporary region and mask.""" - if self.raster_mask_id: - msgr.debug("Remove temp MASK...") - self.del_temp_mask() + """Remove the temporary region.""" if self.region_id: msgr.debug("Remove temp region...") gscript.del_temp_region() - # Thread killswitch - if self.non_blocking_write: - self.raster_writer_queue.put(None) - self.raster_writer_thread.join() - def grass_dtype(self, dtype: str) -> str: + def grass_dtype(self, dtype: str) -> RasterMapType: if dtype in self.dtype_conv["DCELL"]: mtype = "DCELL" elif dtype in self.dtype_conv["CELL"]: @@ -293,101 +198,20 @@ def to_datetime(self, unit: str, time: int) -> datetime: """ return self.start_time + timedelta(seconds=self.to_s(unit, time)) - @staticmethod - def has_mask() -> bool: - """Return True if the mapset has a mask, False otherwise.""" - return bool(gscript.read_command("g.list", type="raster", pattern="MASK")) + def get_domain_data(self) -> DomainData: + return read_domain(None) def get_npmask(self) -> np.ndarray: """Return a boolean numpy ndarray where True is outside the domain.""" - if self.has_mask(): - grass_mask = self.read_raster_map("MASK") - return ~np.isclose(grass_mask, 1.0) - else: + if self.mask_mode == "none": return np.full(shape=(self.yr, self.xr), fill_value=False, dtype=np.bool_) - - def set_temp_mask(self) -> Self: - """If a mask is already set, keep it for later. - Set a new mask. - """ - has_old_mask: bool = self.has_mask() - if has_old_mask: - # Save the current MASK under a temp name - self.old_mask_name = f"itzi_old_MASK_{os.getpid()}" - gscript.run_command( - "g.rename", - quiet=True, - overwrite=True, - raster=f"MASK,{self.old_mask_name}", - ) - gscript.run_command("r.mask", quiet=True, raster=self.raster_mask_id) - assert self.has_mask() - return self - - def del_temp_mask(self) -> Self: - """Reset the old mask, remove if there was not.""" - if self.old_mask_name is not None: - gscript.run_command( - "g.rename", - quiet=True, - overwrite=True, - raster=f"{self.old_mask_name},MASK", - ) - else: - gscript.run_command("r.mask", quiet=True, flags="r") - return self - - def coor2pixel(self, coor: tuple[float, float]) -> tuple[int, int]: - """convert coordinates easting and northing to pixel row and column""" - row, col = gutils.coor2pixel(coor, self.region) - return int(row), int(col) - - def is_in_region(self, x: float, y: float) -> bool: - """For a given coordinate pair(x, y), - return True is inside raster region, False otherwise. - """ - bool_x = self.reg_bbox["w"] < x < self.reg_bbox["e"] - bool_y = self.reg_bbox["s"] < y < self.reg_bbox["n"] - return bool(bool_x and bool_y) - - @staticmethod - def format_id(name: str) -> str: - """Take a map or stds name as input - and return a fully qualified name, i.e. including mapset - """ - if "@" in name: - return name - else: - return "@".join((name, gutils.getenv("MAPSET"))) - - @staticmethod - def get_current_mapset() -> str: - return gutils.getenv("MAPSET") - - @staticmethod - def name_is_stds(name: str) -> bool: - """return True if the name given as input is a registered strds - False if not - """ - # make sure temporal module is initialized - _init_temporal() - return bool(tgis.SpaceTimeRasterDataset(name).is_in_db()) - - @staticmethod - def get_crs_wkt() -> str: - return gscript.read_command("g.proj", flags="fw") - - @staticmethod - def name_is_map(map_id: str) -> bool: - """return True if the given name is a map in the grass database - False if not - """ - return bool(gscript.find_file(name=map_id, element="cell").get("file")) - - @staticmethod - def set_null(map_id: str, threshold: float) -> None: - """Set null values under a given threshold""" - gscript.run_command("r.null", flags="f", map=map_id, setnull=f"0.0-{threshold}") + assert self.mask_source is not None + grass_mask = self.read_raster_map(self.mask_source) + if self.mask_mode == "explicit": + # r.mask accepts every non-NULL cell, including a zero value. + return np.isnan(grass_mask) + # A mapset MASK uses CELL semantics: zero and NULL are outside. + return np.isnan(grass_mask) | (grass_mask == 0) def get_sim_extend_in_stds_unit(self, strds) -> tuple[int | datetime, int | datetime]: """Take a strds object as input @@ -415,7 +239,9 @@ def get_sim_extend_in_stds_unit(self, strds) -> tuple[int | datetime, int | date assert False, "unknown temporal type" return start_time_in_stds_unit, end_time_in_stds_unit - def stds_temporal_sanity(self, stds_id: str) -> bool: + def stds_temporal_sanity( + self, stds_id: str, stds: tgis.SpaceTimeRasterDataset | None = None + ) -> bool: """Make the following check on the given stds: - Topology is valid - No gap @@ -423,19 +249,21 @@ def stds_temporal_sanity(self, stds_id: str) -> bool: return True if all the above is True, False otherwise """ out = True - stds = tgis.open_stds.open_old_stds(stds_id, "strds") + if stds is None: + stds = tgis.open_stds.open_old_stds(stds_id, "strds") stds_start, stds_end = stds.get_temporal_extent_as_tuple() if stds_start is None or stds_end is None: msgr.fatal( f"STRDS <{stds_id}> has no temporal extent; " "make sure it contains registered raster maps" ) + maps = stds.get_registered_maps_as_objects(order="start_time") # valid topology - if not stds.check_temporal_topology(): + if not stds.check_temporal_topology(maps=maps): out = False msgr.warning(f"{stds_id}: invalid topology") # no gap - if stds.count_gaps() != 0: + if stds.count_gaps(maps=maps) != 0: out = False msgr.warning(f"{stds_id}: gaps found") # cover all simulation time @@ -457,6 +285,8 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: # transform simulation start and end time in strds unit strds = tgis.open_stds.open_old_stds(strds_name, "strds") + if not self.stds_temporal_sanity(strds_name, strds): + msgr.fatal(f"{strds_name}: inadequate temporal format") sim_start, sim_end = self.get_sim_extend_in_stds_unit(strds) # retrieve data from DB @@ -465,7 +295,7 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: columns=",".join(MapData._fields), where=where, order="start_time" ) # check if every map exist - maps_not_found = [m[0] for m in maplist if not self.name_is_map(m[0])] + maps_not_found = [m[0] for m in maplist if not name_is_map(m[0])] if any(maps_not_found): err_msg = "STRDS <{}>: Can't find following maps: {}" str_lst = ",".join(maps_not_found) @@ -485,99 +315,89 @@ def raster_list_from_strds(self, strds_name: str) -> list[MapData]: def validate_output_stds_temporal_type( self, - stds_name: str | None, - stds_type: str, + stds_id: str, + stds_type: STDSType, expected_temporal_type: TemporalType, ) -> None: """Fail early when overwriting an STDS with incompatible temporal type.""" - if not stds_name or not self.overwrite: + if not self.overwrite: return - stds_id = self.format_id(stds_name) - stds = tgis.dataset_factory(stds_type, stds_id) - if stds is None or not stds.is_in_db(): + output_name(stds_id) + if not tgis.dataset_factory(stds_type, stds_id).is_in_db(): return - existing_stds = tgis.open_stds.open_old_stds(stds_id, stds_type) existing_temporal_type = TemporalType(existing_stds.get_temporal_type()) if existing_temporal_type != expected_temporal_type: msgr.fatal( - f"Output {stds_type.upper()} <{stds_name}> already exists with " + f"Output {stds_type.upper()} <{stds_id}> already exists with " f"{existing_temporal_type} temporal type and cannot be overwritten " f"by a {expected_temporal_type} simulation" ) - def read_raster_map(self, rast_name: str) -> np.ndarray: - """Read a GRASS raster and return a numpy array""" - if self.non_blocking_write: - with self.raster_lock, raster.RasterRow(rast_name, mode="r") as rast: - array = np.array(rast, dtype=self.dtype) - array = self._replace_cell_null_sentinel(rast.mtype, array) - else: - with raster.RasterRow(rast_name, mode="r") as rast: - array = np.array(rast, dtype=self.dtype) - array = self._replace_cell_null_sentinel(rast.mtype, array) - return array - - @staticmethod - def _replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarray: - """Normalize GRASS CELL nulls to NaN. + def read_raster_map(self, raster_id: str) -> np.ndarray: + """Read a raster through GRASS 8.4's no-mask row APIs. - FCELL/DCELL nulls are already exposed as NaN by pygrass. CELL nulls are - returned as the int32 null sentinel cast to the target dtype. + The active mapset MASK must not alter model input values. The one + effective mask selected for the simulation is applied separately by + :meth:`get_npmask` and passed to itzi-core. """ - if raster_type != "CELL" or not np.issubdtype(array.dtype, np.floating): - return array - - null_sentinel = array.dtype.type(np.iinfo(np.int32).min) - array[array == null_sentinel] = np.nan - return array - - def write_raster_map(self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float) -> Self: - """Take a numpy array and write it to GRASS DB""" - assert isinstance(arr, np.ndarray), "arr not a np array!" - assert isinstance(rast_name, str), "not a string!" - assert isinstance(mkey, str), "not a string!" - if self.non_blocking_write: - self.write_raster_map_nonblocking(arr, rast_name, mkey, hmin) - else: - self.write_raster_map_blocking(arr, rast_name, mkey, hmin) - return self - - def write_raster_map_nonblocking( - self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float - ) -> Self: - mtype = self.grass_dtype(str(arr.dtype)) - assert isinstance(mtype, str), "not a string!" - q_obj = ( - arr.copy(), - copy.deepcopy(rast_name), - copy.deepcopy(mtype), - copy.deepcopy(mkey), - copy.deepcopy(hmin), - self.overwrite, + from grass.lib import raster as libraster + + name, mapset = split_identifier(raster_id) + if not mapset: + msgr.fatal(f"Raster <{raster_id}> not found") + + raster_type = libraster.Rast_map_type(name, mapset) + reader_data = { + libraster.CELL_TYPE: (libraster.Rast_allocate_c_buf, libraster.Rast_get_c_row_nomask), + libraster.FCELL_TYPE: (libraster.Rast_allocate_f_buf, libraster.Rast_get_f_row_nomask), + libraster.DCELL_TYPE: (libraster.Rast_allocate_d_buf, libraster.Rast_get_d_row_nomask), + }.get(raster_type) + if reader_data is None: + msgr.fatal(f"Raster <{raster_id}> has an unsupported data type") + allocate, read_row = reader_data + file_descriptor = libraster.Rast_open_old(name, mapset) + if file_descriptor < 0: + msgr.fatal(f"Raster <{raster_id}> cannot be opened") + buffer = allocate() + try: + array = np.empty((self.yr, self.xr), dtype=self.dtype) + for row_index in range(self.yr): + read_row(file_descriptor, buffer, row_index) + array[row_index] = np.ctypeslib.as_array(buffer, shape=(self.xr,)) + finally: + libraster.Rast_close(file_descriptor) + return replace_cell_null_sentinel( + { + libraster.CELL_TYPE: "CELL", + libraster.FCELL_TYPE: "FCELL", + libraster.DCELL_TYPE: "DCELL", + }[raster_type], + array, ) - self.raster_writer_queue.put(q_obj) - return self - def write_raster_map_blocking( - self, arr: np.ndarray, rast_name: str, mkey: str, hmin: float - ) -> Self: - mtype = self.grass_dtype(str(arr.dtype)) - assert isinstance(mtype, str), "not a string!" + def write_raster_map(self, arr: np.ndarray, raster_id: str, mkey: str, hmin: float) -> None: + """Take a numpy array and write it to GRASS DB""" + map_type = self.grass_dtype(str(arr.dtype)) with raster.RasterRow( - rast_name, mode="w", mtype=mtype, overwrite=self.overwrite + raster_id, + mapset=get_current_mapset(), + mode="w", + mtype=map_type, + overwrite=self.overwrite, ) as newraster: - newrow = raster.Buffer((arr.shape[1],), mtype=mtype) + newrow = raster.Buffer((arr.shape[1],), mtype=map_type) for row in arr: newrow[:] = row[:] newraster.put_row(newrow) # apply color table - apply_color_table(rast_name, mkey) + colors_rules = colors_rules_dict.get(mkey) + if colors_rules is not None: + gscript.run_command("r.colors", quiet=True, rules=colors_rules, map=raster_id) # set null values if mkey == "water_depth" and hmin > 0: - GrassInterface.set_null(rast_name, hmin) - return self + set_null(raster_id, hmin) def create_db_links( self, vect_map: VectorTopo, linking_elem: dict[str, DBLayerDescription] @@ -597,7 +417,7 @@ def create_db_links( vect_map.dblinks.add(dblink) # create table dbtable = dblink.table() - dbtable.create(layer_dscr.columns, overwrite=True) + dbtable.create(layer_dscr.columns, overwrite=self.overwrite) dblinks[layer_name] = DBLinkDescription(dblink.layer, dbtable) return dblinks @@ -605,9 +425,10 @@ def write_vector_map( self, topology: DrainageNetworkTopology, attributes: DrainageNetworkAttributes, - map_name: str, - ) -> Self: + map_id: str, + ) -> None: """Write a vector map to GRASS GIS""" + map_name = output_name(map_id) node_attributes = {node.node_id: node for node in attributes.nodes} link_attributes = {link.link_id: link for link in attributes.links} topology_node_ids = {node.node_id for node in topology.nodes} @@ -630,7 +451,9 @@ def write_vector_map( } # set category manually cat_num = 1 - with VectorTopo(map_name, mode="w", overwrite=self.overwrite) as vector_map: + with VectorTopo( + map_name, mapset=gutils.getenv("MAPSET"), mode="w", overwrite=self.overwrite + ) as vector_map: # create db links and tables dblinks = self.create_db_links(vector_map, linking_elements) # dict to keep DB infos to write DB after geometries @@ -673,21 +496,20 @@ def write_vector_map( for attr in attrs: dbtable.insert(attr) dbtable.conn.commit() - return self def register_maps_in_stds( self, stds_title: str, - stds_name: str, + stds_id: str, map_list: list[tuple[str, datetime | timedelta]], - stds_type: str, - t_type: str, - ): + stds_type: STDSType, + t_type: TemporalType, + ) -> None: """Create a STDS, create one mapdataset for each map and register them in the temporal database """ assert isinstance(stds_title, str), "not a string!" - assert isinstance(stds_name, str), "not a string!" + output_name(stds_id) assert isinstance(t_type, str), "not a string!" # Print message in case of decreased GRASS verbosity if msgr.verbosity() <= 2: @@ -697,7 +519,6 @@ def register_maps_in_stds( if stds_type == "stvds" and grass_verbosity != "-1": os.environ["GRASS_VERBOSE"] = "-1" # create stds - stds_id = self.format_id(stds_name) stds_desc = "" stds = tgis.open_new_stds( stds_id, @@ -709,13 +530,11 @@ def register_maps_in_stds( overwrite=self.overwrite, ) - # create MapDataset objects list + dataset_type = {"strds": tgis.RasterDataset, "stvds": tgis.VectorDataset}[stds_type] map_dts_lst = [] - for map_name, map_time in map_list: - # create MapDataset - map_id = self.format_id(map_name) - map_dts_type = {"strds": tgis.RasterDataset, "stvds": tgis.VectorDataset} - map_dts = map_dts_type[stds_type](map_id) + for map_id, map_time in map_list: + output_name(map_id) + map_dts = dataset_type(map_id) # load spatial data from map map_dts.load() # set time @@ -726,9 +545,6 @@ def register_maps_in_stds( elif t_type == TemporalType.ABSOLUTE: assert isinstance(map_time, datetime) map_dts.set_absolute_time(start_time=map_time) - else: - assert False, "unknown temporal type!" - # populate the list map_dts_lst.append(map_dts) # Finally register the maps t_unit = {TemporalType.RELATIVE: "seconds", TemporalType.ABSOLUTE: ""} @@ -744,4 +560,3 @@ def register_maps_in_stds( # Restore GRASS verbosity if stds_type == "stvds" and grass_verbosity != "-1": os.environ["GRASS_VERBOSE"] = grass_verbosity - return self diff --git a/src/itzi/grass/names.py b/src/itzi/grass/names.py new file mode 100644 index 00000000..21793d88 --- /dev/null +++ b/src/itzi/grass/names.py @@ -0,0 +1,27 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + + +def derived_record_name(dataset_id: str, record_index: int) -> str: + """Return the shared raster/vector child ID for one output record.""" + if record_index < 0: + raise ValueError("record_index must be non-negative") + name, separator, mapset = dataset_id.partition("@") + return f"{name}_{record_index:04d}{separator}{mapset}" + + +def derived_drainage_table_names(vector_id: str) -> tuple[str, str]: + """Return the table names created for a drainage vector child.""" + name = vector_id.partition("@")[0] + return (f"{name}_node", f"{name}_link") diff --git a/src/itzi/grass/session.py b/src/itzi/grass/session.py new file mode 100644 index 00000000..46bae5a6 --- /dev/null +++ b/src/itzi/grass/session.py @@ -0,0 +1,183 @@ +""" +Copyright (C) 2025-2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import itzi.messenger as msgr + +_initialized_temporal_session: tuple[int, str, tuple[str, str, str]] | None = None + + +@dataclass(frozen=True) +class GrassParams: + """Parameters for GRASS GIS session.""" + + grassdata: str | None = None + location: str | None = None + mapset: str | None = None + region: str | None = None + mask: str | None = None + grass_bin: str | None = None + + def __post_init__(self) -> None: + context = (self.grassdata, self.location, self.mapset) + if any(value is not None for value in context) and not all( + value is not None for value in context + ): + raise ValueError("GRASS database, location, and mapset must be supplied together") + + +class GrassSessionManager: + """Manages GRASS session lifecycle.""" + + def __init__(self, grass_params: GrassParams): + self.grass_params = grass_params + self.grass_session = None + + @staticmethod + def current_context() -> tuple[str, str, str] | None: + """Return the active GRASS context, or ``None`` outside a session.""" + # Importing PyGRASS without a GISRC can terminate through GRASS' C API. + # The environment guard keeps configuration parsing and external launches + # independent from GRASS imports. + if not os.environ.get("GISRC"): + return None + try: + from grass.pygrass.utils import getenv + + values = (getenv("GISDBASE"), getenv("LOCATION_NAME"), getenv("MAPSET")) + except (ImportError, RuntimeError, SystemExit): + return None + if not all(values): + return None + return values + + def _validate_requested_context(self) -> None: + """Ensure an explicit request matches the session selected for this process.""" + active = self.current_context() + if active is None: + msgr.fatal("Unable to determine the active GRASS context") + gisdb, location, mapset = ( + self.grass_params.grassdata, + self.grass_params.location, + self.grass_params.mapset, + ) + if gisdb is None: + return + requested_database = str(Path(gisdb).expanduser().resolve()) + active_database = str(Path(active[0]).expanduser().resolve()) + if (requested_database, location, mapset) != ( + active_database, + active[1], + active[2], + ): + msgr.fatal( + "Requested GRASS context does not match the active session " + f"({active_database}/{active[1]}/{active[2]})" + ) + + @staticmethod + def ensure_temporal_initialized() -> None: + """Initialize the temporal framework once for the active GRASS session.""" + global _initialized_temporal_session + + context = GrassSessionManager.current_context() + if context is None: + msgr.fatal("No active GRASS session for temporal initialization") + import grass.script as gscript + import grass.temporal as tgis + + gscript.set_raise_on_error(True) + session = (os.getpid(), os.environ["GISRC"], context) + if _initialized_temporal_session != session: + tgis.init(raise_fatal_error=True) + _initialized_temporal_session = session + tgis.set_raise_on_error(True) + + def open(self) -> None: + """Open a GRASS session if needed.""" + if self.current_context() is not None: + self._validate_requested_context() + self.ensure_temporal_initialized() + return + + gisdb = self.grass_params.grassdata + location = self.grass_params.location + mapset = self.grass_params.mapset + # Check if mandatory GRASS parameters are present + if not gisdb or not location or not mapset: + msgr.fatal("No GRASS parameters to create a session.") + + # Check if the given parameters exist and can be accessed + error_msg = "'{}' does not exist or does not have adequate permissions" + if not os.access(gisdb, os.R_OK): + msgr.fatal(error_msg.format(gisdb)) + elif not os.access(os.path.join(gisdb, location), os.R_OK): + msgr.fatal(error_msg.format(location)) + elif not os.access(os.path.join(gisdb, location, mapset), os.W_OK): + msgr.fatal(error_msg.format(mapset)) + + # Set GRASS python path + if self.grass_params.grass_bin: + grassbin = self.grass_params.grass_bin + else: + grassbin = "grass" + grass_cmd = [grassbin, "--config", "python_path"] + grass_python_path = subprocess.check_output(grass_cmd, text=True).strip() + sys.path.append(grass_python_path) + # Now we can import grass modules + import grass.script as gscript + + # set up session + self.grass_session = gscript.setup.init( + path=gisdb, location=location, mapset=mapset, grass_path=grassbin + ) + if not os.environ.get("GISRC"): + session_env = getattr(self.grass_session, "env", None) + if session_env and session_env.get("GISRC"): + os.environ.update(session_env) + try: + self._validate_requested_context() + self.ensure_temporal_initialized() + except Exception: + self.close() + raise + + def close(self) -> None: + """Stop GRASS session.""" + global _initialized_temporal_session + if self.grass_session is not None: + try: + self.grass_session.finish() + except Exception as e: + print(f"Warning: Error cleaning up GRASS session: {e}") + _initialized_temporal_session = None + self.grass_session = None + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + return False + + def __del__(self): + self.close() diff --git a/src/itzi/grass/utils.py b/src/itzi/grass/utils.py new file mode 100644 index 00000000..5da57dfb --- /dev/null +++ b/src/itzi/grass/utils.py @@ -0,0 +1,277 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Literal, NamedTuple + +import grass.pygrass.utils as gutils +import grass.script as gscript +import grass.temporal as tgis +import numpy as np +from grass.pygrass.gis import Mapset +from grass.pygrass.gis.region import Region +from itzi_core import INPUT_ARRAY_KEYS, DomainData + +import itzi.messenger as msgr +from itzi.ensemble.models import EffectiveMask, EnsembleError +from itzi.grass.session import GrassParams + +if TYPE_CHECKING: + from itzi.grass.interface import GrassInterface + +MIN_GRASS_VERSION = (8, 4) + + +type STDSType = Literal["strds", "stvds"] +type RasterMapType = Literal["CELL", "FCELL", "DCELL"] + + +class MapData(NamedTuple): + id: str + start_time: datetime + end_time: datetime + + +def file_exists(name: str) -> bool: + """Return True if an output exists in the current mapset.""" + if not name: + return False + map_id = qualify_output_id(name, get_current_mapset()) + map_name, mapset = split_identifier(map_id) + return ( + name_is_map(map_id) + or vector_exists(map_name, mapset) + or stds_exists(map_id, "strds") + or stds_exists(map_id, "stvds") + ) + + +def check_output_files(file_list: list[str]) -> None: + """Check if the output files exist""" + if gscript.overwrite() or not file_list: + return + for map_name in file_list: + if file_exists(map_name): + msgr.fatal(f"File {map_name} exists and will not be overwritten") + + +def resolve_effective_mask(mask: str | None) -> EffectiveMask: + """Select the effective mask descriptor without modifying the mapset MASK.""" + current_mapset = gutils.getenv("MAPSET") + if mask is not None: + name, requested_mapset = split_identifier(mask) + mapset = gutils.get_mapset_raster(name, requested_mapset or "") + if not mapset: + raise EnsembleError(f"explicit mask {mask!r} was not found") + return EffectiveMask("explicit", f"{name}@{mapset}") + if gutils.get_mapset_raster("MASK", current_mapset): + return EffectiveMask("active", f"MASK@{current_mapset}") + return EffectiveMask("none", None) + + +def qualify_output_id(name: str, mapset: str) -> str: + """Give a new output its destination mapset, never a searched mapset.""" + map_name, requested_mapset = split_identifier(name) + if requested_mapset and requested_mapset != mapset: + raise EnsembleError(f"output {name!r} must be in the current mapset {mapset!r}") + return f"{map_name}@{mapset}" + + +def output_name(map_id: str) -> str: + """Return the bare name required by GRASS spatial writers.""" + name, mapset = split_identifier(map_id) + if mapset != get_current_mapset(): + raise EnsembleError(f"output {map_id!r} must be in the current mapset") + return name + + +def get_current_mapset() -> str: + return gutils.getenv("MAPSET") + + +def name_is_stds(stds_id: str) -> bool: + """Return True if the qualified ID identifies a registered STRDS.""" + return bool(tgis.SpaceTimeRasterDataset(stds_id).is_in_db()) + + +def get_crs_wkt() -> str: + return gscript.read_command("g.proj", flags="fw") + + +def name_is_map(map_id: str) -> bool: + """return True if the given name is a map in the grass database + False if not + """ + name, mapset = split_identifier(map_id) + return raster_exists(name, mapset) + + +def ensure_min_version() -> None: + full = gscript.parse_command("g.version", flags="g")["version"] + major, minor = (int(part) for part in full.split(".")[:2]) + if (major, minor) < MIN_GRASS_VERSION: + required = f"{MIN_GRASS_VERSION[0]}.{MIN_GRASS_VERSION[1]}" + raise RuntimeError( + f"itzi requires at least GRASS {required}, found version {major}.{minor}" + ) + + +def active_grass_params(requested: GrassParams) -> GrassParams: + """Return the active session context, retaining requested region/mask/bin.""" + return GrassParams( + grassdata=str(Path(gutils.getenv("GISDBASE")).expanduser().resolve()), + location=gutils.getenv("LOCATION_NAME"), + mapset=gutils.getenv("MAPSET"), + region=requested.region, + mask=requested.mask, + grass_bin=requested.grass_bin, + ) + + +def read_domain(region_id: str | None) -> DomainData: + """Read the selected computational region without leaving it changed.""" + if gscript.locn_is_latlong(): + raise EnsembleError("latlong locations are not supported") + using_temp_region = region_id is not None + if using_temp_region: + gscript.use_temp_region() + gscript.run_command("g.region", region=region_id) + try: + region = Region() + if region.cols < 3 or region.rows < 3: + raise EnsembleError("GRASS Region should be at least 3 cells by 3 cells") + return DomainData( + north=region.north, + south=region.south, + east=region.east, + west=region.west, + rows=region.rows, + cols=region.cols, + crs_wkt=get_crs_wkt(), + ) + finally: + if using_temp_region: + gscript.del_temp_region() + + +def split_identifier(identifier: str) -> tuple[str, str]: + name, separator, mapset = identifier.partition("@") + if separator and (not name or not mapset or "@" in mapset): + raise EnsembleError(f"invalid GRASS identifier {identifier!r}") + return name, mapset + + +def resolve_input_identifier(identifier: str) -> tuple[str, Literal["raster", "strds"]]: + """Resolve both input namespaces and reject a visible ambiguity.""" + + name, requested_mapset = split_identifier(identifier) + raster_mapset = gutils.get_mapset_raster(name, requested_mapset or "") + raster_id = f"{name}@{raster_mapset}" if raster_mapset else None + visible_mapsets = [requested_mapset] if requested_mapset else Mapset().visible.read() + strds_id = None + for mapset in visible_mapsets: + candidate = f"{name}@{mapset}" + try: + is_strds = tgis.SpaceTimeRasterDataset(candidate).is_in_db() + except BaseException as error: + # A visible mapset may have no temporal database or may be + # read-inaccessible. It cannot contribute a STRDS candidate. + if isinstance(error, (KeyboardInterrupt, GeneratorExit)): + raise + is_strds = False + if is_strds: + strds_id = candidate + break + if raster_id is not None and strds_id is not None: + raise EnsembleError( + f"input {identifier!r} is ambiguous: raster {raster_id} " + f"and STRDS {strds_id} both exist" + ) + if raster_id is not None: + return raster_id, "raster" + if strds_id is not None: + return strds_id, "strds" + raise EnsembleError(f"input {identifier!r} was not found as a raster or STRDS") + + +def is_clean_name(name: str) -> bool: + """Return True if name is a valid GRASS map name.""" + return bool(gutils.is_clean_name(name)) + + +def raster_exists(name: str, mapset: str) -> bool: + return bool(gutils.get_mapset_raster(name, mapset)) + + +def vector_exists(name: str, mapset: str) -> bool: + """Return True if a vector map exists in the given mapset.""" + return bool(gutils.get_mapset_vector(name, mapset)) + + +def stds_exists(stds_id: str, stds_type: STDSType) -> bool: + """Return True if a space-time dataset is registered in the database.""" + return bool(tgis.dataset_factory(stds_type, stds_id).is_in_db()) + + +def set_null(map_id: str, threshold: float) -> None: + """Set null values under a given threshold""" + gscript.run_command("r.null", flags="f", map=map_id, setnull=f"0.0-{threshold}") + + +def replace_cell_null_sentinel(raster_type: str, array: np.ndarray) -> np.ndarray: + """Normalize GRASS CELL nulls to NaN. + + FCELL/DCELL nulls are already exposed as NaN by pygrass. CELL nulls are + returned as the int32 null sentinel cast to the target dtype. + """ + if raster_type != "CELL" or not np.issubdtype(array.dtype, np.floating): + return array + + null_sentinel = array.dtype.type(np.iinfo(np.int32).min) + array[array == null_sentinel] = np.nan + return array + + +def resolve_input_map_lists( + grass_interface: GrassInterface, + map_names: Mapping[str, str], + start_time: datetime, + end_time: datetime, + input_kinds: Mapping[str, Literal["raster", "strds"]], +) -> dict[str, list[MapData] | None]: + """Resolve provider-ready raster lists without creating an output provider.""" + invalid_input_keys = sorted(set(map_names) - INPUT_ARRAY_KEYS) + if invalid_input_keys: + raise ValueError(f"Invalid input keys found: {', '.join(invalid_input_keys)}") + + map_lists: dict[str, list[MapData] | None] = {arr_key: None for arr_key in INPUT_ARRAY_KEYS} + for key, map_name in map_names.items(): + if not map_name: + continue + kind = input_kinds[key] + if kind == "strds": + if not name_is_stds(map_name): + msgr.fatal(f"STRDS input <{map_name}> is no longer available") + elif kind == "raster" and not name_is_map(map_name): + msgr.fatal(f"raster input <{map_name}> is no longer available") + if kind == "strds": + map_list = grass_interface.raster_list_from_strds(map_name) + else: + map_list = [MapData(id=map_name, start_time=start_time, end_time=end_time)] + map_lists[key] = map_list + return map_lists diff --git a/src/itzi/grass_session.py b/src/itzi/grass_session.py deleted file mode 100644 index 492a0a05..00000000 --- a/src/itzi/grass_session.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Copyright (C) 2025-2026 Laurent Courty - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation; either version 2 -of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. -""" - -from __future__ import annotations - -import importlib.util -import os -import subprocess -import sys - -from pydantic import BaseModel, ConfigDict - -import itzi.messenger as msgr - - -class GrassParams(BaseModel): - """Parameters for GRASS GIS session.""" - - model_config = ConfigDict(frozen=True) - - grassdata: str | None = None - location: str | None = None - mapset: str | None = None - region: str | None = None - mask: str | None = None - grass_bin: str | None = None - - -class GrassSessionManager: - """Manages GRASS session lifecycle.""" - - def __init__(self, grass_params: GrassParams): - self.grass_params = grass_params - self.grass_session = None - if importlib.util.find_spec("grass"): - self._is_active = True - else: - self._is_active = False - - def open(self): - """Open a GRASS session if needed.""" - if self._is_active: - return # Already started - - # Check if mandatory GRASS parameters are present - if not all( - [self.grass_params.grassdata, self.grass_params.location, self.grass_params.mapset] - ): - msgr.fatal("No GRASS parameters to create a session.") - - gisdb = self.grass_params.grassdata - location = self.grass_params.location - mapset = self.grass_params.mapset - - # Check if the given parameters exist and can be accessed - error_msg = "'{}' does not exist or does not have adequate permissions" - if not os.access(gisdb, os.R_OK): - msgr.fatal(error_msg.format(gisdb)) - elif not os.access(os.path.join(gisdb, location), os.R_OK): - msgr.fatal(error_msg.format(location)) - elif not os.access(os.path.join(gisdb, location, mapset), os.W_OK): - msgr.fatal(error_msg.format(mapset)) - - # Set GRASS python path - if self.grass_params.grass_bin: - grassbin = self.grass_params.grass_bin - else: - grassbin = "grass" - grass_cmd = [grassbin, "--config", "python_path"] - grass_python_path = subprocess.check_output(grass_cmd, text=True).strip() - sys.path.append(grass_python_path) - # Now we can import grass modules - import grass.script as gscript - - # set up session - self.grass_session = gscript.setup.init( - path=gisdb, location=location, mapset=mapset, grass_path=grassbin - ) - - self._is_active = True - - def close(self): - """Stop GRASS session.""" - if self.grass_session is not None and self._is_active: - try: - self.grass_session.finish() - except Exception as e: - print(f"Warning: Error cleaning up GRASS session: {e}") - self._is_active = False - - def __enter__(self): - self.open() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - return False - - def __del__(self): - self.close() diff --git a/src/itzi/itzi.py b/src/itzi/itzi.py index 084f2aa9..03dcc24b 100644 --- a/src/itzi/itzi.py +++ b/src/itzi/itzi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ NAME: Itzï @@ -29,27 +28,36 @@ import os import sys import time +from argparse import Namespace from collections.abc import Callable -from datetime import datetime, timedelta +from concurrent.futures import ProcessPoolExecutor +from datetime import timedelta from importlib.metadata import version -from multiprocessing import Process -from typing import TYPE_CHECKING - -import numpy as np -from itzi_core import SimulationBuilder -from itzi_core.providers import CSVMassBalanceOutputProvider +from multiprocessing import Process, get_context +from pathlib import Path import itzi.messenger as msgr from itzi.cli_parser import build_parser from itzi.configreader import ConfigReader -from itzi.grass_session import GrassSessionManager +from itzi.ensemble.models import ( + EnsembleError, + ExpandedSimulation, + ResolvedEnsemble, + ResolvedSimulation, + ValidationFailure, +) +from itzi.ensemble.resolution import resolve_ensemble, validate_resolved_ensemble +from itzi.grass.session import GrassSessionManager +from itzi.manifest import ( + _create_manifest, + _initial_member_states, + _manifest_path, + _update_manifest, + _validate_manifest_destination, +) from itzi.messenger import VerbosityLevel - -if TYPE_CHECKING: - from itzi_core import Simulation, SimulationConfig - - from itzi.grass_session import GrassParams - from itzi.providers.grass_interface import GrassInterface +from itzi.run_plan import load_batch +from itzi.simulation_runner import SimulationRunner def main(argv: list[str] | None = None) -> int: @@ -69,140 +77,6 @@ def main(argv: list[str] | None = None) -> int: return 0 -class SimulationRunner: - """Provide the necessary tools to run one simulation.""" - - def __init__( - self, - sim_config: SimulationConfig, - grass_params: GrassParams, - hotstart_path: str | None = None, - stats_file: str | None = None, - ) -> None: - self.grass_required_version = "8.4.0" - self.g_interface: GrassInterface - self.sim: Simulation - - # display parameters (if verbose) - msgr.display_sim_param(sim_config) - - # Check GRASS version - import grass.script as gscript - - grass_version = gscript.parse_command("g.version", flags="g")["version"] - if grass_version < self.grass_required_version: - msgr.fatal( - f"Itzi requires at least GRASS {self.grass_required_version}, " - f"version {grass_version} detected." - ) - msgr.debug("GRASS session set") - - # return error if output files exist - from itzi.providers import grass_interface - - output_names = list(sim_config.output_map_names.values()) - if sim_config.drainage_output is not None: - output_names.append(sim_config.drainage_output) - grass_interface.check_output_files(output_names) - msgr.debug("Output files OK") - - data_type = np.float32 - # Create the grass_interface object - self.g_interface = grass_interface.GrassInterface( - start_time=sim_config.start_time, - end_time=sim_config.end_time, - dtype=data_type, - region_id=grass_params.region, - raster_mask_id=grass_params.mask, - non_blocking_write=False, - ) - # Create Simulation with GRASS backend - msgr.verbose("Setting up GRASS simulation...") - from itzi.providers.grass_input import GrassRasterInputProvider - from itzi.providers.grass_output import ( - GrassRasterOutputProvider, - GrassVectorOutputProvider, - ) - - raster_input_provider = GrassRasterInputProvider( - { - "grass_interface": self.g_interface, - "input_map_names": sim_config.input_map_names, - "default_start_time": sim_config.start_time, - "default_end_time": sim_config.end_time, - } - ) - raster_output_provider = GrassRasterOutputProvider( - { - "grass_interface": self.g_interface, - "out_map_names": sim_config.output_map_names, - "hmin": sim_config.surface_flow_parameters.hmin, - "temporal_type": sim_config.temporal_type, - } - ) - sim_builder = ( - SimulationBuilder(sim_config, self.g_interface.get_npmask(), data_type) - .with_input_provider(raster_input_provider) - .with_raster_output_provider(raster_output_provider) - ) - if sim_config.drainage_output is not None: - vector_output_provider = GrassVectorOutputProvider( - { - "grass_interface": self.g_interface, - "temporal_type": sim_config.temporal_type, - "drainage_map_name": sim_config.drainage_output, - } - ) - sim_builder.with_vector_output_provider(vector_output_provider) - if stats_file: - sim_builder.with_mass_balance_output_provider(CSVMassBalanceOutputProvider(stats_file)) - if hotstart_path: - sim_builder.with_hotstart(hotstart_path) - self.sim: Simulation = sim_builder.build() - # Initialize the simulation - self.sim.initialize() - - def run(self): - """Run a full simulation""" - sim_start_time = datetime.now() - msgr.verbose("Starting time-stepping...") - while self.sim.sim_time < self.sim.end_time: - # display advance of simulation - msgr.percent( - self.sim.start_time, - self.sim.end_time, - self.sim.sim_time, - sim_start_time, - ) - # step models - self.step() - return self - - def finalize(self): - """Tear down the simulation and return to previous state.""" - self.sim.finalize() - # Cleanup the grass interface object - if hasattr(self, "g_interface"): - self.g_interface.finalize() - self.g_interface.cleanup() - return self - - def step(self): - """Do one simulation step.""" - self.sim.update() - return self - - @property - def origin(self): - return (self.sim.domain_data.north, self.sim.domain_data.west) - - def __del__(self): - # Cleanup the grass interface object - if hasattr(self, "g_interface"): - self.g_interface.finalize() - self.g_interface.cleanup() - - def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: """Run one simulation""" msgr.raise_on_error = True @@ -211,16 +85,16 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: # Run the simulation msgr.message(f"Starting simulation of {os.path.basename(conf_file)}...") conf_data = ConfigReader(conf_file) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params with GrassSessionManager(grass_params): sim_runner = SimulationRunner( sim_params, grass_params, hotstart_path=hotstart_file, - stats_file=conf_data.get_stats_file(), + stats_file=conf_data.stats_file, ) - sim_runner.run().finalize() + sim_runner.initialize().run().finalize() except msgr.FatalError: raise SystemExit(1) from None except SystemExit as error: @@ -232,6 +106,66 @@ def sim_runner_worker(conf_file: str, hotstart_file: str | None) -> None: raise SystemExit(1) from None +def resolved_sim_runner_worker(simulation: ResolvedSimulation) -> tuple[str, str | None]: + """Run one resolved member and return its status.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + try: + with GrassSessionManager(simulation.grass_params): + runner = SimulationRunner( + simulation.simulation_config, + simulation.grass_params, + stats_file=str(simulation.artifacts.statistics_file) + if simulation.artifacts.statistics_file is not None + else None, + effective_mask=simulation.effective_mask, + input_kinds=dict(simulation.input_kinds), + ) + runner.initialize() + try: + runner.run() + except Exception: + try: + runner.finalize() + except Exception as finalize_error: + msgr.warning( + f"Could not finalize partial outputs: " + f"{type(finalize_error).__name__}: {finalize_error}" + ) + raise + runner.finalize() + return "completed", None + except Exception as error: + detail = f"{type(error).__name__}: {error}" + msgr.warning(f"Simulation failed: {detail}") + return "execution_failed", detail + + +def resolver_worker( + expanded: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: + """Resolve one ensemble in a short-lived spawned GRASS process.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + return resolve_ensemble(expanded) + + +def run_validation_worker(simulations: tuple[ResolvedSimulation, ...]) -> dict[str, str]: + """Validate selected members in one GRASS session, retaining individual failures.""" + msgr.raise_on_error = True + msgr._itzi_logger.set_verbosity(msgr.verbosity()) + failures: dict[str, str] = {} + with GrassSessionManager(simulations[0].grass_params): + from itzi.ensemble.run_validation import validate_run_requirements + + for simulation in simulations: + try: + validate_run_requirements(simulation) + except Exception as error: + failures[simulation.simulation_id] = f"{type(error).__name__}: {error}" + return failures + + def itzi_run_one(conf_file: str, hotstart_file: str | None) -> bool: """Run a simulation in a subprocess""" worker_args = (conf_file, hotstart_file) @@ -341,6 +275,25 @@ def itzi_run(cli_args): # only warnings os.environ["GRASS_VERBOSE"] = "0" + if _uses_legacy_execution_path(cli_args): + _run_legacy_batch(cli_args) + return + _run_ensemble_batch(cli_args) + + +def _uses_legacy_execution_path(cli_args) -> bool: + """Keep the one-simulation INI execution API during its deprecation window.""" + return ( + all(Path(path).suffix != ".yaml" for path in cli_args.config_file) + and not getattr(cli_args, "member", []) + and not getattr(cli_args, "dry", False) + ) + + +def _run_legacy_batch(cli_args) -> None: + """Execute the retained pre-ensemble INI path.""" + for config_file in cli_args.config_file: + msgr.warning(f"INI configuration <{config_file}> is deprecated; use YAML instead.") # start total time counter total_sim_start = time.time() # dictionary to store computation times @@ -368,14 +321,301 @@ def itzi_run(cli_args): else: msgr.message("Simulation(s) complete. Elapsed times:") for f, t in times_list: - msgr.message("{}: {}".format(f, t)) - msgr.message("Total: {}".format(total_elapsed_time)) + msgr.message(f"{f}: {t}") + msgr.message(f"Total: {total_elapsed_time}") avg_time_s = int(total_elapsed_time.total_seconds() / len(times_list)) - msgr.message("Average: {}".format(timedelta(seconds=avg_time_s))) + msgr.message(f"Average: {timedelta(seconds=avg_time_s)}") if failed_files: msgr.fatal(f"{len(failed_files)} simulation(s) failed") +def _run_ensemble_batch(cli_args: Namespace) -> None: + """Resolve and execute each ensemble in order through spawn boundaries.""" + if getattr(cli_args, "resume_from", []): + msgr.fatal("Resume is not available for YAML ensembles.") + try: + ensembles, document_failures = load_batch(cli_args.config_file) + except EnsembleError as error: + msgr.fatal(str(error)) + for failure in document_failures: + msgr.warning(failure.format()) + + selectors = getattr(cli_args, "member", []) + has_selectors = bool(selectors) + if has_selectors and len(ensembles) != 1 and any("#" not in s for s in selectors): + msgr.fatal("Unqualified --member IDs require a batch with exactly one ensemble") + qualified_selectors = [ + (selector, selector if "#" in selector else f"{ensembles[0].ensemble_id}#{selector}") + for selector in selectors + ] + ensemble_ids = {ensemble.ensemble_id for ensemble in ensembles} + for selector, qualified in qualified_selectors: + if qualified.partition("#")[0] not in ensemble_ids: + msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") + + dry = getattr(cli_args, "dry", False) + dry_failed = bool(document_failures) + failure_count = len(document_failures) + for ensemble in ensembles: + successful: list[ResolvedSimulation] = [] + member_failures: list[ValidationFailure] = [] + for result in _resolve_ensemble_in_subprocess(ensemble.simulations): + if isinstance(result, ResolvedSimulation): + successful.append(result) + else: + member_failures.append(result) + artifact_failure = None + try: + validate_resolved_ensemble(tuple(successful), (ensemble.source.path,)) + except EnsembleError as error: + artifact_failure = str(error) + resolved = ResolvedEnsemble( + ensemble, tuple(successful), tuple(member_failures), artifact_failure + ) + selected_ids = _select_members(resolved, qualified_selectors) + run_validation_failures, manifest_failure = _validate_ensemble_run_requirements( + resolved, + selected_ids, + has_selectors=has_selectors, + overwrite=bool(getattr(cli_args, "o", False)), + ) + if dry: + _display_dry_plan( + resolved, + selected_ids, + run_validation_failures, + manifest_failure, + has_selectors=has_selectors, + ) + dry_failed |= bool( + (resolved.failures and not has_selectors) + or run_validation_failures + or manifest_failure + ) + continue + + failure_count += _run_one_ensemble( + resolved, + selected_ids, + run_validation_failures, + manifest_failure, + has_selectors=has_selectors, + overwrite=bool(cli_args.o), + ) + + if dry: + if dry_failed: + msgr.fatal("YAML batch validation failed") + return + + if failure_count: + msgr.fatal(f"{failure_count} ensemble simulation(s) failed validation or execution") + msgr.message("Simulation(s) complete.") + + +def _run_one_ensemble( + resolved: ResolvedEnsemble, + selected_ids: set[str], + run_validation_failures: dict[str, dict[str, str]], + manifest_failure: str | None, + *, + has_selectors: bool, + overwrite: bool, +) -> int: + """Run selected members and return the number of failures in this ensemble.""" + ensemble = resolved.ensemble + if not selected_ids and not (not has_selectors and resolved.failures): + return 0 + manifest_path = _manifest_path(ensemble) + states = _initial_member_states( + resolved.simulations, resolved.failures, selected_ids, has_selectors + ) + for simulation_id, failure in run_validation_failures.items(): + states[simulation_id]["status"] = "validation_failed" + states[simulation_id]["failure"] = failure + if manifest_failure is not None: + msgr.warning(manifest_failure) + return max(1, len(selected_ids)) + try: + _create_manifest(manifest_path, ensemble, states, overwrite=overwrite) + except OSError as error: + msgr.warning(f"Cannot create manifest <{manifest_path}>: {error}") + return max(1, len(selected_ids)) + + failure_count = 0 + member_started = False + for simulation in resolved.simulations: + if simulation.simulation_id not in selected_ids: + continue + if states[simulation.simulation_id]["status"] != "planned": + continue + if member_started: + detail = _validate_run_requirements_in_subprocess((simulation,)).get( + simulation.simulation_id + ) + if detail is not None: + states[simulation.simulation_id]["status"] = "validation_failed" + states[simulation.simulation_id]["failure"] = { + "phase": "run_validation", + "detail": detail, + } + _update_manifest(manifest_path, ensemble, states) + continue + states[simulation.simulation_id]["status"] = "running" + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + started = time.monotonic() + member_started = True + status, detail = _run_simulation_in_subprocess(simulation) + state = states[simulation.simulation_id] + state["status"] = status + state["elapsed_seconds"] = time.monotonic() - started + if detail is not None: + states[simulation.simulation_id]["failure"] = { + "phase": "execution", + "detail": detail, + } + failure_count += 1 + try: + _update_manifest(manifest_path, ensemble, states) + except OSError as error: + msgr.warning(f"Cannot update manifest <{manifest_path}>: {error}") + failure_count += 1 + break + failure_count += sum( + 1 + for state in states.values() + if state["status"] == "validation_failed" and state.get("selected", False) + ) + return failure_count + + +def _resolve_ensemble_in_subprocess( + expanded: tuple[ExpandedSimulation, ...], +) -> tuple[ResolvedSimulation | ValidationFailure, ...]: + try: + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(resolver_worker, expanded).result() + except Exception as error: + detail = f"{type(error).__name__}: {error}" + return tuple( + ValidationFailure( + coordinates=simulation.coordinates, + phase="input_resolution", + detail=detail, + ) + for simulation in expanded + ) + + +def _run_simulation_in_subprocess(simulation: ResolvedSimulation) -> tuple[str, str | None]: + try: + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(resolved_sim_runner_worker, simulation).result() + except Exception as error: + return "execution_failed", f"{type(error).__name__}: {error}" + + +def _validate_run_requirements_in_subprocess( + simulations: tuple[ResolvedSimulation, ...], +) -> dict[str, str]: + try: + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as executor: + return executor.submit(run_validation_worker, simulations).result() + except Exception as error: + detail = f"{type(error).__name__}: {error}" + return {simulation.simulation_id: detail for simulation in simulations} + + +def _validate_ensemble_run_requirements( + resolved: ResolvedEnsemble, + selected_ids: set[str], + *, + has_selectors: bool, + overwrite: bool, +) -> tuple[dict[str, dict[str, str]], str | None]: + """Collect one ensemble's selected-member and manifest failures.""" + if not selected_ids and has_selectors: + return {}, None + ensemble = resolved.ensemble + simulations = resolved.simulations + failures: dict[str, dict[str, str]] = {} + if resolved.artifact_failure is not None: + for simulation_id in selected_ids: + failures[simulation_id] = { + "phase": "artifact_validation", + "detail": resolved.artifact_failure, + } + manifest_failure = None + try: + _validate_manifest_destination( + _manifest_path(ensemble), ensemble, simulations, overwrite=overwrite + ) + except EnsembleError as error: + manifest_failure = str(error) + to_validate = tuple( + simulation + for simulation in simulations + if simulation.simulation_id in selected_ids and simulation.simulation_id not in failures + ) + if to_validate: + for simulation_id, detail in _validate_run_requirements_in_subprocess(to_validate).items(): + failures[simulation_id] = {"phase": "run_validation", "detail": detail} + return failures, manifest_failure + + +def _select_members( + resolved: ResolvedEnsemble, + selectors: list[tuple[str, str]], +) -> set[str]: + """Select successfully resolved members of one ensemble.""" + all_members = {simulation.simulation_id for simulation in resolved.simulations} + if not selectors: + return all_members + selected: set[str] = set() + for selector, qualified in selectors: + ensemble_id, _, simulation_id = qualified.partition("#") + if ensemble_id != resolved.ensemble.ensemble_id: + continue + if simulation_id not in all_members: + msgr.fatal(f"--member {selector!r} does not match a successfully resolved simulation") + selected.add(simulation_id) + return selected + + +def _display_dry_plan( + resolved: ResolvedEnsemble, + selected_ids: set[str], + run_validation_failures: dict[str, dict[str, str]], + manifest_failure: str | None, + *, + has_selectors: bool, +) -> None: + ensemble = resolved.ensemble + msgr.message(f"Ensemble {ensemble.ensemble_id}:") + for simulation in resolved.simulations: + selection = "selected" if simulation.simulation_id in selected_ids else "not selected" + msgr.message(f" {simulation.simulation_id} ({selection})") + if simulation.simulation_id in run_validation_failures: + failure = run_validation_failures[simulation.simulation_id] + msgr.warning(f" {failure['phase']} failed: {failure['detail']}") + for variable, name in simulation.artifacts.output_map_names: + msgr.message(f" raster {variable}: {name}") + if simulation.artifacts.drainage_output is not None: + msgr.message(f" drainage: {simulation.artifacts.drainage_output}") + if simulation.artifacts.statistics_file is not None: + msgr.message(f" statistics: {simulation.artifacts.statistics_file}") + for failure in resolved.failures: + status = "not selected" if has_selectors else "validation failed" + msgr.warning(f" {status}: {failure.detail}") + if manifest_failure is not None: + msgr.warning(f" manifest validation failed: {manifest_failure}") + + def itzi_version(cli_args): """Display the software version number from the installed version""" print(version("itzi")) diff --git a/src/itzi/manifest.py b/src/itzi/manifest.py new file mode 100644 index 00000000..70d9c8cf --- /dev/null +++ b/src/itzi/manifest.py @@ -0,0 +1,162 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import os +import tempfile +from datetime import datetime +from pathlib import Path +from typing import cast + +import yaml + +from itzi.ensemble.models import ( + EnsembleError, + ExpandedEnsemble, + JsonValue, + ResolvedSimulation, + ValidationFailure, + render_template, +) +from itzi.ensemble.resolution import _validate_protected_file_output + + +def _manifest_path(ensemble: ExpandedEnsemble) -> Path: + if ensemble.manifest_template is not None: + rendered = render_template( + ensemble.manifest_template, ensemble=ensemble.ensemble_id, simulation=None + ) + path = Path(rendered).expanduser() + return (path if path.is_absolute() else ensemble.source.path.parent / path).resolve() + return ensemble.source.path.parent / "results" / f"{ensemble.ensemble_id}.manifest.yaml" + + +def validate_file_destination(path: Path, *, overwrite: bool, description: str) -> None: + """Check a filesystem output destination without creating it or its parents.""" + if path.exists() or path.is_symlink(): + if not path.is_file(): + raise EnsembleError(f"{description} <{path}> is not a regular file") + if not overwrite: + raise EnsembleError(f"{description} <{path}> already exists") + if not os.access(path, os.W_OK): + raise EnsembleError(f"{description} <{path}> is not writable") + + parent = path.parent + while not parent.exists(): + parent = parent.parent + if not parent.is_dir() or not os.access(parent, os.W_OK | os.X_OK): + raise EnsembleError(f"parent directory <{parent}> for {description} is not writable") + + +def _validate_manifest_destination( + manifest_path: Path, + ensemble: ExpandedEnsemble, + simulations: tuple[ResolvedSimulation, ...], + *, + overwrite: bool, +) -> None: + """Keep the parent-owned manifest away from files required by the run.""" + protected = {ensemble.source.path.resolve()} + for simulation in simulations: + if simulation.simulation_config.swmm_inp is not None: + protected.add(simulation.simulation_config.swmm_inp.resolve()) + if simulation.artifacts.statistics_file is not None: + protected.add(simulation.artifacts.statistics_file.resolve()) + _validate_protected_file_output(manifest_path, protected) + validate_file_destination(manifest_path, overwrite=overwrite, description="manifest") + + +def _initial_member_states( + simulations: tuple[ResolvedSimulation, ...], + failures: tuple[ValidationFailure, ...], + selected_ids: set[str], + has_selectors: bool, +) -> dict[str, dict]: + states: dict[str, dict] = {} + for simulation in simulations: + selected = simulation.simulation_id in selected_ids + states[simulation.simulation_id] = { + "simulation_id": simulation.simulation_id, + "selected": selected, + "status": "planned" if selected else "not_selected", + "elapsed_seconds": None, + "coordinates": _manifest_coordinates(simulation.coordinates), + "artifacts": { + "rasters": dict(simulation.artifacts.output_map_names), + "drainage": simulation.artifacts.drainage_output, + "statistics": str(simulation.artifacts.statistics_file) + if simulation.artifacts.statistics_file is not None + else None, + }, + } + for index, failure in enumerate(failures): + key = f"validation-{index}" + states[key] = { + "simulation_id": None, + "selected": not has_selectors, + "status": "not_selected" if has_selectors else "validation_failed", + "elapsed_seconds": None, + "coordinates": _manifest_coordinates(failure.coordinates), + "failure": {"phase": failure.phase, "detail": failure.detail}, + } + return states + + +def _manifest_coordinates( + coordinates: tuple[tuple[str, JsonValue], ...], +) -> dict[str, JsonValue | dict[str, JsonValue]]: + return { + path: dict(cast(tuple[tuple[str, JsonValue], ...], value)) + if isinstance(value, tuple) + else value + for path, value in coordinates + } + + +def _manifest_document(ensemble: ExpandedEnsemble, states: dict[str, dict]) -> dict: + return { + "manifest_version": 1, + "last_updated_at": datetime.now().astimezone().isoformat(), + "ensemble": {"id": ensemble.ensemble_id, "description": ensemble.ensemble_description}, + "source": { + "path": str(ensemble.source.path), + "document_index": ensemble.source.document_index, + }, + "members": list(states.values()), + } + + +def _create_manifest( + path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict], *, overwrite: bool +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + mode = "w" if overwrite else "x" + with path.open(mode, encoding="utf-8") as file_obj: + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) + + +def _update_manifest(path: Path, ensemble: ExpandedEnsemble, states: dict[str, dict]) -> None: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as file_obj: + temporary_path = Path(file_obj.name) + yaml.safe_dump(_manifest_document(ensemble, states), file_obj, sort_keys=False) + file_obj.flush() + os.fsync(file_obj.fileno()) + try: + temporary_path.replace(path) + except Exception: + temporary_path.unlink(missing_ok=True) + raise diff --git a/src/itzi/messenger.py b/src/itzi/messenger.py index 92785f20..e482cfac 100644 --- a/src/itzi/messenger.py +++ b/src/itzi/messenger.py @@ -140,7 +140,7 @@ def percent(start_time, end_time, sim_time, sim_start_time): advance_perc = sim_time_s / duration_s if verbosity() == VerbosityLevel.QUIET: - print(f"{advance_perc:.1%}", file=sys.stderr, end="\r") + print(f"{advance_perc:.2%}", file=sys.stderr, end="\r") elif verbosity() >= VerbosityLevel.MESSAGE: elapsed_s = (datetime.now() - sim_start_time).total_seconds() @@ -150,7 +150,7 @@ def percent(start_time, end_time, sim_time, sim_start_time): rate = 0 remaining = (end_time - sim_time).total_seconds() eta = timedelta(seconds=int(remaining * rate)) - txt = "Time: {sim} Advance: {perc:.1%} ETA: {eta}{pad}" + txt = "Time: {sim} Advance: {perc:.2%} ETA: {eta}{pad}" disp = txt.format( sim=sim_time.isoformat(" ").split(".")[0], perc=advance_perc, diff --git a/src/itzi/providers/csv_output.py b/src/itzi/providers/csv_output.py new file mode 100644 index 00000000..5300cd63 --- /dev/null +++ b/src/itzi/providers/csv_output.py @@ -0,0 +1,49 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import csv +import numbers +from pathlib import Path + +from itzi_core.data_containers import MassBalanceData +from itzi_core.providers.base import MassBalanceOutputProvider + + +class ExclusiveCSVMassBalanceOutputProvider(MassBalanceOutputProvider): + """Write statistics without silently truncating a pre-existing destination.""" + + def __init__( + self, file_name: str | Path, *, overwrite: bool, float_format: str = ".3f" + ) -> None: + self.fields = list(MassBalanceData.model_fields.keys()) + self.file_name = Path(file_name) + self.file_name.parent.mkdir(parents=True, exist_ok=True) + self.float_format = float_format + mode = "w" if overwrite else "x" + with self.file_name.open(mode, newline="") as file_obj: + csv.DictWriter(file_obj, fieldnames=self.fields).writeheader() + + def log(self, report_data: MassBalanceData) -> None: + line_to_write = {} + for key, value in report_data.model_dump().items(): + if value != value: # noqa: PLR0124 + line_to_write[key] = "-" + elif isinstance(value, numbers.Real) and not isinstance(value, int): + line_to_write[key] = format(value, self.float_format) + else: + line_to_write[key] = value + with self.file_name.open("a", newline="") as file_obj: + csv.DictWriter(file_obj, fieldnames=self.fields).writerow(line_to_write) diff --git a/src/itzi/providers/grass_input.py b/src/itzi/providers/grass_input.py index 4a6f7211..ba98ba98 100644 --- a/src/itzi/providers/grass_input.py +++ b/src/itzi/providers/grass_input.py @@ -15,89 +15,42 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, Literal import numpy as np -from itzi_core import ARRAY_DEFINITIONS, ArrayCategory, DomainData +from itzi_core import DomainData from itzi_core.providers import RasterInputProvider -import itzi.messenger as msgr -from itzi.providers.grass_interface import MapData - if TYPE_CHECKING: from datetime import datetime - from itzi.providers.grass_interface import GrassInterface - - -class GrassRasterInputConfig(TypedDict): - grass_interface: GrassInterface - # A dict of key: input names - input_map_names: Mapping[str, str | None] - default_start_time: datetime - default_end_time: datetime + from itzi.grass.interface import GrassInterface class GrassRasterInputProvider(RasterInputProvider): - def __init__(self, config: GrassRasterInputConfig) -> None: - self.grass_interface = config["grass_interface"] - self.start_time = config["default_start_time"] - self.end_time = config["default_end_time"] - self.map_lists = self._get_map_lists(config["input_map_names"]) - - def get_domain_data(self) -> DomainData: - """Return a DomainData object""" - return DomainData( - north=self.grass_interface.region.north, - south=self.grass_interface.region.south, - east=self.grass_interface.region.east, - west=self.grass_interface.region.west, - rows=self.grass_interface.region.rows, - cols=self.grass_interface.region.cols, - crs_wkt=self.grass_interface.get_crs_wkt(), + def __init__( + self, + grass_interface: GrassInterface, + input_map_names: Mapping[str, str], + default_start_time: datetime, + default_end_time: datetime, + input_kinds: Mapping[str, Literal["raster", "strds"]], + ) -> None: + from itzi.grass.utils import resolve_input_map_lists + + self.grass_interface = grass_interface + self.start_time = default_start_time + self.end_time = default_end_time + self.map_lists = resolve_input_map_lists( + grass_interface, + input_map_names, + default_start_time, + default_end_time, + input_kinds, ) - def _get_map_lists( - self, map_names: Mapping[str, str | None] - ) -> Mapping[str, list[MapData] | None]: - """Read maps names from GIS. - input map_names is a dictionary of maps/STDS names - for each entry in map_names: - if the name is empty or None, store None - if a strds, load all maps in the instance's time extend, - store them as a list - if a single map, set the start and end time to fit simulation. - store it in a list for consistency - each map is stored as a MapData namedtuple - store result in instance's dictionary - """ - map_lists: dict[str, list[MapData] | None] = { - arr_def.key: None - for arr_def in ARRAY_DEFINITIONS - if ArrayCategory.INPUT in arr_def.category - } - for k, map_name in map_names.items(): - if not map_name: - map_list = None - continue - map_id = self.grass_interface.format_id(map_name) - if self.grass_interface.name_is_stds(map_id): - strds_id = map_id - if not self.grass_interface.stds_temporal_sanity(strds_id): - msgr.fatal(f"{map_name}: inadequate temporal format") - map_list = self.grass_interface.raster_list_from_strds(strds_id) - elif self.grass_interface.name_is_map(map_id): - map_list = [ - MapData( - id=map_id, - start_time=self.start_time, - end_time=self.end_time, - ) - ] - else: - msgr.fatal(f"{map_name} not found!") - map_lists[k] = map_list - return map_lists + def get_domain_data(self) -> DomainData: + return self.grass_interface.get_domain_data() def get_array( self, map_key: str, current_time: datetime diff --git a/src/itzi/providers/grass_output.py b/src/itzi/providers/grass_output.py index 0c239198..af2244df 100644 --- a/src/itzi/providers/grass_output.py +++ b/src/itzi/providers/grass_output.py @@ -15,46 +15,47 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING import numpy as np +from itzi_core import OUTPUT_ARRAY_KEYS from itzi_core.providers import RasterOutputProvider, VectorOutputProvider +from itzi.grass.names import derived_record_name + if TYPE_CHECKING: from datetime import datetime, timedelta from itzi_core import TemporalType from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology - from itzi.providers.grass_interface import GrassInterface - - -class GrassRasterOutputConfig(TypedDict): - grass_interface: GrassInterface - out_map_names: Mapping[str, str] - hmin: float - temporal_type: TemporalType - - -class GrassVectorOutputConfig(TypedDict): - grass_interface: GrassInterface - drainage_map_name: str - temporal_type: TemporalType + from itzi.grass.interface import GrassInterface class GrassRasterOutputProvider(RasterOutputProvider): """Write simulation outputs to GRASS.""" - def __init__(self, config: GrassRasterOutputConfig) -> None: - """Initialize output provider with configuration.""" - self.grass_interface = config["grass_interface"] - # user-selected map names. Keys are user-facing names - self.out_map_names = config["out_map_names"] - self.hmin = config["hmin"] - self.temporal_type = config["temporal_type"] - for stds_name in self.out_map_names.values(): + def __init__( + self, + grass_interface: GrassInterface, + out_map_names: Mapping[str, str], + hmin: float, + temporal_type: TemporalType, + ) -> None: + invalid_output_keys = sorted(set(out_map_names) - OUTPUT_ARRAY_KEYS) + if invalid_output_keys: + raise ValueError( + f"out_map_names contains invalid input keys: {', '.join(invalid_output_keys)}" + ) + + self.grass_interface = grass_interface + # Dataset IDs qualified in the current mapset. Keys are output variables. + self.out_map_names = out_map_names + self.hmin = hmin + self.temporal_type = temporal_type + for stds_id in self.out_map_names.values(): self.grass_interface.validate_output_stds_temporal_type( - stds_name=stds_name, + stds_id=stds_id, stds_type="strds", expected_temporal_type=self.temporal_type, ) @@ -64,16 +65,14 @@ def __init__(self, config: GrassRasterOutputConfig) -> None: def _write_array( self, array: np.ndarray, map_key: str, sim_time: datetime | timedelta ) -> None: - """Write simulation data for current time step.""" - suffix = str(self.record_counter[map_key]).zfill(4) - map_name = f"{self.out_map_names[map_key]}_{suffix}" + map_id = derived_record_name(self.out_map_names[map_key], self.record_counter[map_key]) # write the raster - self.grass_interface.write_raster_map(array, map_name, map_key, self.hmin) + self.grass_interface.write_raster_map(array, map_id, map_key, self.hmin) # Set depth values to null under the given threshold. Temporarily in gis.py # if map_key == "water_depth": # self.grass_interface.set_null(map_name, self.hmin) # add map name and time to the corresponding list - self.output_maplist[map_key].append((map_name, sim_time)) + self.output_maplist[map_key].append((map_id, sim_time)) self.record_counter[map_key] += 1 def write_arrays( @@ -86,28 +85,28 @@ def write_arrays( def finalize(self) -> None: """Finalize outputs and cleanup.""" - # Write the final raster maps - self.grass_interface.finalize() # register in GRASS temporal framework for map_key, lst in self.output_maplist.items(): - strds_name = self.out_map_names[map_key] - if strds_name is None: - continue + strds_id = self.out_map_names[map_key] self.grass_interface.register_maps_in_stds( - map_key, strds_name, lst, "strds", self.temporal_type + map_key, strds_id, lst, "strds", self.temporal_type ) class GrassVectorOutputProvider(VectorOutputProvider): """Write drainage simulation outputs to GRASS.""" - def __init__(self, config: GrassVectorOutputConfig) -> None: - """Initialize output provider with simulation configuration.""" - self.grass_interface = config["grass_interface"] - self.drainage_map_name = config["drainage_map_name"] - self.temporal_type = config["temporal_type"] + def __init__( + self, + grass_interface: GrassInterface, + drainage_map_id: str, + temporal_type: TemporalType, + ) -> None: + self.grass_interface = grass_interface + self.drainage_map_id = drainage_map_id + self.temporal_type = temporal_type self.grass_interface.validate_output_stds_temporal_type( - stds_name=self.drainage_map_name, + stds_id=self.drainage_map_id, stds_type="stvds", expected_temporal_type=self.temporal_type, ) @@ -128,10 +127,9 @@ def write_attributes( """Write drainage simulation data for current time step.""" if self.drainage_topology is None: raise RuntimeError("Drainage attributes cannot be written before topology.") - suffix = str(self.record_counter).zfill(4) - map_name = f"{self.drainage_map_name}_{suffix}" - self.grass_interface.write_vector_map(self.drainage_topology, attributes, map_name) - self.vector_drainage_maplist.append((map_name, sim_time)) + map_id = derived_record_name(self.drainage_map_id, self.record_counter) + self.grass_interface.write_vector_map(self.drainage_topology, attributes, map_id) + self.vector_drainage_maplist.append((map_id, sim_time)) self.record_counter += 1 def finalize(self) -> None: @@ -139,7 +137,7 @@ def finalize(self) -> None: if self.vector_drainage_maplist: self.grass_interface.register_maps_in_stds( stds_title="Itzï drainage results", - stds_name=self.drainage_map_name, + stds_id=self.drainage_map_id, map_list=self.vector_drainage_maplist, stds_type="stvds", t_type=self.temporal_type, diff --git a/src/itzi/run_plan.py b/src/itzi/run_plan.py new file mode 100644 index 00000000..175ea4f4 --- /dev/null +++ b/src/itzi/run_plan.py @@ -0,0 +1,143 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from itzi_core import TemporalType + +import itzi.messenger as msgr +from itzi.configreader import ConfigReader +from itzi.ensemble import load_yaml_stream +from itzi.ensemble.models import ( + DIRECT_INPUT_KEYS, + GREEN_AMPT_KEYS, + DocumentFailure, + EnsembleError, + ExpandedEnsemble, + ExpandedSimulation, + GrassConfig, + NormalizedInfiltration, + NormalizedTime, + OutputTemplates, + SourceDocument, + check_batch_limits, + check_unique_ensemble_ids, +) + + +def load_batch( + config_paths: list[str], +) -> tuple[tuple[ExpandedEnsemble, ...], tuple[DocumentFailure, ...]]: + """Load an ordered mixed configuration batch without initializing GRASS.""" + ensembles: list[ExpandedEnsemble] = [] + failures: list[DocumentFailure] = [] + for raw_path in config_paths: + path = Path(raw_path) + if path.suffix == ".yaml": + stream = load_yaml_stream(path) + ensembles.extend(stream.ensembles) + failures.extend(stream.failures) + else: + msgr.warning(f"INI configuration <{path}> is deprecated; use YAML instead.") + try: + ensembles.append(_legacy_ensemble(path)) + except Exception as error: # noqa: BLE001 - preserve document-local continuation. + failures.append( + DocumentFailure( + path=path.expanduser().resolve(), + document_index=0, + line=None, + column=None, + phase="schema", + detail=str(error), + ) + ) + check_unique_ensemble_ids(tuple(ensembles)) + check_batch_limits(tuple(ensembles)) + return tuple(ensembles), tuple(failures) + + +def _legacy_ensemble(path: Path) -> ExpandedEnsemble: + """Adapt the public one-member INI reader to the scalar batch model.""" + reader = ConfigReader(str(path)) + config = reader.sim_config + if config.hotstart_config is not None: + raise EnsembleError("hotstart output requires Stage 2 checkpoint support in mixed batches") + grass = reader.grass_params + source_path = path.expanduser().resolve() + source = SourceDocument( + path=source_path, + document_index=0, + ) + ensemble_id = f"legacy-{hashlib.blake2b(str(source_path).encode(), digest_size=8).hexdigest()}" + grass_config = GrassConfig( + database=grass.grassdata, + project=grass.location, + mapset=grass.mapset, + executable=grass.grass_bin if grass.grassdata is not None else None, + region=grass.region, + mask=grass.mask, + ) + input_names = config.input_map_names + infiltration_maps = { + key: input_names[key] for key in ("infiltration", *GREEN_AMPT_KEYS) if key in input_names + } + direct_inputs = {key: input_names[key] for key in DIRECT_INPUT_KEYS if key in input_names} + options = { + **config.surface_flow_parameters.model_dump(), + "dtinf": config.dtinf, + } + drainage = None + if config.swmm_inp is not None: + drainage = { + "swmm_input": str(config.swmm_inp), + "orifice_coeff": config.orifice_coeff, + "free_weir_coeff": config.free_weir_coeff, + "submerged_weir_coeff": config.submerged_weir_coeff, + } + normalized_time = NormalizedTime( + config.temporal_type, + None if config.temporal_type == TemporalType.RELATIVE else config.start_time, + None if config.temporal_type == TemporalType.RELATIVE else config.end_time, + config.end_time - config.start_time, + config.record_step, + False, + ) + stats_file = reader.stats_file + if stats_file is not None: + # The INI deprecation release preserves cwd-relative destinations. + stats_file = str((Path.cwd() / stats_file).resolve()) + expanded = ExpandedSimulation( + source=source, + ensemble_id=ensemble_id, + coordinates=(), + grass=grass_config, + time=normalized_time, + input_maps=tuple(sorted(direct_inputs.items())), + infiltration=NormalizedInfiltration( + config.infiltration_model, tuple(sorted(infiltration_maps.items())) + ), + options=tuple(sorted(options.items())), + drainage=tuple(sorted(drainage.items())) if drainage is not None else None, + outputs=OutputTemplates( + raster_prefix=reader.out_prefix, + raster_variables=tuple(reader.out_values), + statistics_file=stats_file, + drainage_dataset=config.drainage_output, + ), + ) + return ExpandedEnsemble(source, ensemble_id, source_path.stem, None, (expanded,)) diff --git a/src/itzi/simulation_runner.py b/src/itzi/simulation_runner.py new file mode 100644 index 00000000..f1c62e47 --- /dev/null +++ b/src/itzi/simulation_runner.py @@ -0,0 +1,186 @@ +""" +Copyright (C) 2026 Laurent Courty + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +""" + +from collections.abc import Mapping +from datetime import datetime +from time import monotonic +from typing import TYPE_CHECKING, Literal, Self + +import numpy as np +from itzi_core import SimulationBuilder, SimulationConfig + +import itzi.messenger as msgr +from itzi.ensemble.models import EffectiveMask +from itzi.grass.session import GrassParams +from itzi.providers.csv_output import ExclusiveCSVMassBalanceOutputProvider +from itzi.providers.grass_input import GrassRasterInputProvider +from itzi.providers.grass_output import GrassRasterOutputProvider, GrassVectorOutputProvider + +if TYPE_CHECKING: + from itzi_core import Simulation + + from itzi.grass.interface import GrassInterface + + +class SimulationRunner: + """Provide the necessary tools to run one simulation. + Must be instantiated within an active GRASS session.""" + + def __init__( + self, + sim_config: SimulationConfig, + grass_params: GrassParams, + hotstart_path: str | None = None, + stats_file: str | None = None, + effective_mask: EffectiveMask | None = None, + input_kinds: Mapping[str, Literal["raster", "strds"]] | None = None, + ) -> None: + from itzi.grass.utils import ( + check_output_files, + ensure_min_version, + get_current_mapset, + qualify_output_id, + resolve_input_identifier, + ) + + self.grass_params = grass_params + self.hotstart_path = hotstart_path + self.stats_file = stats_file + self.effective_mask = effective_mask + ensure_min_version() + if input_kinds is None: + resolved_inputs = { + key: resolve_input_identifier(name) + for key, name in sim_config.input_map_names.items() + } + self.input_kinds = {key: kind for key, (_, kind) in resolved_inputs.items()} + input_names = {key: map_id for key, (map_id, _) in resolved_inputs.items()} + else: + self.input_kinds = input_kinds + input_names = sim_config.input_map_names + mapset = get_current_mapset() + self.sim_config = sim_config.model_copy( + update={ + "input_map_names": input_names, + "output_map_names": { + key: qualify_output_id(name, mapset) + for key, name in sim_config.output_map_names.items() + }, + "drainage_output": qualify_output_id(sim_config.drainage_output, mapset) + if sim_config.drainage_output is not None + else None, + } + ) + self.g_interface: GrassInterface + self.sim: Simulation + + msgr.display_sim_param(self.sim_config) + + output_names = list(self.sim_config.output_map_names.values()) + if self.sim_config.drainage_output is not None: + output_names.append(self.sim_config.drainage_output) + check_output_files(output_names) + msgr.debug("Output files OK") + + def initialize(self) -> Self: + from itzi.grass.interface import GrassInterface + + data_type = np.float32 + self.g_interface = GrassInterface( + start_time=self.sim_config.start_time, + end_time=self.sim_config.end_time, + dtype=data_type, + region_id=self.grass_params.region, + raster_mask_id=self.grass_params.mask, + effective_mask=self.effective_mask, + ) + msgr.verbose("Setting up GRASS simulation...") + + raster_input_provider = GrassRasterInputProvider( + grass_interface=self.g_interface, + input_map_names=self.sim_config.input_map_names, + default_start_time=self.sim_config.start_time, + default_end_time=self.sim_config.end_time, + input_kinds=self.input_kinds, + ) + raster_output_provider = GrassRasterOutputProvider( + grass_interface=self.g_interface, + out_map_names=self.sim_config.output_map_names, + hmin=self.sim_config.surface_flow_parameters.hmin, + temporal_type=self.sim_config.temporal_type, + ) + sim_builder = ( + SimulationBuilder(self.sim_config, self.g_interface.get_npmask(), data_type) + .with_input_provider(raster_input_provider) + .with_raster_output_provider(raster_output_provider) + ) + if self.sim_config.drainage_output is not None: + vector_output_provider = GrassVectorOutputProvider( + grass_interface=self.g_interface, + temporal_type=self.sim_config.temporal_type, + drainage_map_id=self.sim_config.drainage_output, + ) + sim_builder.with_vector_output_provider(vector_output_provider) + if self.stats_file: + stats_provider = ExclusiveCSVMassBalanceOutputProvider( + self.stats_file, overwrite=self.g_interface.overwrite + ) + sim_builder.with_mass_balance_output_provider(stats_provider) + if self.hotstart_path: + sim_builder.with_hotstart(self.hotstart_path) + self.sim: Simulation = sim_builder.build() + # Initialize the simulation + self.sim.initialize() + return self + + def run(self): + """Run a full simulation""" + sim_start_time = datetime.now() + last_progress_update = monotonic() - 0.5 + msgr.verbose("Starting time-stepping...") + while self.sim.sim_time < self.sim.end_time: + # display advance of simulation + now = monotonic() + if now - last_progress_update >= 0.2: + msgr.percent( + self.sim.start_time, + self.sim.end_time, + self.sim.sim_time, + sim_start_time, + ) + last_progress_update = now + self.step() + return self + + def finalize(self): + """Tear down the simulation and return to previous state.""" + self.sim.finalize() + # Cleanup the grass interface object + if hasattr(self, "g_interface"): + self.g_interface.cleanup() + return self + + def step(self): + """Do one simulation step.""" + self.sim.update() + return self + + @property + def origin(self): + return (self.sim.domain_data.north, self.sim.domain_data.west) + + def __del__(self): + # Cleanup the grass interface object + if hasattr(self, "g_interface"): + self.g_interface.cleanup() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 02f24cc9..625fd25b 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2,17 +2,36 @@ import argparse import os +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import cast import pytest +from itzi_core import DomainData, SimulationConfig import itzi.messenger as msgr from itzi.cli_parser import build_parser +from itzi.ensemble.models import ( + ArtifactSummary, + EffectiveMask, + ExpandedEnsemble, + ResolvedEnsemble, + ResolvedSimulation, +) +from itzi.grass.session import GrassParams from itzi.itzi import ( VerbosityLevel, + _run_ensemble_batch, + _run_one_ensemble, + _select_members, + _validate_ensemble_run_requirements, itzi_run, itzi_run_one, main, reconcile_hotstart_commands, + resolved_sim_runner_worker, + run_validation_worker, sim_runner_worker, ) @@ -40,6 +59,332 @@ def test_run_parser_accepts_resume_from_args(): assert args.resume_from == [("a.ini", "restart_a.zip"), ("b.ini", "restart_b.zip")] +@pytest.mark.parametrize("dry_option", ["--dry-run", "-d"]) +def test_run_parser_accepts_yaml_dry_run_and_member_selection(dry_option): + args = build_parser().parse_args( + ["run", "study.yaml", dry_option, "--member", "study#sim-a", "--member", "study#sim-b"] + ) + + assert args.dry is True + assert args.member == ["study#sim-a", "study#sim-b"] + + +def test_ensemble_members_are_resolved_in_one_spawn(monkeypatch): + simulations = (object(), object()) + ensemble = SimpleNamespace( + ensemble_id="study", + simulations=simulations, + source=SimpleNamespace(path=Path("study.yaml")), + manifest_template=None, + ) + calls = [] + + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: ((ensemble,), ())) + monkeypatch.setattr( + "itzi.itzi._resolve_ensemble_in_subprocess", + lambda received: calls.append(received) or (), + ) + monkeypatch.setattr("itzi.itzi._display_dry_plan", lambda *_, **__: None) + + _run_ensemble_batch( + argparse.Namespace(config_file=["study.yaml"], resume_from=[], member=[], dry=True) + ) + + assert calls == [simulations] + + +@pytest.mark.parametrize("dry", [False, True]) +def test_ensembles_are_checked_and_run_in_order_without_cross_ensemble_validation( + tmp_path, monkeypatch, dry +): + ensembles = tuple( + SimpleNamespace( + ensemble_id=name, + simulations=(SimpleNamespace(simulation_id=name),), + source=SimpleNamespace(path=tmp_path / f"{name}.yaml"), + manifest_template=None, + ) + for name in ("first", "second") + ) + members = { + name: ResolvedSimulation( + simulation_id=name, + coordinates=(), + grass_params=cast(GrassParams, None), + domain_data=cast(DomainData, None), + effective_mask=cast(EffectiveMask, None), + input_kinds=(), + simulation_config=cast( + SimulationConfig, + SimpleNamespace( + input_map_names={"friction": "first_water_depth@PERMANENT"} + if name == "second" + else {} + ), + ), + artifacts=ArtifactSummary( + output_map_names=(("water_depth", "first_water_depth"),) + if name == "first" + else (), + drainage_output=None, + statistics_file=None, + ), + normalized_payload=name, + ) + for name in ("first", "second") + } + calls = [] + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: (ensembles, ())) + + def resolve(expanded): + name = expanded[0].simulation_id + calls.append(("resolve", name)) + return (members[name],) + + def validate(simulations, _sources): + calls.append(("validate", simulations[0].simulation_id)) + assert len(simulations) == 1 + + def validate_run(simulations): + calls.append(("run_validation", simulations[0].simulation_id)) + return {} + + def run(resolved, selected_ids, *_args, **_kwargs): + calls.append(("run", resolved.ensemble.ensemble_id)) + assert selected_ids == {resolved.ensemble.ensemble_id} + return 0 + + def display(resolved, *_args, **_kwargs): + calls.append(("display", resolved.ensemble.ensemble_id)) + + monkeypatch.setattr("itzi.itzi._resolve_ensemble_in_subprocess", resolve) + monkeypatch.setattr("itzi.itzi.validate_resolved_ensemble", validate) + monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_, **__: None) + monkeypatch.setattr("itzi.itzi._validate_run_requirements_in_subprocess", validate_run) + monkeypatch.setattr("itzi.itzi._run_one_ensemble", run) + monkeypatch.setattr("itzi.itzi._display_dry_plan", display) + + _run_ensemble_batch( + argparse.Namespace(config_file=["first.yaml", "second.yaml"], member=[], dry=dry, o=True) + ) + + assert calls == [ + (step, name) + for name in ("first", "second") + for step in ("resolve", "validate", "run_validation", "display" if dry else "run") + ] + + +def test_run_validation_only_checks_selected_members(tmp_path, monkeypatch): + selected_simulation = SimpleNamespace(simulation_id="sim-selected") + selected_with_failure = SimpleNamespace(simulation_id="sim-failed") + unselected_simulation = SimpleNamespace(simulation_id="sim-unselected") + ensemble = SimpleNamespace( + ensemble_id="study", + manifest_template=None, + source=SimpleNamespace(path=tmp_path / "study.yaml"), + ) + calls = [] + monkeypatch.setattr("itzi.itzi._validate_manifest_destination", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + "itzi.itzi._validate_run_requirements_in_subprocess", + lambda simulations: ( + calls.append(tuple(s.simulation_id for s in simulations)) + or {"sim-failed": "invalid input"} + ), + ) + + member_failures, manifest_failure = _validate_ensemble_run_requirements( + ResolvedEnsemble( + cast(ExpandedEnsemble, ensemble), + cast( + tuple[ResolvedSimulation, ...], + (selected_simulation, selected_with_failure, unselected_simulation), + ), + (), + ), + {"sim-selected", "sim-failed"}, + has_selectors=True, + overwrite=False, + ) + + assert calls == [("sim-selected", "sim-failed")] + assert member_failures == { + "sim-failed": {"phase": "run_validation", "detail": "invalid input"} + } + assert manifest_failure is None + + +def test_select_members_only_matches_current_ensemble(): + resolved = ResolvedEnsemble( + cast(ExpandedEnsemble, SimpleNamespace(ensemble_id="first")), + cast(tuple[ResolvedSimulation, ...], (SimpleNamespace(simulation_id="member"),)), + (), + ) + assert _select_members(resolved, [("second#member", "second#member")]) == set() + assert _select_members(resolved, [("first#member", "first#member")]) == {"member"} + + +def test_unqualified_selector_requires_one_ensemble(monkeypatch): + ensembles = tuple(SimpleNamespace(ensemble_id=name) for name in ("first", "second")) + monkeypatch.setattr("itzi.itzi.load_batch", lambda _: (ensembles, ())) + + with pytest.raises(RuntimeError, match="Unqualified --member IDs"): + _run_ensemble_batch( + argparse.Namespace(config_file=["first.yaml", "second.yaml"], member=["member"]) + ) + + +def test_one_ensemble_runs_only_planned_members_and_counts_failures(tmp_path, monkeypatch): + ensemble = SimpleNamespace( + ensemble_id="study", + source=SimpleNamespace(path=tmp_path / "study.yaml"), + manifest_template=None, + ) + artifacts = SimpleNamespace(output_map_names=(), drainage_output=None, statistics_file=None) + simulations = tuple( + SimpleNamespace(simulation_id=name, coordinates=(), artifacts=artifacts) + for name in ("invalid", "unselected", "good", "bad") + ) + runs = [] + updates = [] + run_validations = [] + monkeypatch.setattr("itzi.itzi._create_manifest", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + "itzi.itzi._update_manifest", + lambda _path, _ensemble, states: updates.append( + {key: value["status"] for key, value in states.items()} + ), + ) + monkeypatch.setattr( + "itzi.itzi._validate_run_requirements_in_subprocess", + lambda simulations: run_validations.append(simulations[0].simulation_id) or {}, + ) + monkeypatch.setattr( + "itzi.itzi._run_simulation_in_subprocess", + lambda simulation: ( + runs.append(simulation.simulation_id) + or ( + ("execution_failed", "failed") + if simulation.simulation_id == "bad" + else ("completed", None) + ) + ), + ) + + count = _run_one_ensemble( + ResolvedEnsemble( + cast(ExpandedEnsemble, ensemble), cast(tuple[ResolvedSimulation, ...], simulations), () + ), + {"good", "bad", "invalid"}, + {"invalid": {"phase": "run_validation", "detail": "invalid input"}}, + None, + has_selectors=True, + overwrite=False, + ) + + assert count == 2 + assert runs == ["good", "bad"] + assert run_validations == ["bad"] + assert updates[-1] == { + "good": "completed", + "bad": "execution_failed", + "invalid": "validation_failed", + "unselected": "not_selected", + } + + +def test_run_validation_worker_keeps_member_failures(monkeypatch): + calls = [] + + class FakeGrassSessionManager: + def __init__(self, params): + assert params == "shared" + + def __enter__(self): + calls.append("open") + + def __exit__(self, *_): + calls.append("close") + + def validate_run_requirements(simulation): + calls.append(simulation.simulation_id) + if simulation.simulation_id == "bad": + raise ValueError("invalid input") + + monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) + fake_run_validation = ModuleType("itzi.ensemble.run_validation") + setattr(fake_run_validation, "validate_run_requirements", validate_run_requirements) + monkeypatch.setitem(sys.modules, "itzi.ensemble.run_validation", fake_run_validation) + simulations = tuple( + SimpleNamespace(simulation_id=simulation_id, grass_params="shared") + for simulation_id in ("good", "bad", "another") + ) + + assert run_validation_worker(cast(tuple[ResolvedSimulation, ...], simulations)) == { + "bad": "ValueError: invalid input" + } + assert calls == ["open", "good", "bad", "another", "close"] + + +@pytest.mark.parametrize("finalize_fails", [False, True]) +def test_failed_ensemble_member_finalizes_in_session_and_reports_error( + monkeypatch, itzi_stderr, finalize_fails: bool +) -> None: + calls = [] + + class FakeGrassSessionManager: + def __init__(self, _params): + pass + + def __enter__(self): + calls.append("open") + + def __exit__(self, *_): + calls.append("close") + + class FakeSimulationRunner: + def __init__(self, *_args, **_kwargs): + pass + + def initialize(self): + return self + + def run(self): + calls.append("run") + raise ValueError("tiny computed dt") + + def finalize(self): + calls.append("finalize") + if finalize_fails: + raise RuntimeError("registration failed") + + monkeypatch.setattr("itzi.itzi.GrassSessionManager", FakeGrassSessionManager) + monkeypatch.setattr("itzi.itzi.SimulationRunner", FakeSimulationRunner) + simulation = cast( + ResolvedSimulation, + SimpleNamespace( + grass_params=None, + simulation_config=None, + artifacts=SimpleNamespace(statistics_file=None), + effective_mask=None, + input_kinds=(), + ), + ) + + assert resolved_sim_runner_worker(simulation) == ( + "execution_failed", + "ValueError: tiny computed dt", + ) + assert calls == ["open", "run", "finalize", "close"] + stderr = itzi_stderr.getvalue() + assert stderr.count("WARNING: Simulation failed: ValueError: tiny computed dt") == 1 + assert ("Could not finalize partial outputs: RuntimeError: registration failed" in stderr) == ( + finalize_fails + ) + assert "Traceback" not in stderr + + def test_run_parser_rejects_v_and_q_together(): with pytest.raises(SystemExit): build_parser().parse_args(["run", "a.ini", "-v", "-q"]) @@ -63,6 +408,38 @@ def fail(_): assert "Traceback" not in stderr +@pytest.mark.parametrize("same_file", [True, False]) +def test_main_reports_duplicate_ensemble_ids(tmp_path, itzi_stderr, same_file): + document = """\ +schema_version: 1 +ensemble: + id: repeated +grass: {} +time: + duration: "01:00:00" + record_step: "00:05:00" +input: + ground_elevation: elevation + friction: manning +parameters: {} +outputs: {} +""" + first = tmp_path / "first.yaml" + second = tmp_path / "second.yaml" + if same_file: + first.write_text(f"{document}---\n{document}", encoding="utf-8") + paths = [str(first)] + else: + first.write_text(document, encoding="utf-8") + second.write_text(document, encoding="utf-8") + paths = [str(first), str(second)] + + assert main(["run", *paths]) == 1 + stderr = itzi_stderr.getvalue() + assert stderr.count("ERROR: duplicate ensemble ID 'repeated'") == 1 + assert "Traceback" not in stderr + + def test_main_propagates_unexpected_error(monkeypatch): def fail(_): raise ValueError("unexpected") @@ -142,16 +519,9 @@ def test_worker_passes_statistics_file_to_simulation_runner(monkeypatch): class FakeConfigReader: def __init__(self, _): - pass - - def get_sim_params(self): - return sim_params - - def get_grass_params(self): - return grass_params - - def get_stats_file(self): - return "statistics.csv" + self.sim_config = sim_params + self.grass_params = grass_params + self.stats_file = "statistics.csv" class FakeGrassSessionManager: def __init__(self, received_grass_params): @@ -168,6 +538,9 @@ def __init__(self, *args, **kwargs): runner_arguments["args"] = args runner_arguments["kwargs"] = kwargs + def initialize(self): + return self + def run(self): return self @@ -218,7 +591,10 @@ def test_reconcile_hotstart_commands_accepts_single_resume_for_single_config(): def test_reconcile_hotstart_commands_matches_multiple_named_values(): config_file_list = ["/tmp/a.ini", "/tmp/b.ini", "/tmp/c.ini"] - resume_from_list = [("c.ini", "restart_c.zip"), ("a.ini", "restart_a.zip")] + resume_from_list: list[tuple[str | None, str]] = [ + ("c.ini", "restart_c.zip"), + ("a.ini", "restart_a.zip"), + ] assert reconcile_hotstart_commands(config_file_list, resume_from_list) == [ ("/tmp/a.ini", "restart_a.zip"), @@ -229,7 +605,7 @@ def test_reconcile_hotstart_commands_matches_multiple_named_values(): def test_reconcile_hotstart_commands_accepts_duplicate_basenames_with_paths(): config_file_list = ["./sim1/config.ini", "sim2/config.ini"] - resume_from_list = [ + resume_from_list: list[tuple[str | None, str]] = [ ("./sim1/config.ini", "sim1/hotstart.zip"), ("sim2/config.ini", "sim2/hotstart.zip"), ] diff --git a/tests/cli/test_configreader.py b/tests/cli/test_configreader.py index 9c9e2d08..4c667e37 100644 --- a/tests/cli/test_configreader.py +++ b/tests/cli/test_configreader.py @@ -2,6 +2,7 @@ import logging from configparser import ConfigParser +from dataclasses import asdict from datetime import datetime, timedelta from pathlib import Path @@ -58,12 +59,12 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): config_file = write_config_file(tmp_path, make_config_dict()) reader = ConfigReader(config_file) - sim_config = reader.get_sim_params() - grass_params = reader.get_grass_params() + sim_config = reader.sim_config + grass_params = reader.grass_params assert sim_config.hotstart_config is None assert sim_config.surface_flow_parameters == SurfaceFlowParameters() - assert reader.get_stats_file() is None + assert reader.stats_file is None assert sim_config.dtinf == DefaultValues.DTINF assert sim_config.swmm_inp is None assert sim_config.drainage_output is None @@ -72,7 +73,7 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): assert sim_config.submerged_weir_coeff == DefaultValues.SUBMERGED_WEIR_COEFF assert sim_config.input_map_names == {"ground_elevation": "z", "friction": "n"} assert sim_config.output_map_names == {"water_depth": "out_water_depth"} - assert grass_params.model_dump() == { + assert asdict(grass_params) == { "grassdata": None, "location": None, "mapset": None, @@ -82,6 +83,17 @@ def test_reader_uses_defaults_when_optional_sections_are_missing(tmp_path): } +def test_reader_rejects_partial_grass_context(tmp_path): + config_file = write_config_file( + tmp_path, make_config_dict(grass={"grassdata": "/grassdata", "location": "project"}) + ) + + with pytest.raises( + RuntimeError, match="GRASS database, location, and mapset must be supplied together" + ): + ConfigReader(config_file) + + @pytest.mark.parametrize( ("statistics", "expected_stats_file"), [ @@ -102,8 +114,8 @@ def test_reader_exposes_stats_file_separately_from_simulation_config( reader = ConfigReader(config_file) - assert reader.get_stats_file() == expected_stats_file - assert "stats_file" not in type(reader.get_sim_params()).model_fields + assert reader.stats_file == expected_stats_file + assert "stats_file" not in type(reader.sim_config).model_fields def test_reader_normalizes_deprecated_aliases(tmp_path, caplog): @@ -127,7 +139,7 @@ def test_reader_normalizes_deprecated_aliases(tmp_path, caplog): with caplog.at_level(logging.WARNING, logger="itzi"): itzi_logger.addHandler(caplog.handler) try: - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config finally: itzi_logger.removeHandler(caplog.handler) @@ -189,7 +201,7 @@ def test_reader_prefers_canonical_names_over_input_aliases(tmp_path): ), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.input_map_names["ground_elevation"] == "canonical_elevation" @@ -217,7 +229,7 @@ def test_reader_normalizes_legacy_created_volume_aliases(tmp_path, caplog, legac with caplog.at_level(logging.WARNING, logger="itzi"): itzi_logger.addHandler(caplog.handler) try: - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config finally: itzi_logger.removeHandler(caplog.handler) @@ -268,7 +280,7 @@ def test_reader_accepts_supported_time_combinations( ): config_file = write_config_file(tmp_path, make_config_dict(time=time_section)) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.temporal_type == expected_temporal_type assert sim_config.start_time == expected_start @@ -331,7 +343,7 @@ def test_reader_infers_green_ampt_model_from_complete_parameter_set(tmp_path): ), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.infiltration_model == InfiltrationModelType.GREEN_AMPT @@ -368,7 +380,7 @@ def test_reader_falls_back_to_config_dir_for_relative_swmm_inp(tmp_path, monkeyp make_config_dict(drainage={"swmm_inp": "swmm_config.inp"}), ) - sim_config = ConfigReader(config_file).get_sim_params() + sim_config = ConfigReader(config_file).sim_config assert sim_config.swmm_inp == Path(config_swmm_inp) diff --git a/tests/cli/test_ensemble_config.py b/tests/cli/test_ensemble_config.py new file mode 100644 index 00000000..bb72c5b1 --- /dev/null +++ b/tests/cli/test_ensemble_config.py @@ -0,0 +1,491 @@ +"""Pure YAML ensemble loader and expansion tests.""" + +from __future__ import annotations + +import sys +from dataclasses import replace +from datetime import datetime, timedelta +from types import SimpleNamespace +from typing import cast + +import pytest +from itzi_core import InfiltrationModelType, SurfaceFlowParameters, TemporalType +from itzi_core.const import DefaultValues +from pydantic import ValidationError + +from itzi.ensemble import load_yaml_stream +from itzi.ensemble.models import ( + MAX_ENSEMBLE_MEMBERS, + ArtifactSummary, + GrassConfig, + ResolvedSimulation, + format_duration, + parse_duration, + render_template, +) +from itzi.ensemble.resolution import ( + _build_simulation_config, + _last_record_index, + _render_artifacts, + _requested_grass_params, + _resolve_inputs, + validate_resolved_ensemble, +) +from itzi.ensemble.schema import TimeConfig +from itzi.manifest import _manifest_document + + +def write_yaml(tmp_path, content: str): + path = tmp_path / "study.yaml" + path.write_text(content, encoding="utf-8") + return path + + +def document(*, ensemble_id: str = "study", extra: str = "") -> str: + body = ( + extra + or """\ +input: + ground_elevation: "elevation" + friction: "manning" +parameters: {} +outputs: + rasters: + prefix: "results_{ensemble}_{simulation}" + variables: [water_depth] +""" + ) + return f"""\ +schema_version: 1 +ensemble: + id: {ensemble_id} +grass: {{}} +time: + duration: "02:00:00" + record_step: "00:05:00" +{body}""" + + +def test_scalar_document_expands_to_one_member(tmp_path): + stream = load_yaml_stream(write_yaml(tmp_path, document())) + + assert stream.failures == () + assert len(stream.ensembles) == 1 + simulation = stream.ensembles[0].simulations[0] + assert simulation.coordinates == () + assert simulation.time.temporal_type == TemporalType.RELATIVE + assert simulation.time.start is None + assert simulation.time.end is None + assert simulation.time.duration == timedelta(hours=2) + assert dict(simulation.parameters) == SurfaceFlowParameters().model_dump() | { + "dtinf": DefaultValues.DTINF + } + + +def test_ensemble_description_reaches_manifest(tmp_path): + description = "Central city rainfall study" + content = document().replace(" id: study", f" id: study\n description: {description}") + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + assert _manifest_document(stream.ensembles[0], {})["ensemble"] == { + "id": "study", + "description": description, + } + + +def test_drainage_coefficients_are_forwarded(tmp_path): + expanded = load_yaml_stream(write_yaml(tmp_path, document())).ensembles[0].simulations[0] + expanded = replace( + expanded, + drainage=( + ("swmm_input", "network.inp"), + ("orifice_coeff", 0.2), + ("free_weir_coeff", 0.3), + ("submerged_weir_coeff", 0.4), + ), + ) + config = _build_simulation_config( + expanded, + dict(expanded.input_maps), + ArtifactSummary((), None, None), + tmp_path / "network.inp", + ) + assert ( + config.orifice_coeff, + config.free_weir_coeff, + config.submerged_weir_coeff, + ) == (0.2, 0.3, 0.4) + + +def test_drainage_coefficients_can_be_omitted(tmp_path): + content = document().replace( + "outputs:\n", 'drainage:\n swmm_input: "network.inp"\noutputs:\n', 1 + ) + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + expanded = stream.ensembles[0].simulations[0] + assert dict(expanded.drainage) == {"swmm_input": "network.inp"} + config = _build_simulation_config( + expanded, + dict(expanded.input_maps), + ArtifactSummary((), None, None), + tmp_path / "network.inp", + ) + assert ( + config.orifice_coeff, + config.free_weir_coeff, + config.submerged_weir_coeff, + ) == ( + DefaultValues.ORIFICE_COEFF, + DefaultValues.FREE_WEIR_COEFF, + DefaultValues.SUBMERGED_WEIR_COEFF, + ) + + +def test_drainage_coefficients_allow_partial_sweep(tmp_path): + content = document().replace( + "outputs:\n", + 'drainage:\n swmm_input: "network.inp"\n orifice_coeff: 0.2\n' + " free_weir_coeff: [0.3, 0.4]\noutputs:\n", + 1, + ) + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + simulations = stream.ensembles[0].simulations + assert [dict(simulation.drainage) for simulation in simulations] == [ + {"swmm_input": "network.inp", "orifice_coeff": 0.2, "free_weir_coeff": coefficient} + for coefficient in (0.3, 0.4) + ] + assert [dict(simulation.coordinates) for simulation in simulations] == [ + {"drainage.free_weir_coeff": coefficient} for coefficient in (0.3, 0.4) + ] + + +def test_time_config_accepts_only_supported_combinations(): + start = "2026-09-01T00:00:00" + end = "2026-09-01T02:00:00" + duration = "02:00:00" + for fields in ( + {"duration": duration}, + {"start": start, "duration": duration}, + {"start": start, "end": end}, + ): + TimeConfig.model_validate({"record_step": "00:05:00", **fields}) + for fields in ( + {}, + {"start": start}, + {"end": end}, + {"end": end, "duration": duration}, + {"start": start, "end": end, "duration": duration}, + ): + with pytest.raises(ValidationError): + TimeConfig.model_validate({"record_step": "00:05:00", **fields}) + + +def test_grass_context_requires_complete_context(): + with pytest.raises(ValidationError, match="mapset must be supplied together"): + GrassConfig(database="/grassdata", project="project") + + +def test_grass_region_and_mask_work_with_active_session(tmp_path): + content = document().replace("grass: {}", "grass:\n region: local_region\n mask: mask_map") + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + params = _requested_grass_params(stream.ensembles[0].simulations[0]) + assert (params.grassdata, params.region, params.mask) == (None, "local_region", "mask_map") + + +def test_cartesian_sweeps_use_canonical_coordinate_order(tmp_path): + content = document( + extra="""\ +input: + ground_elevation: ["elevation_a", "elevation_b"] + friction: "manning" + rainfall_rate: ["rain_a", "rain_b"] +parameters: + cfl: [0.5, 0.7] +outputs: + rasters: + prefix: "out_{simulation}" + variables: [water_depth] +""" + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + simulations = stream.ensembles[0].simulations + assert len(simulations) == 8 + assert tuple(path for path, _ in simulations[0].coordinates) == ( + "input.ground_elevation", + "input.rainfall_rate", + "parameters.cfl", + ) + assert dict(simulations[0].coordinates)["parameters.cfl"] == 0.5 + assert dict(simulations[-1].coordinates)["parameters.cfl"] == 0.7 + + +def test_loader_continues_after_failed_explicit_document(tmp_path): + content = """\ +schema_version: 1 +ensemble: + id: first +grass: {} +time: + duration: "01:00:00" + record_step: "00:05:00" +input: + ground_elevation: elevation + friction: manning +parameters: + cfl: "0.7" +outputs: {} +--- +""" + document(ensemble_id="second") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert len(stream.failures) == 1 + assert stream.failures[0].document_index == 0 + assert stream.failures[0].phase == "schema" + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["second"] + + +def test_loader_rejects_duplicate_keys_and_keeps_later_document(tmp_path): + content = """\ +schema_version: 1 +schema_version: 1 +--- +""" + document(ensemble_id="second") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures[0].phase == "parse" + assert stream.failures[0].line == 2 + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["second"] + + +def test_loader_rejects_empty_documents_and_merge_keys(tmp_path): + content = """\ +--- +--- +defaults: &defaults + cfl: 0.7 +schema_version: 1 +ensemble: + id: invalid-merge +grass: {} +time: + duration: "01:00:00" + record_step: "00:05:00" +input: + ground_elevation: elevation + friction: manning +parameters: + <<: *defaults +outputs: {} +--- +""" + document(ensemble_id="valid") + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert [failure.phase for failure in stream.failures] == ["parse", "parse"] + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] + + +def test_loader_rejects_non_json_yaml_and_keeps_later_document(tmp_path): + content = ( + document(ensemble_id="binary") + + "extra: !!binary AQI=\n---\n" + + document(ensemble_id="valid") + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert [failure.phase for failure in stream.failures] == ["parse"] + assert [ensemble.ensemble_id for ensemble in stream.ensembles] == ["valid"] + + +def test_infiltration_alternatives_are_explicit(tmp_path): + content = document( + extra="""\ +input: + ground_elevation: elevation + friction: manning + infiltration: + - type: none + - type: constant + rate: infiltration_rate +parameters: {} +outputs: {} +""" + ) + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.failures == () + simulations = stream.ensembles[0].simulations + assert [simulation.infiltration.model for simulation in simulations] == [ + InfiltrationModelType.NULL, + InfiltrationModelType.CONSTANT, + ] + + duplicate = content.replace(" - type: none\n", " - type: none\n - type: none\n") + assert load_yaml_stream(write_yaml(tmp_path, duplicate)).failures[0].phase == "schema" + + +def test_absolute_offsets_keep_wall_clock_values(tmp_path): + content = document().replace( + ' duration: "02:00:00"\n record_step: "00:05:00"', + ' start: "2026-09-01T00:00:00+02:00"\n duration: "02:00:00"\n record_step: "00:05:00"', + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + simulation = stream.ensembles[0].simulations[0] + assert simulation.time.temporal_type == TemporalType.ABSOLUTE + assert simulation.time.start == datetime(2026, 9, 1, 0, 0) # noqa: DTZ001 + assert simulation.time.end == datetime(2026, 9, 1, 2, 0) # noqa: DTZ001 + assert simulation.time.had_timezone_offset is True + + +def test_expansion_limit_is_checked_before_materialization(tmp_path): + values = ", ".join(f"{0.01 + index / 1000:.3f}" for index in range(MAX_ENSEMBLE_MEMBERS + 1)) + content = document( + extra=f"""\ +input: + ground_elevation: elevation + friction: manning +parameters: + cfl: [{values}] +outputs: {{}} +""" + ) + + stream = load_yaml_stream(write_yaml(tmp_path, content)) + + assert stream.ensembles == () + assert stream.failures[0].phase == "expansion" + assert "exceeding the 100 limit" in stream.failures[0].detail + + +def test_duration_and_template_helpers_are_strict(): + for str_value, expected in ( + ("25:02:03", timedelta(hours=25, minutes=2, seconds=3)), + ("123:55:20", timedelta(hours=123, minutes=55, seconds=20)), + ): + assert parse_duration(str_value) == expected + assert format_duration(timedelta(days=1, hours=2, minutes=3, seconds=4)) == "26:03:04" + for value in ( + "PT1H", + "01:00", + "01:60:00", + "00:00:00", + "-01:00:00", + "2:31:00:00", + "une:heure:trente", + "00:90:00", + ): + with pytest.raises(ValueError): + parse_duration(value) + assert ( + render_template("{{{ensemble}}}-{simulation}", ensemble="study", simulation="sim-a") + == "{study}-sim-a" + ) + + +def test_statistics_file_template_is_rendered(tmp_path, monkeypatch): + expanded = load_yaml_stream(write_yaml(tmp_path, document())).ensembles[0].simulations[0] + expanded = replace( + expanded, + outputs=replace( + expanded.outputs, + raster_prefix=None, + raster_variables=(), + statistics_file="results/{simulation}.csv", + ), + ) + monkeypatch.setattr("itzi.ensemble.resolution._validate_output_names", lambda *_: None) + + artifacts = _render_artifacts(expanded, "sim-a", "PERMANENT") + + assert artifacts.statistics_file == tmp_path / "results/sim-a.csv" + + +def test_generated_output_inventory_includes_final_partial_record(): + assert _last_record_index(timedelta(seconds=60), timedelta(seconds=30)) == 2 + assert _last_record_index(timedelta(seconds=61), timedelta(seconds=30)) == 3 + + +def test_later_generated_output_cannot_alias_an_input(): + start = datetime(2026, 1, 1) # noqa: DTZ001 + config = SimpleNamespace( + start_time=start, + end_time=start + timedelta(minutes=1), + record_step=timedelta(seconds=30), + input_map_names={}, + swmm_inp=None, + ) + output = SimpleNamespace( + simulation_id="sim-output", + normalized_payload="output", + grass_params=SimpleNamespace(mapset="PERMANENT"), + effective_mask=SimpleNamespace(source=None), + simulation_config=config, + artifacts=SimpleNamespace( + output_map_names=(("water_depth", "result@PERMANENT"),), + drainage_output=None, + statistics_file=None, + ), + ) + input_simulation = SimpleNamespace( + simulation_id="sim-input", + normalized_payload="input", + grass_params=SimpleNamespace(mapset="PERMANENT"), + effective_mask=SimpleNamespace(source=None), + simulation_config=SimpleNamespace( + start_time=start, + end_time=start + timedelta(minutes=1), + record_step=timedelta(seconds=30), + input_map_names={"ground_elevation": "result_0002@PERMANENT"}, + swmm_inp=None, + ), + artifacts=SimpleNamespace( + output_map_names=(), + drainage_output=None, + statistics_file=None, + ), + ) + + with pytest.raises(ValueError, match="result_0002@PERMANENT.*aliases"): + validate_resolved_ensemble( + (cast(ResolvedSimulation, output), cast(ResolvedSimulation, input_simulation)) + ) + + +def test_input_resolution_cache_reuses_shared_identifiers(monkeypatch): + calls = [] + + def resolve(identifier): + calls.append(identifier) + return f"{identifier}@PERMANENT", "raster" + + cache = {} + monkeypatch.setitem( + sys.modules, "itzi.grass.utils", SimpleNamespace(resolve_input_identifier=resolve) + ) + + _resolve_inputs( + {"ground_elevation": "shared", "rainfall_rate": "first"}, + cache=cache, + ) + resolved, _ = _resolve_inputs( + {"ground_elevation": "shared", "rainfall_rate": "second"}, + cache=cache, + ) + + assert calls == ["shared", "first", "second"] + assert resolved["ground_elevation"] == "shared@PERMANENT" diff --git a/tests/grass/conftest.py b/tests/grass/conftest.py index 1fc5512f..da5d35c5 100644 --- a/tests/grass/conftest.py +++ b/tests/grass/conftest.py @@ -3,10 +3,11 @@ """ import os +import subprocess import sys import tempfile +from collections.abc import Iterator from pathlib import Path -import subprocess import pytest @@ -17,10 +18,26 @@ sys.path.append(grass_python_path) import grass.script as gscript # noqa: E402 -from itzi import SimulationRunner # noqa: E402 +from itzi import ( # noqa: E402 + SimulationRunner, # noqa: E402 +) from itzi.configreader import ConfigReader # noqa: E402 +from itzi.grass import session # noqa: E402 TESTS_ROOT = Path.cwd() +TEST_PROCESS = os.getpid() + + +@pytest.fixture(autouse=True) +def stop_temporal_subprocesses_after_test() -> Iterator[None]: + """Forked tests must stop their own GRASS RPC children before os._exit.""" + yield + if os.getpid() != TEST_PROCESS and session._initialized_temporal_session is not None: + session_pid = session._initialized_temporal_session[0] + if session_pid == os.getpid(): + import grass.temporal as tgis + + tgis.stop_subprocesses() @pytest.fixture(scope="session") @@ -152,19 +169,18 @@ def grass_5by5(grass_xy_session, test_data_path): gscript.mapcalc("infiltration_rate=2") gscript.mapcalc("loss_rate=1.5") gscript.mapcalc("inflow_rate=0.1") - return None + with session.GrassSessionManager(session.GrassParams()): + yield @pytest.fixture(scope="class") def grass_5by5_sim(grass_5by5, test_data_path): - """ """ config_file = os.path.join(test_data_path, "5by5", "5by5.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + ).initialize() assert isinstance(sim_runner, SimulationRunner) sim_runner.run().finalize() return sim_runner diff --git a/tests/grass/test_bmi.py b/tests/grass/test_bmi.py index f194d58f..ab896d24 100644 --- a/tests/grass/test_bmi.py +++ b/tests/grass/test_bmi.py @@ -1,6 +1,7 @@ """Test the Basic Model Interface implementation.""" import os +from collections.abc import Iterator import numpy as np import pytest @@ -16,6 +17,12 @@ def bmi_object(grass_5by5, test_data_path): return itzi_bmi +@pytest.fixture(scope="class") +def finalize_bmi_object(bmi_object: BmiItzi) -> Iterator[None]: + yield + bmi_object.finalize() + + @pytest.mark.forked class TestBmiMutating: """Those functions mutate the BMI model state. @@ -53,6 +60,7 @@ def test_set_value(self, bmi_object): assert np.all(value == bmi_object.get_value_ptr(var_name)) +@pytest.mark.usefixtures("finalize_bmi_object") class TestBmi: """Test non-mutating functions. No forking needed.""" diff --git a/tests/grass/test_ensemble.py b/tests/grass/test_ensemble.py new file mode 100644 index 00000000..1b91291c --- /dev/null +++ b/tests/grass/test_ensemble.py @@ -0,0 +1,570 @@ +"""GRASS integration coverage for basic spawned YAML ensemble execution.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from typing import cast +from uuid import uuid4 + +import grass.script as gscript +import numpy as np +import pytest +import yaml +from itzi_core import TemporalType + +from itzi.cli_parser import build_parser +from itzi.ensemble.models import ResolvedSimulation, ValidationFailure +from itzi.grass.names import derived_record_name +from itzi.grass.session import GrassSessionManager +from itzi.itzi import _resolve_ensemble_in_subprocess, itzi_run, run_validation_worker +from itzi.run_plan import load_batch + + +def write_study(tmp_path: Path, **changes: dict) -> Path: + """Write a minimal runnable ensemble with optional nested YAML overrides.""" + document = { + "schema_version": 1, + "ensemble": {"id": "stage-one"}, + "grass": {}, + "time": {"duration": "00:01:00", "record_step": "00:00:30"}, + "input": {"ground_elevation": "z", "friction": "n"}, + "parameters": {"dtmax": 0.3}, + "outputs": {}, + } + document["parameters"] = changes.pop("parameters", document["parameters"]) + for section, fields in changes.items(): + values = document[section] + assert isinstance(values, dict) + values.update(fields) + path = tmp_path / "study.yaml" + path.write_text(yaml.safe_dump(document), encoding="utf-8") + return path + + +def read_manifest(tmp_path: Path) -> dict: + return yaml.safe_load( + (tmp_path / "results" / "stage-one.manifest.yaml").read_text(encoding="utf-8") + ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_input_resolution_qualifies_rasters_and_strds_and_rejects_ambiguity( + tmp_path: Path, +) -> None: + from grass.pygrass import utils as gutils + + GrassSessionManager.ensure_temporal_initialized() + mapset = gutils.getenv("MAPSET") + gscript.mapcalc("stage_rain=1") + gscript.run_command( + "t.create", + output="stage_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="stage_series", + description="stage_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="stage_series", + type="raster", + maps="stage_rain", + start="0", + increment="60", + unit="seconds", + ) + path = write_study( + tmp_path, + input={"ground_elevation": f"z@{mapset}", "rainfall_rate": "stage_series"}, + ) + expanded = load_batch([str(path)])[0][0].simulations + resolved = _resolve_ensemble_in_subprocess(expanded)[0] + assert isinstance(resolved, ResolvedSimulation) + assert resolved.simulation_config.input_map_names == { + "ground_elevation": f"z@{mapset}", + "friction": f"n@{mapset}", + "rainfall_rate": f"stage_series@{mapset}", + } + assert dict(resolved.input_kinds)["rainfall_rate"] == "strds" + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None + + path = write_study(tmp_path, input={"rainfall_rate": f"stage_series@{mapset}"}) + qualified = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(qualified, ResolvedSimulation) + assert qualified.simulation_id == resolved.simulation_id + + gscript.mapcalc("stage_series=1") + ambiguous = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(ambiguous, ValidationFailure) + assert f"raster stage_series@{mapset}" in ambiguous.detail + assert f"STRDS stage_series@{mapset}" in ambiguous.detail + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_temporal_run_validation_failure_does_not_stop_next_member(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + + GrassSessionManager.ensure_temporal_initialized() + gscript.mapcalc("stage_short_rain=1") + gscript.run_command( + "t.create", + output="stage_short_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="stage_short_series", + description="stage_short_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="stage_short_series", + type="raster", + maps="stage_short_rain", + start="0", + increment="30", + unit="seconds", + ) + path = write_study( + tmp_path, + input={"water_depth": "start_h", "rainfall_rate": ["stage_short_series", "rainfall"]}, + outputs={"rasters": {"prefix": "stage_{simulation}", "variables": ["water_depth"]}}, + ) + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(path)])) + + members = read_manifest(tmp_path)["members"] + assert sorted(member["status"] for member in members) == ["completed", "validation_failed"] + failed = next(member for member in members if member["status"] == "validation_failed") + completed = next(member for member in members if member["status"] == "completed") + assert failed["failure"]["phase"] == "run_validation" + assert ( + "ends before simulation" in failed["failure"]["detail"] + or "inadequate temporal" in failed["failure"]["detail"] + ) + output = completed["artifacts"]["rasters"]["water_depth"] + assert output.endswith(f"@{gutils.getenv('MAPSET')}") + assert gutils.get_mapset_raster(f"{output.partition('@')[0]}_0000", gutils.getenv("MAPSET")) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_worker_rejects_changed_mask_region_and_inputs_after_resolution(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + + from itzi.grass.interface import GrassInterface + + gscript.mapcalc("stage_mask_values=if(col()==1, null(), if(col()==2, 0, 1))") + gscript.run_command("g.copy", raster="stage_mask_values,MASK") + try: + path = write_study(tmp_path, grass={"mask": "stage_mask_values"}) + expanded = load_batch([str(path)])[0][0].simulations + resolved = _resolve_ensemble_in_subprocess(expanded)[0] + assert isinstance(resolved, ResolvedSimulation) + assert resolved.effective_mask.mode == "explicit" + + config = resolved.simulation_config + interface = GrassInterface( + config.start_time, + config.end_time, + np.float32, + None, + "stage_mask_values", + resolved.effective_mask, + ) + try: + explicit = interface.get_npmask() + assert explicit[:, 0].all() + assert not explicit[:, 1:].any() # zero is inside an explicit mask + assert np.isfinite( + interface.read_raster_map(f"n@{gutils.getenv('MAPSET')}") + ).all() # bypass active MASK + interface.mask_mode, interface.mask_source = ( + "active", + f"MASK@{gutils.getenv('MAPSET')}", + ) + active = interface.get_npmask() + assert active[:, :2].all() # zero is outside an active CELL mask + assert not active[:, 2:].any() + finally: + interface.cleanup() + + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None + gscript.run_command("r.mask", flags="r") + assert run_validation_worker((resolved,)).get(resolved.simulation_id) is None + + active_path = write_study(tmp_path, grass={}) + active_resolved = _resolve_ensemble_in_subprocess( + load_batch([str(active_path)])[0][0].simulations + )[0] + assert isinstance(active_resolved, ResolvedSimulation) + assert active_resolved.effective_mask.mode == "none" + gscript.run_command("g.copy", raster="stage_mask_values,MASK") + assert "effective GRASS mask changed" in ( + run_validation_worker((active_resolved,)).get(active_resolved.simulation_id) or "" + ) + assert gutils.get_mapset_raster("MASK", gutils.getenv("MAPSET")) + finally: + gscript.run_command("r.mask", flags="r") + + gscript.run_command("g.copy", raster="n,stage_worker_friction") + path = write_study(tmp_path, input={"friction": "stage_worker_friction"}) + resolved = _resolve_ensemble_in_subprocess(load_batch([str(path)])[0][0].simulations)[0] + assert isinstance(resolved, ResolvedSimulation) + + gscript.run_command("g.region", n=60) + assert "GRASS domain changed" in ( + run_validation_worker((resolved,)).get(resolved.simulation_id) or "" + ) + gscript.run_command("g.region", n=50) + gscript.run_command("g.remove", flags="f", type="raster", name="stage_worker_friction") + assert "stage_worker_friction" in ( + run_validation_worker((resolved,)).get(resolved.simulation_id) or "" + ) + assert not (tmp_path / "results").exists() + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_rejects_raster_outside_domain_without_writing(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + + gscript.use_temp_region() + try: + gscript.run_command("g.region", n=150, s=100, e=150, w=100, res=10) + gscript.mapcalc("stage_outside=1") + finally: + gscript.del_temp_region() + path = write_study( + tmp_path, + input={"friction": "stage_outside"}, + outputs={ + "rasters": {"prefix": "stage_spatial", "variables": ["water_depth"]}, + "statistics": {"file": "stats/results.csv"}, + }, + ) + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(path), "--dry-run"])) + assert not (tmp_path / "results").exists() + assert not (tmp_path / "stats").exists() + assert not gutils.get_mapset_raster("stage_spatial_water_depth_0000", gutils.getenv("MAPSET")) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_ensemble_output_collision_with_input_rejects_all_members(tmp_path: Path) -> None: + from grass.pygrass import utils as gutils + + gscript.mapcalc("stage_alias_water_depth_0002=1") + path = write_study( + tmp_path, + input={"rainfall_rate": ["rainfall", "stage_alias_water_depth_0002"]}, + outputs={"rasters": {"prefix": "stage_alias", "variables": ["water_depth"]}}, + ) + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(path), "-o"])) + members = read_manifest(tmp_path)["members"] + assert len(members) == 2 + assert all(member["status"] == "validation_failed" for member in members) + assert all(member["failure"]["phase"] == "artifact_validation" for member in members) + assert not gutils.get_mapset_raster("stage_alias_water_depth_0000", gutils.getenv("MAPSET")) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_yaml_ensemble_runs_in_spawned_resolver_and_worker(test_data_temp_path): + config_path = Path(test_data_temp_path) / "source-file-name.yaml" + config_path.write_text( + """\ +schema_version: 1 +ensemble: + id: basic +grass: {} +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" + infiltration: + - type: "constant" + rate: "infiltration_rate" +parameters: + cfl: [0.2, 0.3] + dtmax: 0.3 +outputs: + rasters: + prefix: "basic_{simulation}" + variables: [water_depth] +""", + encoding="utf-8", + ) + args = build_parser().parse_args(["run", str(config_path), "-o"]) + + itzi_run(args) + + manifest_path = Path(test_data_temp_path) / "results" / "basic.manifest.yaml" + manifest_text = manifest_path.read_text(encoding="utf-8") + assert not manifest_text.lstrip().startswith("{") + assert manifest_text.splitlines()[1].startswith("last_updated_at: ") + manifest = yaml.safe_load(manifest_text) + last_updated_at = manifest["last_updated_at"] + assert isinstance(last_updated_at, str) + assert datetime.fromisoformat(last_updated_at).utcoffset() is not None + assert len(manifest["members"]) == 2 + assert all(member["status"] == "completed" for member in manifest["members"]) + assert [member["coordinates"] for member in manifest["members"]] == [ + { + "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, + "parameters.cfl": 0.2, + }, + { + "input.infiltration": {"rate": "infiltration_rate", "type": "constant"}, + "parameters.cfl": 0.3, + }, + ] + assert all( + member["artifacts"]["rasters"]["water_depth"].startswith("basic_sim-") + and member["artifacts"]["rasters"]["water_depth"].endswith("@5by5") + for member in manifest["members"] + ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_failed_ensemble_member_registers_partial_maps(tmp_path: Path) -> None: + import grass.temporal as tgis + + context = gscript.gisenv() + depth_map = f"stage_unstable_{uuid4().hex[:8]}" + gscript.mapcalc(f"{depth_map}=1000000000000000.0") + path = write_study( + tmp_path, + grass={ + "database": context["GISDBASE"], + "project": context["LOCATION_NAME"], + "mapset": context["MAPSET"], + "executable": "grass", + }, + input={"water_depth": depth_map}, + outputs={ + "rasters": { + "prefix": "stage_partial_{simulation}", + "variables": ["max_water_depth"], + } + }, + ) + + gisrc = os.environ.pop("GISRC") + try: + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(path), "-o"])) + finally: + os.environ["GISRC"] = gisrc + + member = read_manifest(tmp_path)["members"][0] + assert member["status"] == "execution_failed" + assert member["failure"]["detail"].startswith("DtError:") + stds_id = member["artifacts"]["rasters"]["max_water_depth"] + record_id = derived_record_name(stds_id, 0) + assert gscript.find_file(name=record_id, element="cell").get("file") + GrassSessionManager.ensure_temporal_initialized() + stds = tgis.open_stds.open_old_stds(stds_id, "strds") + assert [row["id"] for row in stds.get_registered_maps(columns="id")] == [record_id] + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_explicit_grass_context_runs_two_members_without_parent_session(test_data_temp_path): + context = gscript.gisenv() + config_path = Path(test_data_temp_path) / "explicit.yaml" + config_path.write_text( + f"""\ +schema_version: 1 +ensemble: + id: explicit +grass: + database: "{context["GISDBASE"]}" + project: "{context["LOCATION_NAME"]}" + mapset: "{context["MAPSET"]}" + executable: "grass" +time: + duration: "00:01:00" + record_step: "00:00:30" +input: + ground_elevation: "z" + friction: "n" + water_depth: "start_h" +parameters: + cfl: 0.2 + theta: [0.8, 0.9] + dtmax: 0.3 +outputs: + rasters: + prefix: "explicit_{{simulation}}" + variables: [water_depth] + manifest: + file: "explicit-manifest.yaml" +""", + encoding="utf-8", + ) + args = build_parser().parse_args(["run", str(config_path), "-o"]) + + gisrc = os.environ.pop("GISRC") + try: + itzi_run(args) + finally: + os.environ["GISRC"] = gisrc + + manifest_path = Path(test_data_temp_path) / "explicit-manifest.yaml" + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + assert len(manifest["members"]) == 2 + assert all(member["status"] == "completed" for member in manifest["members"]) + mapset_path = ( + Path(context["GISDBASE"]) / context["LOCATION_NAME"] / context["MAPSET"] / "cellhd" + ) + assert { + f"{member['artifacts']['rasters']['water_depth'].partition('@')[0]}_0000" + for member in manifest["members"] + } <= {path.name for path in mapset_path.iterdir()} + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_validates_multiple_members_in_one_session(tmp_path): + prefix = f"dry_multi_{uuid4().hex[:8]}" + config_path = write_study( + tmp_path, + ensemble={"id": "dry-multi"}, + parameters={"cfl": [0.2, 0.3]}, + outputs={"rasters": {"prefix": f"{prefix}_{{simulation}}", "variables": ["water_depth"]}}, + ) + + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (tmp_path / "results" / "dry-multi.manifest.yaml").exists() + assert not gscript.list_strings(type="raster", pattern=f"{prefix}_*") + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_rejects_null_elevation_without_creating_outputs(tmp_path): + gscript.mapcalc("dry_null_dem=null()") + config_path = write_study( + tmp_path, + ensemble={"id": "dry"}, + input={"ground_elevation": "dry_null_dem"}, + parameters={}, + outputs={ + "rasters": {"prefix": "dry_output", "variables": ["water_depth"]}, + "statistics": {"file": "dry-results/statistics.csv"}, + }, + ) + + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (tmp_path / "results" / "dry.manifest.yaml").exists() + assert not (tmp_path / "dry-results").exists() + assert not gscript.find_file(name="dry_output_water_depth_0000", element="cell").get("file") + + with pytest.raises(RuntimeError, match="failed validation or execution"): + itzi_run(build_parser().parse_args(["run", str(config_path)])) + + manifest = yaml.safe_load( + (tmp_path / "results" / "dry.manifest.yaml").read_text(encoding="utf-8") + ) + assert manifest["members"][0]["status"] == "validation_failed" + assert manifest["members"][0]["failure"]["phase"] == "run_validation" + assert not (tmp_path / "dry-results").exists() + assert not gscript.find_file(name="dry_output_water_depth_0000", element="cell").get("file") + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_does_not_read_rainfall_cells(tmp_path): + gscript.mapcalc("dry_null_rain_0=null()") + gscript.mapcalc("dry_null_rain_1=1") + gscript.run_command( + "t.create", + output="dry_rain_series", + type="strds", + temporaltype="relative", + semantictype="mean", + title="dry_rain_series", + description="dry_rain_series", + ) + gscript.run_command( + "t.register", + flags="i", + input="dry_rain_series", + type="raster", + maps="dry_null_rain_0,dry_null_rain_1", + start="0", + increment="30", + unit="seconds", + ) + config_path = write_study( + tmp_path, + ensemble={"id": "dry-rain"}, + input={"rainfall_rate": "dry_rain_series"}, + parameters={}, + outputs={"rasters": {"prefix": "dry_rain_output", "variables": ["water_depth"]}}, + ) + + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + assert not (tmp_path / "results" / "dry-rain.manifest.yaml").exists() + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_dry_run_checks_every_generated_output_record(test_data_temp_path): + gscript.mapcalc("existing_output_water_depth_0002=1") + config_path = write_study( + Path(test_data_temp_path), + ensemble={"id": "existing-output"}, + parameters={}, + outputs={"rasters": {"prefix": "existing_output", "variables": ["water_depth"]}}, + ) + + with pytest.raises(RuntimeError, match="YAML batch validation failed"): + itzi_run(build_parser().parse_args(["run", str(config_path), "--dry-run"])) + + assert not (Path(test_data_temp_path) / "results" / "existing-output.manifest.yaml").exists() + assert not gscript.find_file(name="existing_output_water_depth_0000", element="cell").get( + "file" + ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_xy_session") +def test_run_validation_accepts_unoccupied_drainage_id(): + from itzi.ensemble.run_validation import _validate_grass_outputs + from itzi.grass.interface import GrassInterface + from itzi.grass.utils import get_current_mapset + + GrassSessionManager.ensure_temporal_initialized() + name = f"drainage_collision_{uuid4().hex[:8]}" + simulation = SimpleNamespace( + simulation_config=SimpleNamespace( + start_time=datetime(2020, 1, 1), + end_time=datetime(2020, 1, 1) + timedelta(minutes=1), + record_step=timedelta(seconds=30), + output_map_names={}, + drainage_output=f"{name}@{get_current_mapset()}", + temporal_type=TemporalType.RELATIVE, + ) + ) + interface = SimpleNamespace( + overwrite=False, + validate_output_stds_temporal_type=lambda *_: None, + ) + + _validate_grass_outputs(cast(ResolvedSimulation, simulation), cast(GrassInterface, interface)) diff --git a/tests/grass/test_grass_input.py b/tests/grass/test_grass_input.py new file mode 100644 index 00000000..092fcd51 --- /dev/null +++ b/tests/grass/test_grass_input.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, cast + +import numpy as np +import pytest + +if TYPE_CHECKING: + from itzi.grass.utils import MapData + + +@pytest.fixture(scope="module", autouse=True) +def grass_runtime_env() -> None: + import grass.script as gscript + + gscript.setup.setup_runtime_env() + + +class FakeGrassInterface: + def __init__(self) -> None: + self.temporal_checks: list[str] = [] + + def stds_temporal_sanity(self, strds_id: str) -> bool: + self.temporal_checks.append(strds_id) + return True + + def raster_list_from_strds(self, strds_id: str) -> list[MapData]: + from itzi.grass.utils import MapData + + self.stds_temporal_sanity(strds_id) + start = datetime(2020, 1, 1) + return [MapData(strds_id, start, start + timedelta(hours=1))] + + +def test_resolved_input_kinds_share_raster_and_strds_construction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from itzi.grass import utils + from itzi.grass.interface import GrassInterface + from itzi.grass.utils import MapData + from itzi.providers.grass_input import GrassRasterInputProvider + + monkeypatch.setattr(utils, "name_is_stds", lambda map_id: map_id == "series@test") + monkeypatch.setattr(utils, "name_is_map", lambda map_id: map_id == "raster@test") + + start = datetime(2020, 1, 1) + end = start + timedelta(hours=1) + interface = FakeGrassInterface() + + provider = GrassRasterInputProvider( + cast(GrassInterface, interface), + {"ground_elevation": "raster@test", "rainfall_rate": "series@test"}, + start, + end, + {"ground_elevation": "raster", "rainfall_rate": "strds"}, + ) + + assert provider.map_lists["ground_elevation"] == [MapData("raster@test", start, end)] + assert provider.map_lists["rainfall_rate"] == [MapData("series@test", start, end)] + assert interface.temporal_checks == ["series@test"] + + +@pytest.mark.forked +def test_visible_input_and_current_output_with_same_name(grass_xy_session) -> None: + import grass.script as gscript + + from itzi.grass.interface import GrassInterface + from itzi.grass.utils import output_name, qualify_output_id + from itzi.providers.grass_output import derived_record_name + + gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) + gscript.mapcalc("visible_output=2") + gscript.run_command("g.mapset", mapset="child", flags="c") + output_id = qualify_output_id("visible_output", "child") + assert output_id == "visible_output@child" + assert derived_record_name(output_id, 0) == "visible_output_0000@child" + assert output_name(output_id) == "visible_output" + with pytest.raises(ValueError, match="current mapset"): + qualify_output_id("visible_output@PERMANENT", "child") + + gscript.run_command("g.region", n=3, s=0, e=3, w=0, res=1) + start = datetime(2020, 1, 1) + with GrassInterface(start, start + timedelta(seconds=1), np.int32, None, None) as interface: + interface.write_raster_map( + np.ones((3, 3), dtype=np.int32), output_id, "ground_elevation", 0 + ) + assert gscript.parse_command("r.univar", map=output_id, flags="g")["min"] == "1" + assert ( + gscript.parse_command("r.univar", map="visible_output@PERMANENT", flags="g")["min"] == "2" + ) + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_direct_runner_qualifies_ids_before_creating_providers(test_data_path: str) -> None: + from pathlib import Path + + from itzi.configreader import ConfigReader + from itzi.grass.session import GrassSessionManager + from itzi.simulation_runner import SimulationRunner + + GrassSessionManager.ensure_temporal_initialized() + reader = ConfigReader(str(Path(test_data_path) / "5by5" / "5by5.ini")) + config = reader.sim_config.model_copy( + update={"output_map_names": {"water_depth": "direct_id_check"}} + ) + runner = SimulationRunner(config, reader.grass_params) + + assert runner.sim_config.input_map_names["ground_elevation"] == "z@5by5" + assert runner.sim_config.output_map_names["water_depth"] == "direct_id_check@5by5" + + +@pytest.mark.forked +@pytest.mark.usefixtures("grass_5by5") +def test_vector_writer_uses_bare_name_from_qualified_output_id() -> None: + from grass.pygrass.utils import get_mapset_vector + from itzi_core.data_containers import DrainageNetworkAttributes, DrainageNetworkTopology + + from itzi.grass.interface import GrassInterface + + start = datetime(2020, 1, 1) + with GrassInterface(start, start + timedelta(seconds=1), np.float32, None, None) as interface: + interface.write_vector_map( + DrainageNetworkTopology(nodes=(), links=()), + DrainageNetworkAttributes(nodes=(), links=()), + "vector_id_check@5by5", + ) + assert get_mapset_vector("vector_id_check", "5by5") == "5by5" diff --git a/tests/grass/test_itzi.py b/tests/grass/test_itzi.py index 47f41cb3..68e3f148 100644 --- a/tests/grass/test_itzi.py +++ b/tests/grass/test_itzi.py @@ -9,6 +9,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader +from itzi.grass.session import GrassSessionManager def _create_timed_rain_inputs(name_prefix: str) -> dict[str, str]: @@ -89,11 +90,12 @@ def _build_timed_rain_runner( parser.write(file_handle) conf_data = ConfigReader(config_file) + GrassSessionManager.ensure_temporal_initialized() return SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, + ).initialize() @pytest.mark.forked @@ -155,15 +157,14 @@ def test_region_mask(test_data_path): # Set simulation (should set region and mask) config_file = os.path.join(test_data_path, "5by5", "5by5_mask.ini") conf_data = ConfigReader(config_file) - sim_params = conf_data.get_sim_params() - grass_params = conf_data.get_grass_params() + sim_params = conf_data.sim_config + grass_params = conf_data.grass_params sim_runner = SimulationRunner( sim_params, grass_params, - stats_file=conf_data.get_stats_file(), ) # Run simulation - sim_runner.run().finalize() + sim_runner.initialize().run().finalize() # Check temporary mask and region assert ( int(gscript.parse_command("r.univar", map="out_5by5_max_flow_speed_0002", flags="g")["n"]) @@ -216,10 +217,10 @@ def test_fails_when_region_has_no_dem_data(test_data_temp_path): RuntimeError, match=r"input map contains only NULL/NaN cells" ): SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, + ).initialize() @pytest.mark.forked @@ -340,7 +341,6 @@ def test_timed_grass_rain_switches_cleanly_around_boundary( seconds=expected_input_deadline_seconds ) finally: - sim_runner.g_interface.finalize() sim_runner.g_interface.cleanup() @@ -371,5 +371,4 @@ def test_timed_grass_rain_is_applied_before_a_step_crosses_its_boundary(test_dat 360.0 / (1000 * 3600), ) finally: - sim_runner.g_interface.finalize() sim_runner.g_interface.cleanup() diff --git a/tests/grass/test_temporal_overwrite.py b/tests/grass/test_temporal_overwrite.py index 954baa64..8fac527b 100644 --- a/tests/grass/test_temporal_overwrite.py +++ b/tests/grass/test_temporal_overwrite.py @@ -1,25 +1,16 @@ from __future__ import annotations -from collections.abc import Iterator -from configparser import ConfigParser import os +from configparser import ConfigParser from uuid import uuid4 import grass.script as gscript import pytest +from itzi_core.const import TemporalType from itzi import SimulationRunner from itzi.configreader import ConfigReader -from itzi_core.const import TemporalType - - -@pytest.fixture(autouse=True) -def stop_temporal_subprocesses() -> Iterator[None]: - yield - - import grass.temporal as tgis - - tgis.stop_subprocesses() +from itzi.grass.session import GrassSessionManager def _build_runner( @@ -65,9 +56,9 @@ def _build_runner( conf_data = ConfigReader(config_file) return SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, ) @@ -77,12 +68,13 @@ def test_temporal_fatal_errors_are_raised_as_exceptions() -> None: import grass.temporal as tgis from grass.exceptions import FatalError - from itzi.providers.grass_interface import GrassInterface + from itzi.grass.utils import name_is_stds tgis.set_raise_on_error(False) + GrassSessionManager.ensure_temporal_initialized() with pytest.raises(FatalError, match="mapset is missing"): - GrassInterface.name_is_stds("missing_strds") + name_is_stds("missing_strds") assert gscript.get_raise_on_error() is True assert tgis.get_raise_on_error() is True @@ -107,7 +99,7 @@ def test_runner_creation_fails_when_overwriting_strds_with_different_temporal_ty prefix = f"out_overwrite_{existing_temporal_type}_{simulation_temporal_type}_{uuid4().hex[:8]}" initial_runner = _build_runner(test_data_temp_path, prefix, existing_temporal_type) - initial_runner.run().finalize() + initial_runner.initialize().run().finalize() with pytest.raises(RuntimeError, match=r"temporal type"): - _build_runner(test_data_temp_path, prefix, simulation_temporal_type) + _build_runner(test_data_temp_path, prefix, simulation_temporal_type).initialize() diff --git a/tests/grass/test_tutorial.py b/tests/grass/test_tutorial.py index 8bed70c6..46175694 100644 --- a/tests/grass/test_tutorial.py +++ b/tests/grass/test_tutorial.py @@ -14,6 +14,7 @@ from itzi import SimulationRunner from itzi.configreader import ConfigReader +from itzi.grass.session import GrassSessionManager DATA_EPSG = "3358" @@ -29,15 +30,20 @@ def grass_tutorial_session(test_data_temp_path): """Create a GRASS session in a new location and PERMANENT mapset""" # Keep all generated files in the test_data_temp_path os.chdir(test_data_temp_path) + workdir = tempfile.TemporaryDirectory(dir=test_data_temp_path) + os.chdir(workdir.name) tmpdir = tempfile.TemporaryDirectory() gscript.create_project(tmpdir.name, name="itzi_tutorial", epsg=DATA_EPSG) grass_session = gscript.setup.init( path=tmpdir.name, location="itzi_tutorial", mapset="PERMANENT" ) + GrassSessionManager.ensure_temporal_initialized() os.environ["GRASS_VERBOSE"] = "1" yield grass_session grass_session.finish() tmpdir.cleanup() + os.chdir(test_data_temp_path) + workdir.cleanup() @pytest.fixture(scope="class") @@ -95,17 +101,17 @@ def itzi_tutorial(grass_tutorial_session, tutorial_test_file): @pytest.mark.slow @pytest.mark.usefixtures("itzi_tutorial", "test_data_path") class TestItziTutorial: - def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): + def test_tutorial(itzi_tutorial, test_data_path): """Run the tutorial simulation. Check the results.""" # Run the simulation config_file = os.path.join(test_data_path, "tutorial_files", "tutorial.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, + ).initialize() sim_runner.run().finalize() # Check the results hmax_maps = gscript.list_grouped("raster", pattern="nc_itzi_tutorial_max_water_depth_*")[ @@ -116,7 +122,7 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): assert float(h_max_univar["mean_of_abs"]) == pytest.approx(0.0355, abs=1e-3) # Test consistency of stats file - stat_file_path = pathlib.Path(test_data_temp_path) / pathlib.Path("nc_itzi_tutorial.csv") + stat_file_path = pathlib.Path("nc_itzi_tutorial.csv") df_stats = pd.read_csv(stat_file_path, na_values="-") df_stats.set_index("simulation_time", drop=True, inplace=True, verify_integrity=True) df_stats.index = pd.to_timedelta(df_stats.index) @@ -134,7 +140,7 @@ def test_tutorial(itzi_tutorial, test_data_path, test_data_temp_path): "'volume_change' inconsistent with other values" ) - def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, helpers): + def test_tutorial_drainage(itzi_tutorial, test_data_path, helpers): """Run the tutorial simulation with drainage.""" # Set the config file dynamically to make sure it can find the INP file inp_file = os.path.join(test_data_path, "tutorial_files", "tutorial_drainage.inp") @@ -157,23 +163,21 @@ def test_tutorial_drainage(itzi_tutorial, test_data_path, test_data_temp_path, h } parser = ConfigParser() parser.read_dict(config_dict) - config_file = os.path.join(test_data_temp_path, "tutorial_drainage.ini") + config_file = "tutorial_drainage.ini" with open(config_file, "w") as f: parser.write(f) # Run the simulation conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, + ).initialize() sim_runner.run().finalize() # Test consistency of stats file - stat_file_path = pathlib.Path(test_data_temp_path) / pathlib.Path( - "nc_itzi_tutorial_drainage.csv" - ) + stat_file_path = pathlib.Path("nc_itzi_tutorial_drainage.csv") df_stats = pd.read_csv(stat_file_path, na_values="-") df_stats.set_index("simulation_time", drop=True, inplace=True, verify_integrity=True) df_stats.index = pd.to_timedelta(df_stats.index) @@ -275,10 +279,10 @@ def test_tutorial_drainage_without_vector_output(itzi_tutorial, test_data_path): config_file = os.path.join(test_data_path, "tutorial_files", "tutorial_drainage.ini") conf_data = ConfigReader(config_file) sim_runner = SimulationRunner( - conf_data.get_sim_params(), - conf_data.get_grass_params(), - stats_file=conf_data.get_stats_file(), - ) + conf_data.sim_config, + conf_data.grass_params, + stats_file=conf_data.stats_file, + ).initialize() try: assert sim_runner.sim.report.vector_provider is None diff --git a/tests/test_data/5by5/5by5.ini b/tests/test_data/5by5/5by5.ini index a8b49768..f7b1c622 100644 --- a/tests/test_data/5by5/5by5.ini +++ b/tests/test_data/5by5/5by5.ini @@ -11,9 +11,6 @@ record_step = 00:00:30 prefix = out_5by5 values = water_depth, max_water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, froude, flow_rate_x, flow_rate_y, created_volume -[statistics] -stats_file = 5by5.csv - [options] hmin = 0.0001 dtmax = 0.3 diff --git a/tests/test_data/5by5/5by5_mask.ini b/tests/test_data/5by5/5by5_mask.ini index 4feaac48..c29e28a9 100644 --- a/tests/test_data/5by5/5by5_mask.ini +++ b/tests/test_data/5by5/5by5_mask.ini @@ -11,9 +11,6 @@ record_step = 00:00:30 prefix = out_5by5 values = water_depth, water_surface_elevation, flow_speed, max_flow_speed, flow_velocity_direction, flow_rate_x, flow_rate_y -[statistics] -stats_file = 5by5.csv - [options] hmin = 0.001 dtmax = 0.3 diff --git a/tests/test_grass_session.py b/tests/test_grass_session.py index 99e5e9c1..ca9bc1c8 100644 --- a/tests/test_grass_session.py +++ b/tests/test_grass_session.py @@ -1,33 +1,67 @@ +import os import sys from types import ModuleType, SimpleNamespace -from itzi.grass_session import GrassSessionManager +import pytest + +from itzi.grass.session import GrassParams, GrassSessionManager + + +@pytest.mark.parametrize( + "values", + ( + {"grassdata": "/grassdata"}, + {"grassdata": "/grassdata", "location": "project"}, + {"mapset": "PERMANENT"}, + ), +) +def test_grass_params_requires_complete_context(values: dict[str, str]) -> None: + with pytest.raises( + ValueError, match="GRASS database, location, and mapset must be supplied together" + ): + GrassParams(**values) + + assert GrassParams(region="region").grassdata is None + assert GrassParams("/grassdata", "project", "PERMANENT").mapset == "PERMANENT" class FakeGrassSession: - def __init__(self) -> None: + def __init__(self, env: dict[str, str]) -> None: + self.env = env self.finished = False def finish(self) -> None: self.finished = True -def test_close_finishes_session_created_by_manager(monkeypatch) -> None: - session = FakeGrassSession() +def test_manager_activates_and_finishes_created_session(monkeypatch) -> None: + session = FakeGrassSession({"GISRC": "/tmp/worker-gisrc"}) grass_script = ModuleType("grass.script") - grass_script.setup = SimpleNamespace(init=lambda **_: session) + grass_script.__dict__["setup"] = SimpleNamespace(init=lambda **_: session) grass_package = ModuleType("grass") - grass_package.script = grass_script + grass_package.__dict__["script"] = grass_script - monkeypatch.setattr("itzi.grass_session.importlib.util.find_spec", lambda _: None) - monkeypatch.setattr("itzi.grass_session.os.access", lambda *_: True) + monkeypatch.setattr("itzi.grass.session.os.access", lambda *_: True) + monkeypatch.setattr(GrassSessionManager, "ensure_temporal_initialized", lambda self: None) + monkeypatch.setattr( + "itzi.grass.session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" + ) + monkeypatch.delenv("GISRC", raising=False) monkeypatch.setattr( - "itzi.grass_session.subprocess.check_output", lambda *_args, **_kwargs: "/tmp" + GrassSessionManager, + "current_context", + staticmethod( + lambda: ( + ("/grassdata", "location", "mapset") + if os.environ.get("GISRC") == session.env["GISRC"] + else None + ) + ), ) monkeypatch.setitem(sys.modules, "grass", grass_package) monkeypatch.setitem(sys.modules, "grass.script", grass_script) - grass_params = SimpleNamespace( + grass_params = GrassParams( grassdata="/grassdata", location="location", mapset="mapset", @@ -38,4 +72,35 @@ def test_close_finishes_session_created_by_manager(monkeypatch) -> None: manager.open() manager.close() + assert os.environ["GISRC"] == "/tmp/worker-gisrc" assert session.finished + + +def test_temporal_initialization_once_per_active_session(monkeypatch) -> None: + calls: list[str] = [] + script = ModuleType("grass.script") + script.__dict__["set_raise_on_error"] = lambda _: calls.append("script") + temporal = ModuleType("grass.temporal") + temporal.__dict__["init"] = lambda **_: calls.append("init") + temporal.__dict__["set_raise_on_error"] = lambda _: calls.append("temporal") + package = ModuleType("grass") + package.__dict__["script"] = script + package.__dict__["temporal"] = temporal + monkeypatch.setitem(sys.modules, "grass", package) + monkeypatch.setitem(sys.modules, "grass.script", script) + monkeypatch.setitem(sys.modules, "grass.temporal", temporal) + monkeypatch.setattr("itzi.grass.session._initialized_temporal_session", None) + monkeypatch.setattr( + GrassSessionManager, + "current_context", + staticmethod(lambda: ("/grassdata", "location", "mapset")), + ) + monkeypatch.setenv("GISRC", "/tmp/first-gisrc") + + GrassSessionManager(GrassParams()).open() + GrassSessionManager.ensure_temporal_initialized() + assert calls.count("init") == 1 + + monkeypatch.setenv("GISRC", "/tmp/second-gisrc") + GrassSessionManager(GrassParams()).open() + assert calls.count("init") == 2 diff --git a/uv.lock b/uv.lock index 71aa33b8..7328cc9f 100644 --- a/uv.lock +++ b/uv.lock @@ -346,6 +346,7 @@ dependencies = [ { name = "itzi-core" }, { name = "numpy" }, { name = "pydantic" }, + { name = "pyyaml" }, ] [package.dev-dependencies] @@ -368,6 +369,7 @@ requires-dist = [ { name = "itzi-core", specifier = "==0.8.0" }, { name = "numpy", specifier = ">=2.5" }, { name = "pydantic", specifier = ">=2.10" }, + { name = "pyyaml", specifier = ">=6.0" }, ] [package.metadata.requires-dev] @@ -381,7 +383,7 @@ dev = [ { name = "sphinx", specifier = ">=9.1.0" }, { name = "sphinx-argparse", specifier = ">=0.5.2" }, { name = "sphinx-rtd-theme", specifier = ">=3.1.0" }, - { name = "ty", specifier = ">=0.0.55" }, + { name = "ty", specifier = ">=0.0.84" }, ] [[package]] @@ -1143,27 +1145,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.56" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/07/fb29aea5235b0aa8ecfc4d1cc6ddf9fba8b863d67d96c6d345694d644c43/ty-0.0.56.tar.gz", hash = "sha256:84d114dc3796361c0fc72945016eabd74d46b9ee64f198cb0e485719704681e5", size = 6050123, upload-time = "2026-07-01T16:44:56.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/48/bce79e7ca5c1cc529d3e0d37ddd1121aea4b68a4f749974ad1cc77161871/ty-0.0.56-py3-none-linux_armv6l.whl", hash = "sha256:186d4a53e15747c947e1ec3d7eec8e345d8e40a1ca10e634c585db52497e87dd", size = 11643066, upload-time = "2026-07-01T16:44:18.374Z" }, - { url = "https://files.pythonhosted.org/packages/80/d1/22555d8a1d719661f10050f3865d877bbf497da908961c75fe22217dd18a/ty-0.0.56-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aae1a980fd9535da0469b7ba2b2e1b54a907743a5e0f442dd57eee9f5bfd034c", size = 11407487, upload-time = "2026-07-01T16:44:20.956Z" }, - { url = "https://files.pythonhosted.org/packages/cf/2d/b3b7a74ce8bc59ef48843ad80179bb0d9598bbd6cfc0d11d519bdf6b1352/ty-0.0.56-py3-none-macosx_11_0_arm64.whl", hash = "sha256:afd3058c0a6c5f241e814734f133008c93ee805f61c9cf4ce7412b8822b5d9ad", size = 10962270, upload-time = "2026-07-01T16:44:22.959Z" }, - { url = "https://files.pythonhosted.org/packages/64/ac/6c2fd7de0304a8a7218a756af74f7e62a5e8540fdb175e0a869e51042345/ty-0.0.56-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:058b52f7a823ac13aae3cae30809dd6b5145794b64d8478f9ef38c75d79b4483", size = 11471406, upload-time = "2026-07-01T16:44:25.327Z" }, - { url = "https://files.pythonhosted.org/packages/50/b6/11d861156861c03c7726b74558f9a0e0092661aff83a4fda1279df28c425/ty-0.0.56-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c66e00c1522add1f2bbdd2e45828c953b35c306b7bef03ec9169c75a63699a0", size = 11445612, upload-time = "2026-07-01T16:44:27.531Z" }, - { url = "https://files.pythonhosted.org/packages/fb/ba/09df108582090f3c0770ec4bc8675affed60248f6793a78d909be16211d9/ty-0.0.56-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40903d71c669a30691b5a5d5728056c7877a1bd6be4f233a38883a8b28cf34d7", size = 12093889, upload-time = "2026-07-01T16:44:29.548Z" }, - { url = "https://files.pythonhosted.org/packages/d7/f7/dbb4b4ccb69cd64c209ae55b1ab788ace8222c2bc1f6845be9e7cbedbf25/ty-0.0.56-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63fe3947fe0c46c69a7d950e6832ee70a9ec17321fefbff3d2e3c20baf9e5bd0", size = 12666337, upload-time = "2026-07-01T16:44:31.586Z" }, - { url = "https://files.pythonhosted.org/packages/86/e9/73f903fe4a3d9ea02f26f57c1eb07e3b1029ec92b0e8c2364718893440e3/ty-0.0.56-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71a0c1a72f9854532e710e119b6871ffe4542c8a65146f1f65dcd78fecd885b4", size = 12280247, upload-time = "2026-07-01T16:44:33.637Z" }, - { url = "https://files.pythonhosted.org/packages/d6/90/cebd222495832f1a00dcd321ba25f3cab804221a4991b992c2178bec68ee/ty-0.0.56-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70d1665596494e24d8ebd198438872b5a56ec3cae5f2bcf6c673be797acc4e3c", size = 11991107, upload-time = "2026-07-01T16:44:36.122Z" }, - { url = "https://files.pythonhosted.org/packages/b7/07/8f7337a07250f42d975cdb6decf47fc5b421e6c7da5e3e7be1e85f63a7e5/ty-0.0.56-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:778f99e51558afc1dbbe48ee38ab6aae7b31390ed8c1a1ef1499b295e9f1e82f", size = 12298970, upload-time = "2026-07-01T16:44:38.243Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b9/a52cd59034a48f5f18c6b155cc2cc36861d874b6d0af204b12c898024c3d/ty-0.0.56-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:867bc5708e0066bb4ff6c7db524bd5deea2676c62bfe71d3303138b3be850af0", size = 11425683, upload-time = "2026-07-01T16:44:40.473Z" }, - { url = "https://files.pythonhosted.org/packages/1d/2e/48e42d33357d52eefb695c0c3fcfc96879b73668a7447d1d1e0ad774fedc/ty-0.0.56-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a6012f4189c928edb330a37deb9930f982380bd4aa7c4b8e0428eec9651c7551", size = 11469258, upload-time = "2026-07-01T16:44:42.513Z" }, - { url = "https://files.pythonhosted.org/packages/d5/01/ad1b4138be1e3fa97863af3925aa2134f17a593240c35dc38c3429fb5ad1/ty-0.0.56-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee83de1a7ff4cc32837ec06134ce391d441bc5b35ecd8d3cfe053f120f3e4c1", size = 11758736, upload-time = "2026-07-01T16:44:44.567Z" }, - { url = "https://files.pythonhosted.org/packages/09/34/9d81967ff240eaa57e9249728ef7b7790747cf6d3c9a98ec86b2cfdcc8ee/ty-0.0.56-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:62619b3b0e2c6248ef30d3f0e2f2217ae9893040585be07f32324242f197cd6f", size = 12100242, upload-time = "2026-07-01T16:44:46.584Z" }, - { url = "https://files.pythonhosted.org/packages/c3/36/f51d4666d2de6cf33c1f3a1fcc4bb6b70b197dd6ceaa491eef71d78fe8e8/ty-0.0.56-py3-none-win32.whl", hash = "sha256:b30687bb5cd9729d34c889a289edf32770388d9bb05243e534e723fb45e0381b", size = 11093759, upload-time = "2026-07-01T16:44:49.171Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b4/8fb5d4acfa4afb152245b20fa263069a7547bd1f8e4bfca4eda280c897d7/ty-0.0.56-py3-none-win_amd64.whl", hash = "sha256:ad4c8c47b6f4e3f9ed3fc0b1a5d650088d229e17dd8f63c1826d6bbe94cc3235", size = 12100327, upload-time = "2026-07-01T16:44:51.26Z" }, - { url = "https://files.pythonhosted.org/packages/b8/fc/6a183e71edde90d0c35c2303f23f7a45b6891d1a2c45daf7b8f869831e19/ty-0.0.56-py3-none-win_arm64.whl", hash = "sha256:57538f273d444a5f1293fa7860e967178afe3917611fc5eff16b64e1204fe0d6", size = 11538780, upload-time = "2026-07-01T16:44:53.8Z" }, +version = "0.0.84" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/f6/34f8869c45ea87fe6a931ffa824b1fe4428167173543d92988d631add355/ty-0.0.84.tar.gz", hash = "sha256:0cefdb0cd5d399418dbe83c349ba605047b7dff815ae6f460c80e8522b40be67", size = 7409537, upload-time = "2026-09-24T13:16:24.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/0c/600648778e1c79133ce6b6a5c4c2531ae16b90c9b30f5041b38b574f61b8/ty-0.0.84-py3-none-linux_armv6l.whl", hash = "sha256:868438e2b9abfc835dd5b5b07ed693db95d8348ac40e3228252511d63ae87695", size = 13935333, upload-time = "2026-09-24T13:15:31.167Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/27f06f49945c36170765fad82076898039ae3873338e9865f876dd62f12c/ty-0.0.84-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b00f39b521f0b559cde0f6abf8dd47213b513b04f33d58efeb9b98486a16bb17", size = 13511328, upload-time = "2026-09-24T13:15:34.446Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/b12dd130e7f30141f0645614f454c925991cd389b1cc6c8284fd6bf32326/ty-0.0.84-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f78c2915567f2ee62ce0c24d86091bc76f81577677755e242d09951db588b9a", size = 13445445, upload-time = "2026-09-24T13:15:37.476Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/f012de4b3c1d9a4773326621380d768b4d48c47e620dc012775c5e01c7d7/ty-0.0.84-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61155cad0921e890d86d1d911a40b3a0e924ef4bdc3b4f3865a6c89c7e3012dc", size = 13471235, upload-time = "2026-09-24T13:15:40.64Z" }, + { url = "https://files.pythonhosted.org/packages/05/32/dca630f5f5353fa7bf67eebccf1549df99b16656d21071d0df6677c826a5/ty-0.0.84-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:417c3bf14947ed16b0b92976ab333c27915590354e96d4f581709068d3811355", size = 13680704, upload-time = "2026-09-24T13:15:43.817Z" }, + { url = "https://files.pythonhosted.org/packages/44/8d/33702c8f62f05ad45b33644fb914d483de6a0a7bc8e51884e9b8773e6d69/ty-0.0.84-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e06ef117759f654f5379efe9ae79bda8a186016c016f1023d05c34f8e15c1546", size = 14518923, upload-time = "2026-09-24T13:15:46.802Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a1/da3246e6ef2ae0e842bca4058b25c54bee55ab058ffff4ee54e72517ec01/ty-0.0.84-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a05d1d53db8b64410ab38c6e32c5898510584869d1d5dd9c0605db81c9aef7df", size = 15027731, upload-time = "2026-09-24T13:15:49.755Z" }, + { url = "https://files.pythonhosted.org/packages/20/17/b0099098a560aeb4f9fde6c912f9a81622c3e44e6555e1fbe322b368ec31/ty-0.0.84-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b92400494ab855cfb1204ac1101e3f53b2d09a8e21d171758b2cf9acdab1809e", size = 14748260, upload-time = "2026-09-24T13:15:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/60/d5/32cd67aedb271f006e716ad093806c96c5d7e2af86be8255469dbd3d76c8/ty-0.0.84-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af17eb391ae3027fed9be1780bea555c2b8a25ba68e184c627b4981b2c9eaab7", size = 14205858, upload-time = "2026-09-24T13:15:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/638ac62ca2e8eca1b92e3e16273085744fada728b993500819b5c258ca15/ty-0.0.84-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:44cf06af0e7ec78ea6d8ff33a3450fc19627756a4fba681f7410d4e3ef63047a", size = 14573196, upload-time = "2026-09-24T13:15:59.165Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f6/9ee81c5d8921e5fd5f10889ae563c95cd3272b31df2ada528919ca1cef6b/ty-0.0.84-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c6e56b443ff9ba462e34bfbc1e582004c2719ed35f8f93a433744b0ba0afb2d9", size = 13480760, upload-time = "2026-09-24T13:16:02.313Z" }, + { url = "https://files.pythonhosted.org/packages/f6/83/fbe5d1177667ae16d84c3242846b143809c9cc09568b333e5a9b4d5908f6/ty-0.0.84-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8b6562fcc48820c1030e2a4509ef2c14ef93043ca4860280074314ac3a825b95", size = 13696346, upload-time = "2026-09-24T13:16:05.391Z" }, + { url = "https://files.pythonhosted.org/packages/86/9a/1b4ac9ddc34cc0ab4624bc219825d66af52f1ad87f4778ad08ad03fa06d5/ty-0.0.84-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dee7421451efcbd70f03e95a8f2c8d9e5ec3d35edcd27b4095ee00ee062abfc4", size = 13959410, upload-time = "2026-09-24T13:16:08.661Z" }, + { url = "https://files.pythonhosted.org/packages/c9/04/70e289437a67dfd686056ab4fd91880261a8233033a6a95d29393741fb97/ty-0.0.84-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:31e96307cf621babe969307dd657783d672e2d8bdbd06493710da635bf63fa07", size = 14350294, upload-time = "2026-09-24T13:16:11.865Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f4/ae7dbd118425c850981d23efb850f0d9d7f6b8d2b852bde2767434f668b4/ty-0.0.84-py3-none-win32.whl", hash = "sha256:4bdc85f91cbaae35825f2b17bb4ac5de0563367875cb0f42b2bf07a1bf7ff2ba", size = 13176439, upload-time = "2026-09-24T13:16:15.049Z" }, + { url = "https://files.pythonhosted.org/packages/29/ec/994d87252f065dda5a500bc8afc726d18f6582d81d6f4ff1feb8a267c800/ty-0.0.84-py3-none-win_amd64.whl", hash = "sha256:71cc9aa7a7c89d4e12c6a814e9c0c3310ad5bee44efedbd6f5a42d91e9ac0f0d", size = 13962879, upload-time = "2026-09-24T13:16:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, ] [[package]]