From 1d3533cb10df2ef4be83a3d3c6f2a278dd1d1a81 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Tue, 18 Aug 2026 14:19:12 +0200 Subject: [PATCH 01/23] debug log for each utp with values from worker --- .../calculations/systems/build_and_run.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 085970c9..4bb661ff 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -95,16 +95,27 @@ def _worker(row_unique: dict): # noinspection PyBroadException try: with redirect_stdout(log_buffer), redirect_stderr(log_buffer): + logger = logging.getLogger(__name__) + logger.debug("Start berekening voor %s", row_unique) # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() - + logger.debug("SystemCalculation voltooid:") + # Collect results df_limit_state = collect_df_beta_limit_state(calc) + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") df_scenario_rp = collect_df_beta_scenario_rp(calc) + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast}") df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) @@ -115,7 +126,7 @@ def _worker(row_unique: dict): df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, validation_message=calc.validation_messages - ), None, None + ), log_buffer.getvalue(), row_unique except Exception: tb = traceback.format_exc() From 0fc6d2c1b21f01a7d467f8c66a35d891d9a69e65 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 09:19:21 +0200 Subject: [PATCH 02/23] worker error log renamed and only messeges on actual errors --- .../calculations/systems/build_and_run.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 4bb661ff..e4b4687e 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -100,8 +100,9 @@ def _worker(row_unique: dict): # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() - logger.debug("SystemCalculation voltooid:") - + logger.debug("SystemCalculation voltooid.") + logger.debug("Validation messeges:") + logger.debug(f"\n{calc.validation_messages.df}") # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") @@ -176,24 +177,27 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes last_report = time.time() done = 0 + log_errors = 0 results: List[CalcResult] = [] pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) # Multiprocessing setup - error_rows = [] + log_rows = [] with Pool(processes=pool_size, initializer=_init_worker, initargs=( geohydrologisch_model, geopackage_filepath, to_run_vakken_ids)) as pool: - for res, error_logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): + for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): if isinstance(res, CalcResult): results.append(res) - if isinstance(error_logs, str): - error_rows.append({ + if isinstance(logs, str): + log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], "ondergrondscenario_naam": row["ondergrondscenario_naam"], "vak_id": row["vak_id"], - "error_logs": error_logs, + "logs": logs, }) + if "ERROR" in logs: + log_errors += 1 done += 1 # Alleen kijken of er gelogd moet worden bij de laatste @@ -208,25 +212,21 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes # Log error_count_append = "" - if error_rows.__len__() > 0: - error_count_append = f" (of which {error_rows.__len__()} failed calculations)" + + if log_errors > 0: + error_count_append = f" (of which {log_errors} failed calculations)" logger.info(f"Progress: {done:>{char_len_total}} / {n_calc_totaal} calculations{error_count_append}.") last_report = now # Push errors to database (if any) conn = sqlite3.connect(geoprob_pipe.input_data.app_settings.geopackage_filepath) - table_name = "calculation_error_logs" - if error_rows.__len__() > 0: - df_errors = DataFrame(data=error_rows) - df_errors.to_sql(table_name, conn, if_exists="replace", index=False) - conn.close() - logger.error(f"There are {error_rows.__len__()} failed calculations. Error logs are stored inside the " + table_name = "calculation_logs" + df_logs = DataFrame(data=log_rows) + df_logs.to_sql(table_name, conn, if_exists="replace", index=False) + conn.commit() + conn.close() + if log_errors > 0: + logger.error(f"There are {log_errors} failed calculations. Error logs are stored inside the " f"GeoPacakge in table '{table_name}'.") - else: - # Remove old table (if exists) - cur = conn.cursor() - cur.execute(f"DROP TABLE IF EXISTS {table_name};") - conn.commit() - conn.close() - + return results From d5effc79430f694af3cf4734ca8a5ef807213985 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 10:07:26 +0200 Subject: [PATCH 03/23] 0 total model runs warning --- geoprob_pipe/calculations/systems/build_and_run.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index e4b4687e..645b54ee 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -101,12 +101,14 @@ def _worker(row_unique: dict): calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() logger.debug("SystemCalculation voltooid.") - logger.debug("Validation messeges:") + logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") logger.debug(f"\n{df_limit_state}") + if any(r == 0 for r in df_limit_state.total_model_runs): + logger.warning("Limit state with 0 total model runs encountered.") df_scenario_rp = collect_df_beta_scenario_rp(calc) logger.debug("df_scenario_rp:") logger.debug(f"\n{df_scenario_rp}") @@ -178,6 +180,7 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes last_report = time.time() done = 0 log_errors = 0 + error_rows = [] results: List[CalcResult] = [] pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) @@ -196,8 +199,9 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes "vak_id": row["vak_id"], "logs": logs, }) - if "ERROR" in logs: + if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): log_errors += 1 + error_rows.append(row) done += 1 # Alleen kijken of er gelogd moet worden bij de laatste @@ -226,7 +230,8 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes conn.commit() conn.close() if log_errors > 0: - logger.error(f"There are {log_errors} failed calculations. Error logs are stored inside the " - f"GeoPacakge in table '{table_name}'.") + logger.error(f"There are {log_errors} failed calculations. The calculation logs are stored inside the " + f"GeoPackage in table '{table_name}'. The following rows are marked:" + f"\n{error_rows}") return results From bdb16a14c4fe12aa99cf445ea172274e30e326cc Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 11:50:40 +0200 Subject: [PATCH 04/23] Form settings test --- .../systems/base_objects/system_calculation.py | 2 ++ geoprob_pipe/calculations/systems/build_and_run.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 112135d0..c7d66583 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,7 +13,9 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, + # "minimum_iterations": 10, "relaxation_factor": 0.4, + "reuse_calculations": False, } # DEFAULT_RELIABILITY_SETTINGS: Dict[str, Union[str, float, int]] = { diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 645b54ee..635e2ca1 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -97,12 +97,25 @@ def _worker(row_unique: dict): with redirect_stdout(log_buffer), redirect_stderr(log_buffer): logger = logging.getLogger(__name__) logger.debug("Start berekening voor %s", row_unique) + # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() + logger.debug("SystemCalculation voltooid.") logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") + + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() + + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() + # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") From f7df84ccec898a5d71e42e639d6ee32d71fe6384 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 14:08:45 +0200 Subject: [PATCH 05/23] test setting update --- .../calculations/systems/base_objects/system_calculation.py | 3 ++- geoprob_pipe/calculations/systems/build_and_run.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index c7d66583..95abe9f1 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,7 +13,8 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, - # "minimum_iterations": 10, + # "minimum_iterations": 5, # Don't allow zero iterations + # "minimum_directions": 2, # Prevent one direction solutions (single dominant alpha) "relaxation_factor": 0.4, "reuse_calculations": False, } diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 635e2ca1..f8320caa 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -131,7 +131,11 @@ def _worker(row_unique: dict): df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast}") + logger.debug(f"\n{df_stochast.to_string()}") + if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") + if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) From 9657ebe0e7fc257a73dca96de1f737db646a27ed Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Thu, 20 Aug 2026 10:55:12 +0200 Subject: [PATCH 06/23] debug only tabel in calculation logs --- .../calculations/systems/build_and_run.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index f8320caa..6691a69f 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -17,7 +17,7 @@ collect_stochast_values, calculate_derived_values) import logging from pandas import DataFrame - +import os if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe from geoprob_pipe.calculations.systems.base_objects\ @@ -103,35 +103,40 @@ def _worker(row_unique: dict): calc.run() logger.debug("SystemCalculation voltooid.") + logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") - - logger.debug("Limit states print:") - for lm in calc.results.dps_limit_states: - lm.print() - - logger.debug("Combine project print:") - calc.results.combine_project.design_point.print() + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() - logger.debug("Reliability project print:") - calc.results.reliability_project.design_point.print() + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() # Collect results df_limit_state = collect_df_beta_limit_state(calc) - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") if any(r == 0 for r in df_limit_state.total_model_runs): logger.warning("Limit state with 0 total model runs encountered.") df_scenario_rp = collect_df_beta_scenario_rp(calc) - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast.to_string()}") if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): From 675d8f3d3e4f8f7731e5cdc9fd213c197a27d788 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Thu, 20 Aug 2026 16:13:12 +0200 Subject: [PATCH 07/23] Herindeling build_and_run.py --- geoprob_pipe/app_object.py | 4 +- .../base_objects/system_calculation.py | 9 - .../calculations/systems/build_and_run.py | 351 ++++++++++-------- 3 files changed, 198 insertions(+), 166 deletions(-) diff --git a/geoprob_pipe/app_object.py b/geoprob_pipe/app_object.py index 549c6692..32716b57 100644 --- a/geoprob_pipe/app_object.py +++ b/geoprob_pipe/app_object.py @@ -21,7 +21,7 @@ from geoprob_pipe.results import Results from geoprob_pipe.spatial import Spatial from geoprob_pipe.visualizations import Visualizations -from geoprob_pipe.calculations.systems.build_and_run import build_and_run_system_calculations +from geoprob_pipe.calculations.systems.build_and_run import BuildAndRunCalculations from geoprob_pipe.utils.update_metadata import update_metadata import logging if TYPE_CHECKING: @@ -57,7 +57,7 @@ def __init__(self, app_settings: ApplicationSettings) -> None: # self._read_calculation_settings() # TODO: Not part of new version # TODO: Unsure if the single statement belongs here. Wouldn't it be part of input data? - self.calc_results: List[CalcResult] = build_and_run_system_calculations(self) + self.calc_results: List[CalcResult] = BuildAndRunCalculations(self).run() self.results = Results(self) # Log finish diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 95abe9f1..85c68fce 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -19,15 +19,6 @@ "reuse_calculations": False, } -# DEFAULT_RELIABILITY_SETTINGS: Dict[str, Union[str, float, int]] = { -# "reliability_method": ReliabilityMethod.form.__str__(), -# "variation_coefficient": 0.02, -# "maximum_iterations": 10_000, -# "relaxation_factor": 0.4, -# "epsilon_beta": 0.05, -# "relaxation_loops": 10, -# } - class SystemSetup: diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 6691a69f..325ec027 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -19,7 +19,7 @@ from pandas import DataFrame import os if TYPE_CHECKING: - from geoprob_pipe import GeoProbPipe + from geoprob_pipe import GeoProbPipe, SystemCalculation from geoprob_pipe.calculations.systems.base_objects\ .base_system_build import BaseSystemBuilder from geoprob_pipe.utils.validation_messages import ValidationMessages @@ -70,13 +70,7 @@ def _init_worker( to_run_vakken_ids=to_run_vakken_ids)) -def _worker(row_unique: dict): - """ De worker functie die op de parallelle rekenkernen wordt gedraaid. - - :param row_unique: Identificatie naar unieke berekening, bijvoorbeeld - {'uittredepunt_id': 1, 'ondergrondscenario_naam': 'scenario1', 'vak_id': 4}. - :return: Tuple[Optional[CalcResult], Optional[str], Optional[dict]] - """ +def _logging_code(): log_buffer = StringIO() buffer_handler = logging.StreamHandler(log_buffer) buffer_handler.setLevel(logging.DEBUG) @@ -92,72 +86,96 @@ def _worker(row_unique: dict): root.addHandler(buffer_handler) logging.captureWarnings(True) + return log_buffer, buffer_handler, root, prev_level + + +def _run_calculation(row_unique: dict) -> SystemCalculation: + logger.debug("Start berekening voor %s", row_unique) + calc: SystemCalculation = _BUILDER.build_instance(row_unique=row_unique) + calc.run() + logger.debug("SystemCalculation voltooid.") # TODO: Skip if no issue? + + logger.debug("Validation messages:") + logger.debug(f"\n{calc.validation_messages.df}") # TODO: Only if there are any? + + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() + + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() + + return calc + + +def _collect_results(calc: SystemCalculation) -> CalcResult: + df_limit_state = collect_df_beta_limit_state(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") + if any(r == 0 for r in df_limit_state.total_model_runs): + logger.warning("Limit state with 0 total model runs encountered. " + "Notify developer and re-run calculations. ") + + df_scenario_rp = collect_df_beta_scenario_rp(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") + + df_scenario_cp = collect_df_beta_scenario_cp(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") + + df_scenario_final = collect_df_beta_scenario_final(calc) + + df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast.to_string()}") + + if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") + if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") + df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) + df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) + df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) + df_scenario_final = df_scenario_final.drop(columns=["system_calculation"]) + + return CalcResult( + df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, + df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, + validation_message=calc.validation_messages) + + +def _worker(row_unique: dict): + """ De worker functie die op de parallelle rekenkernen wordt gedraaid. + + :param row_unique: Identificatie naar unieke berekening, bijvoorbeeld + {'uittredepunt_id': 1, 'ondergrondscenario_naam': 'scenario1', 'vak_id': 4}. + :return: Tuple[Optional[CalcResult], Optional[str], Optional[dict]] + """ + log_buffer, buffer_handler, root, prev_level = _logging_code() + # noinspection PyBroadException try: with redirect_stdout(log_buffer), redirect_stderr(log_buffer): - logger = logging.getLogger(__name__) - logger.debug("Start berekening voor %s", row_unique) - - # Build and run calculations - calc = _BUILDER.build_instance(row_unique=row_unique) - calc.run() - - logger.debug("SystemCalculation voltooid.") - - logger.debug("Validation messages:") - logger.debug(f"\n{calc.validation_messages.df}") - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("Limit states print:") - for lm in calc.results.dps_limit_states: - lm.print() - - logger.debug("Combine project print:") - calc.results.combine_project.design_point.print() - - logger.debug("Reliability project print:") - calc.results.reliability_project.design_point.print() - - # Collect results - df_limit_state = collect_df_beta_limit_state(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") - if any(r == 0 for r in df_limit_state.total_model_runs): - logger.warning("Limit state with 0 total model runs encountered.") - df_scenario_rp = collect_df_beta_scenario_rp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") - df_scenario_cp = collect_df_beta_scenario_cp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") - df_scenario_final = collect_df_beta_scenario_final(calc) - df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") - if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): - logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") - if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): - logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") - df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) - df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) - df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) - df_scenario_final = df_scenario_final.drop(columns=["system_calculation"]) - - # Return results (without calculation object) - return CalcResult( - df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, - df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, - validation_message=calc.validation_messages - ), log_buffer.getvalue(), row_unique + # logger = logging.getLogger(__name__) + calc = _run_calculation(row_unique) + result = _collect_results(calc) + return result, log_buffer.getvalue(), row_unique except Exception: tb = traceback.format_exc() log_buffer.write(tb) buffer_handler.flush() return None, log_buffer.getvalue(), row_unique + finally: # Handler altijd verwijderen logging.captureWarnings(False) @@ -166,94 +184,117 @@ def _worker(row_unique: dict): buffer_handler.close() -def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcResult]: - """ In deze functie worden de parameters voor de berekeningen verzamelt, - aan de workers gegeven en vervolgens de resultaten verzameld. - """ - geohydrologisch_model = geoprob_pipe.input_data.geohydrologisch_model - geopackage_filepath = ( - geoprob_pipe.input_data.app_settings.geopackage_filepath) - to_run_vakken_ids = geoprob_pipe.input_data.app_settings.to_run_vakken_ids - system_builder: BaseSystemBuilder = ( - CALCULATION_MAPPER[geohydrologisch_model]['system_builder']( - geopackage_filepath=geopackage_filepath, - to_run_vakken_ids=to_run_vakken_ids)) +class BuildAndRunCalculations: + """ In dit object worden de parameters voor de berekeningen verzamelt, aan de workers gegeven en vervolgens de + resultaten verzameld. """ + + def __init__(self, geoprob_pipe: GeoProbPipe): + """ Init zet dit object op, maar het uitvoeren van de berekeningen gaat via de method .run(). """ + + self.geoprob_pipe: GeoProbPipe = geoprob_pipe + self.geohydrologisch_model: str = geoprob_pipe.input_data.geohydrologisch_model + self.geopackage_filepath: str = geoprob_pipe.input_data.app_settings.geopackage_filepath + self.to_run_vakken_ids: str = geoprob_pipe.input_data.app_settings.to_run_vakken_ids + + self._construct_run_settings() + self._setup_progress_variables() + + # Run logic with method .run() + + def _construct_run_settings(self): + """ Opzetten van de system builder en andere settings voor de berekeningen. """ + + logger.info("Now preparing for calculations...") + self.system_builder: BaseSystemBuilder = CALCULATION_MAPPER[self.geohydrologisch_model]['system_builder']( + geopackage_filepath=self.geopackage_filepath, + to_run_vakken_ids=self.to_run_vakken_ids) + self.df_unique_combos = self.system_builder.setup_iteration_df() + + # Bepaal de parameters voor de multiprocessing setup en de logger + self.n_threads: int = cpu_count() - 1 + self.n_calc_totaal: int = len(self.df_unique_combos) + + # Minimaal 5 berekeningen per chunk en grootte van chunk beperken zodat er gelogd kan worden. + self.chunk_size: int = max(math.ceil(self.n_calc_totaal / (self.n_threads * 10)), 5) + + def _setup_progress_variables(self): + """ Simpel initiëren van een aantal variabelen die nodig zijn tijdens de berekeningen. """ + + self.last_report: float = time.time() + self.done = 0 + self.log_errors = 0 + self.error_rows = [] + self.char_len_total: int = str(self.n_calc_totaal).__len__() + self.log_rows = [] + + def _report_calculation_progress_to_user(self): + """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ + + # Alleen kijken of er gelogd moet worden bij de laatste + # berekening die uit de chunk komt. + if self.done % self.chunk_size != 0: + return + + # Alleen loggen wanneer 30 seconden is gepasseerd + now = time.time() + if now - self.last_report < 30.0: + return + + # Log + error_count_append = "" + if self.log_errors > 0: + error_count_append = f" (of which {self.log_errors} failed calculations)" + logger.info( + f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations{error_count_append}.") + self.last_report = now + + def _push_errors_to_database(self): + """ Aan eind van run pushed deze method de errors (if any) naar de database. """ + + conn = sqlite3.connect(self.geopackage_filepath) + table_name = "calculation_logs" + df_logs = DataFrame(data=self.log_rows) + df_logs.to_sql(table_name, conn, if_exists="replace", index=False) + conn.commit() + conn.close() + if self.log_errors > 0: + logger.error(f"There are {self.log_errors} failed calculations. The calculation logs are stored inside the " + f"GeoPackage in table '{table_name}'. The following rows are marked:" + f"\n{self.error_rows}") + + def run(self) -> List[CalcResult]: + + logger.info( + f"Running {self.n_calc_totaal} calculations in chunks of {self.chunk_size}" + f" with {self.n_threads} parallel threads.") + logger.info( + f"Progress: {0:>{self.char_len_total}} / {self.n_calc_totaal} calculations.") + + rows = [dict(zip(self.df_unique_combos.columns, r)) + for r in self.df_unique_combos.itertuples(index=False, name=None)] + results: List[CalcResult] = [] + pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) + + # Multiprocessing setup + with Pool(processes=pool_size, initializer=_init_worker, initargs=( + self.geohydrologisch_model, self.geopackage_filepath, self.to_run_vakken_ids)) as pool: + + for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): + if isinstance(res, CalcResult): + results.append(res) + if isinstance(logs, str): + self.log_rows.append({ + "uittredepunt_id": row["uittredepunt_id"], + "ondergrondscenario_naam": row["ondergrondscenario_naam"], + "vak_id": row["vak_id"], + "logs": logs, + }) + if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): + self.log_errors += 1 + self.error_rows.append(row) + self.done += 1 + + self._report_calculation_progress_to_user() - logger.info("Now building and running calculations...") - df_unique_combos = system_builder.setup_iteration_df() - # Bepaal de parameters voor de multiprocessing setup en de logger - n_threads: int = cpu_count() - 1 - n_calc_totaal: int = len(df_unique_combos) - # Minimaal 5 berekeningen per chunk en grootte van chunk beperken - # zodat er gelogd kan worden. - chunk_size: int = max(math.ceil(n_calc_totaal / (n_threads * 10)), 5) - logger.info( - f"Running {n_calc_totaal} calculations in chunks of {chunk_size}" - f" with {n_threads} parallel threads.") - char_len_total = str(n_calc_totaal).__len__() - logger.info( - f"Progress: {0:>{char_len_total}} / {n_calc_totaal} calculations.") - - # Dicts zijn gemakkelijker te pickelen en daardoor sneller te - # verwerken dan pandas series. - rows = [dict(zip(df_unique_combos.columns, r)) - for r in df_unique_combos.itertuples(index=False, name=None)] - - last_report = time.time() - done = 0 - log_errors = 0 - error_rows = [] - results: List[CalcResult] = [] - pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) - - # Multiprocessing setup - log_rows = [] - with Pool(processes=pool_size, initializer=_init_worker, initargs=( - geohydrologisch_model, geopackage_filepath, to_run_vakken_ids)) as pool: - - for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): - if isinstance(res, CalcResult): - results.append(res) - if isinstance(logs, str): - log_rows.append({ - "uittredepunt_id": row["uittredepunt_id"], - "ondergrondscenario_naam": row["ondergrondscenario_naam"], - "vak_id": row["vak_id"], - "logs": logs, - }) - if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): - log_errors += 1 - error_rows.append(row) - done += 1 - - # Alleen kijken of er gelogd moet worden bij de laatste - # berekening die uit de chunk komt. - if done % chunk_size != 0: - continue - - # Alleen loggen wanneer 30 seconden is gepasseerd - now = time.time() - if now - last_report < 30.0: - continue - - # Log - error_count_append = "" - - if log_errors > 0: - error_count_append = f" (of which {log_errors} failed calculations)" - logger.info(f"Progress: {done:>{char_len_total}} / {n_calc_totaal} calculations{error_count_append}.") - last_report = now - - # Push errors to database (if any) - conn = sqlite3.connect(geoprob_pipe.input_data.app_settings.geopackage_filepath) - table_name = "calculation_logs" - df_logs = DataFrame(data=log_rows) - df_logs.to_sql(table_name, conn, if_exists="replace", index=False) - conn.commit() - conn.close() - if log_errors > 0: - logger.error(f"There are {log_errors} failed calculations. The calculation logs are stored inside the " - f"GeoPackage in table '{table_name}'. The following rows are marked:" - f"\n{error_rows}") - - return results + self._push_errors_to_database() + return results From 99c537bcca6af219dc091ea62155983c9905fee0 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 08:57:52 +0200 Subject: [PATCH 08/23] Small change --- .../calculations/systems/base_objects/system_calculation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 85c68fce..73618798 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,8 +13,6 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, - # "minimum_iterations": 5, # Don't allow zero iterations - # "minimum_directions": 2, # Prevent one direction solutions (single dominant alpha) "relaxation_factor": 0.4, "reuse_calculations": False, } From 541f0cc49b0d7ab29ca90fe5996c02fe9e79d597 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 12:24:39 +0200 Subject: [PATCH 09/23] Afgerond --- .../calculations/systems/build_and_run.py | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 325ec027..1a2146a6 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -90,15 +90,18 @@ def _logging_code(): def _run_calculation(row_unique: dict) -> SystemCalculation: - logger.debug("Start berekening voor %s", row_unique) + debug: bool = os.environ.get("GEOPROB_DEBUG", False) + if debug: logger.debug("Start berekening voor %s", row_unique) + + # Run calculation calc: SystemCalculation = _BUILDER.build_instance(row_unique=row_unique) calc.run() - logger.debug("SystemCalculation voltooid.") # TODO: Skip if no issue? + if debug: logger.debug("SystemCalculation voltooid.") - logger.debug("Validation messages:") - logger.debug(f"\n{calc.validation_messages.df}") # TODO: Only if there are any? + # Remainder is logging in case of validation messages or debug modus + if calc.validation_messages.df: logger.debug(f"Validation messages:\n{calc.validation_messages.df}") - if os.environ.get("GEOPROB_DEBUG", False): + if debug: logger.debug("Limit states print:") for lm in calc.results.dps_limit_states: lm.print() @@ -113,30 +116,22 @@ def _run_calculation(row_unique: dict) -> SystemCalculation: def _collect_results(calc: SystemCalculation) -> CalcResult: + debug: bool = os.environ.get("GEOPROB_DEBUG", False) df_limit_state = collect_df_beta_limit_state(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") + if debug: logger.debug(f"df_limit_state:\n{df_limit_state}") if any(r == 0 for r in df_limit_state.total_model_runs): - logger.warning("Limit state with 0 total model runs encountered. " - "Notify developer and re-run calculations. ") + logger.warning("Limit state with 0 total model runs encountered. Notify developer and re-run calculations. ") df_scenario_rp = collect_df_beta_scenario_rp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") + if debug: logger.debug(f"df_scenario_rp:\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") + if debug: logger.debug(f"df_scenario_cp:\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") + if debug: logger.debug(f"df_stochast:\n{df_stochast.to_string()}") if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") @@ -196,12 +191,12 @@ def __init__(self, geoprob_pipe: GeoProbPipe): self.geopackage_filepath: str = geoprob_pipe.input_data.app_settings.geopackage_filepath self.to_run_vakken_ids: str = geoprob_pipe.input_data.app_settings.to_run_vakken_ids - self._construct_run_settings() - self._setup_progress_variables() + self._construct_system_builder_and_settings() + self._setup_calculation_progress_variables() # Run logic with method .run() - def _construct_run_settings(self): + def _construct_system_builder_and_settings(self): """ Opzetten van de system builder en andere settings voor de berekeningen. """ logger.info("Now preparing for calculations...") @@ -217,7 +212,7 @@ def _construct_run_settings(self): # Minimaal 5 berekeningen per chunk en grootte van chunk beperken zodat er gelogd kan worden. self.chunk_size: int = max(math.ceil(self.n_calc_totaal / (self.n_threads * 10)), 5) - def _setup_progress_variables(self): + def _setup_calculation_progress_variables(self): """ Simpel initiëren van een aantal variabelen die nodig zijn tijdens de berekeningen. """ self.last_report: float = time.time() @@ -226,12 +221,20 @@ def _setup_progress_variables(self): self.error_rows = [] self.char_len_total: int = str(self.n_calc_totaal).__len__() self.log_rows = [] + self.results: List[CalcResult] = [] def _report_calculation_progress_to_user(self): """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ - # Alleen kijken of er gelogd moet worden bij de laatste - # berekening die uit de chunk komt. + # If finished + error_count_append = "" + if self.log_errors > 0: + error_count_append = f" (of which {self.log_errors} failed calculations)" + if self.n_calc_totaal == self.done: + logger.info(f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations" + f"{error_count_append}.") + + # Alleen kijken of er gelogd moet worden bij de laatste berekening die uit de chunk komt. if self.done % self.chunk_size != 0: return @@ -240,15 +243,11 @@ def _report_calculation_progress_to_user(self): if now - self.last_report < 30.0: return - # Log - error_count_append = "" - if self.log_errors > 0: - error_count_append = f" (of which {self.log_errors} failed calculations)" logger.info( f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations{error_count_append}.") self.last_report = now - def _push_errors_to_database(self): + def _push_resulting_error_messages_to_database(self): """ Aan eind van run pushed deze method de errors (if any) naar de database. """ conn = sqlite3.connect(self.geopackage_filepath) @@ -272,7 +271,6 @@ def run(self) -> List[CalcResult]: rows = [dict(zip(self.df_unique_combos.columns, r)) for r in self.df_unique_combos.itertuples(index=False, name=None)] - results: List[CalcResult] = [] pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) # Multiprocessing setup @@ -281,7 +279,7 @@ def run(self) -> List[CalcResult]: for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): if isinstance(res, CalcResult): - results.append(res) + self.results.append(res) if isinstance(logs, str): self.log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], @@ -296,5 +294,5 @@ def run(self) -> List[CalcResult]: self._report_calculation_progress_to_user() - self._push_errors_to_database() - return results + self._push_resulting_error_messages_to_database() + return self.results From 4f08f996d32fc78d6ba05c9fa3816b58cb22389b Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 12:30:33 +0200 Subject: [PATCH 10/23] Tijdelijke aanpassing teruggedraaid --- geoprob_pipe/calculations/systems/build_and_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 1a2146a6..e46354a5 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -221,7 +221,6 @@ def _setup_calculation_progress_variables(self): self.error_rows = [] self.char_len_total: int = str(self.n_calc_totaal).__len__() self.log_rows = [] - self.results: List[CalcResult] = [] def _report_calculation_progress_to_user(self): """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ @@ -274,12 +273,13 @@ def run(self) -> List[CalcResult]: pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) # Multiprocessing setup + results: List[CalcResult] = [] with Pool(processes=pool_size, initializer=_init_worker, initargs=( self.geohydrologisch_model, self.geopackage_filepath, self.to_run_vakken_ids)) as pool: for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): if isinstance(res, CalcResult): - self.results.append(res) + results.append(res) if isinstance(logs, str): self.log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], @@ -295,4 +295,4 @@ def run(self) -> List[CalcResult]: self._report_calculation_progress_to_user() self._push_resulting_error_messages_to_database() - return self.results + return results From 2bc33abbfc9800d60ef740172d9cee196fde92a5 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 16:27:35 +0200 Subject: [PATCH 11/23] First question setup --- geoprob_pipe/workflow/__init__.py | 2 + geoprob_pipe/workflow/base_objects.py | 58 +++++++++++++++++++ geoprob_pipe/workflow/cmd.py | 22 +++++++ geoprob_pipe/workflow/questions/__init__.py | 0 geoprob_pipe/workflow/questions/import_hrd.py | 35 +++++++++++ geoprob_pipe/workflow/state.py | 51 ++++++++++++++++ geoprob_pipe/workflow/workflow.py | 27 +++++++++ 7 files changed, 195 insertions(+) create mode 100644 geoprob_pipe/workflow/__init__.py create mode 100644 geoprob_pipe/workflow/base_objects.py create mode 100644 geoprob_pipe/workflow/cmd.py create mode 100644 geoprob_pipe/workflow/questions/__init__.py create mode 100644 geoprob_pipe/workflow/questions/import_hrd.py create mode 100644 geoprob_pipe/workflow/state.py create mode 100644 geoprob_pipe/workflow/workflow.py diff --git a/geoprob_pipe/workflow/__init__.py b/geoprob_pipe/workflow/__init__.py new file mode 100644 index 00000000..f163e28d --- /dev/null +++ b/geoprob_pipe/workflow/__init__.py @@ -0,0 +1,2 @@ +from geoprob_pipe.workflow.workflow import steps +from geoprob_pipe.workflow.state import State diff --git a/geoprob_pipe/workflow/base_objects.py b/geoprob_pipe/workflow/base_objects.py new file mode 100644 index 00000000..d3ccc6d2 --- /dev/null +++ b/geoprob_pipe/workflow/base_objects.py @@ -0,0 +1,58 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING +from dataclasses import dataclass +if TYPE_CHECKING: + from geoprob_pipe.workflow.state import State + + +class Step(ABC): + + def __init__(self, state: State): + self.state = state + + @property + @abstractmethod + def completed(self) -> bool: + """ Return True if the is already completed. Otherwise, step must be run. """ + raise NotImplementedError() + + @abstractmethod + def execute(self): + raise NotImplementedError() + + +@dataclass +class ValidationResult: + is_valid: bool + message: str | None = None + + +class Question(Step): + question_label: str = None + + @abstractmethod + def ask(self): + raise NotImplementedError() + + def execute(self): + answer = self.ask_until_valid() + self.state.store_question_answer(question_label=self.question_label, answer=answer) + + @abstractmethod + def validate(self, answer) -> ValidationResult: + raise NotImplementedError() + + def ask_until_valid(self): + while True: + answer = self.ask() + result = self.validate(answer) + if result.is_valid: + return answer + + +class Action(Step): + + @abstractmethod + def execute(self): + raise NotImplementedError() diff --git a/geoprob_pipe/workflow/cmd.py b/geoprob_pipe/workflow/cmd.py new file mode 100644 index 00000000..5ade8575 --- /dev/null +++ b/geoprob_pipe/workflow/cmd.py @@ -0,0 +1,22 @@ +from geoprob_pipe.workflow import steps, State +import typer + + +state = State(geoprob_pipe_file_path=r"C:\Users\CP\Downloads\false_fix\tmp.gpkg") + +app = typer.Typer(help="GeoProb-Pipe - CLI applicatie voor probabilistische piping berekeningen.", add_completion=False) + + +@app.callback(invoke_without_command=True) +def main(ctx: typer.Context): + """ Default entry point for `geoprob-pipe`. Runs when no subcommand is specified. """ + if ctx.invoked_subcommand is None: + for obj in steps: + step = obj(state=state) + print(f"{step.completed=}") + if not step.completed: + step.execute() + + +if __name__ == "__main__": + app() diff --git a/geoprob_pipe/workflow/questions/__init__.py b/geoprob_pipe/workflow/questions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/geoprob_pipe/workflow/questions/import_hrd.py b/geoprob_pipe/workflow/questions/import_hrd.py new file mode 100644 index 00000000..a9fbf66b --- /dev/null +++ b/geoprob_pipe/workflow/questions/import_hrd.py @@ -0,0 +1,35 @@ +from geoprob_pipe.workflow.base_objects import Question, ValidationResult +from typing import Optional +from InquirerPy import inquirer + + +CHOICES = [ + "Hydra-NL database", + "Ander GeoProb-Pipe bestand", + "Nee", +] + + +class QuestionImportHRD(Question): + question_label = "import_hrd" + + def ask(self) -> str: + return inquirer.select( + message="Wil je overschrijdingsfrequentielijnen importeren? En zo ja, uit welke bron? " + "Je kunt op een later moment handmatig overschrijdingsfrequentielijnen toevoegen aan het " + "bestand met invoer-Excel.", + choices=CHOICES, + default=CHOICES[0], + ).execute() + + def validate(self, answer): + if answer not in CHOICES: + return ValidationResult(False, f"Kies één van de volgende opties: {CHOICES}.") + return ValidationResult(True) + + @property + def completed(self) -> bool: + import_hrd: Optional[str] = self.state.retrieve_question_answer(self.question_label) + if import_hrd is None or import_hrd == CHOICES[2]: + return False + return True diff --git a/geoprob_pipe/workflow/state.py b/geoprob_pipe/workflow/state.py new file mode 100644 index 00000000..6edb7a82 --- /dev/null +++ b/geoprob_pipe/workflow/state.py @@ -0,0 +1,51 @@ +from typing import Optional +import sqlite3 + + +class State: + + def __init__(self, geoprob_pipe_file_path: str): + self.geoprob_pipe_file_path: str = geoprob_pipe_file_path + + def retrieve_question_answer(self, question: str) -> Optional[str]: + """ The terminal user interface has a workflow of questions that the users answers. This + method retrieves the answer to a question (if already stored). """ + conn = sqlite3.connect(self.geoprob_pipe_file_path) + cursor = conn.cursor() + try: + cursor.execute(f""" + SELECT answer + FROM workflow_questions + WHERE question_label = '{question}' + LIMIT 1; + """) + except sqlite3.OperationalError: # table does not exist + return None + result = cursor.fetchone() + if not result: + return None + return result[0] + + def store_question_answer(self, question_label: str, answer: str): + conn = sqlite3.connect(self.geoprob_pipe_file_path) + cursor = conn.cursor() + + cursor.execute(""" + CREATE TABLE IF NOT EXISTS workflow_questions ( + question_label TEXT PRIMARY KEY, + answer TEXT + ) + """) + + cursor.execute(f""" + INSERT INTO workflow_questions ( + question_label, + answer + ) + VALUES (?, ?) + ON CONFLICT(question_label) + DO UPDATE SET answer = excluded.answer + """, (question_label, answer)) + + conn.commit() + conn.close() diff --git a/geoprob_pipe/workflow/workflow.py b/geoprob_pipe/workflow/workflow.py new file mode 100644 index 00000000..c68668b5 --- /dev/null +++ b/geoprob_pipe/workflow/workflow.py @@ -0,0 +1,27 @@ +from geoprob_pipe.workflow.base_objects import Step +from typing import List, Type +from geoprob_pipe.workflow.questions.import_hrd import QuestionImportHRD + + +steps: List[Type[Step]] = [ + + # HRD + QuestionImportHRD, + # QuestionDirectoryPathHydraNLDatabase, + # ActionImportHRDFromHydraNLDatabase, + # ActionImportTrajectParametersFromHydraNLDatabase, + # QuestionFilePathGeoProbPipeFileWithHRD, + # ActionImportHRDFromOtherGeoProbPipeFile, + # ActionImportTrajectParametersFromOtherGeoProbPipeFile, + + # Traject parameters + # QuestionTrajectID, + # QuestionSignaleringswaarde, + # QuestionOndergrens, + # QuestionW, + # QuestionIsBovenrivierengebied, + + # Uittredepunten + # QuestionPathToUittredepuntenGISFile, + # ActionImportUittredepuntenGISFile, +] From 49350ec8e3e499373218f4ef1278ef5dd18ef31c Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 16:37:07 +0200 Subject: [PATCH 12/23] First question setup --- geoprob_pipe/workflow/questions/dir_hydranl_db.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 geoprob_pipe/workflow/questions/dir_hydranl_db.py diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py new file mode 100644 index 00000000..e69de29b From 251f5e5876427aeebdbf781d8e5ab710418d1773 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 17:51:09 +0200 Subject: [PATCH 13/23] Workflow with two questions --- geoprob_pipe/workflow/base_objects.py | 27 ++++++++++---- geoprob_pipe/workflow/cmd.py | 4 +-- geoprob_pipe/workflow/questions/__init__.py | 2 ++ .../workflow/questions/dir_hydranl_db.py | 36 +++++++++++++++++++ geoprob_pipe/workflow/questions/import_hrd.py | 10 ++++-- geoprob_pipe/workflow/workflow.py | 4 +-- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/geoprob_pipe/workflow/base_objects.py b/geoprob_pipe/workflow/base_objects.py index d3ccc6d2..a47f48f4 100644 --- a/geoprob_pipe/workflow/base_objects.py +++ b/geoprob_pipe/workflow/base_objects.py @@ -1,20 +1,28 @@ from __future__ import annotations from abc import ABC, abstractmethod from typing import TYPE_CHECKING +from geoprob_pipe.utils.validation_messages import BColors from dataclasses import dataclass if TYPE_CHECKING: from geoprob_pipe.workflow.state import State class Step(ABC): + label: str = None def __init__(self, state: State): self.state = state + @property + @abstractmethod + def should_run(self) -> bool: + """ Return True if this step should be run, based on the state of the project. """ + raise NotImplementedError() + @property @abstractmethod def completed(self) -> bool: - """ Return True if the is already completed. Otherwise, step must be run. """ + """ Return True if this step is already completed. Otherwise, step must be run. """ raise NotImplementedError() @abstractmethod @@ -26,10 +34,10 @@ def execute(self): class ValidationResult: is_valid: bool message: str | None = None + manipulated_answer: str | None = None class Question(Step): - question_label: str = None @abstractmethod def ask(self): @@ -37,7 +45,7 @@ def ask(self): def execute(self): answer = self.ask_until_valid() - self.state.store_question_answer(question_label=self.question_label, answer=answer) + self.state.store_question_answer(question_label=self.label, answer=answer) @abstractmethod def validate(self, answer) -> ValidationResult: @@ -46,9 +54,16 @@ def validate(self, answer) -> ValidationResult: def ask_until_valid(self): while True: answer = self.ask() - result = self.validate(answer) - if result.is_valid: - return answer + + # Validate + result: ValidationResult = self.validate(answer) + if not result.is_valid: + print(f"{BColors.WARNING}{result.message}{BColors.ENDC}") + + # Is valid + if result.manipulated_answer: + return result.manipulated_answer + return answer class Action(Step): diff --git a/geoprob_pipe/workflow/cmd.py b/geoprob_pipe/workflow/cmd.py index 5ade8575..09768caa 100644 --- a/geoprob_pipe/workflow/cmd.py +++ b/geoprob_pipe/workflow/cmd.py @@ -13,8 +13,8 @@ def main(ctx: typer.Context): if ctx.invoked_subcommand is None: for obj in steps: step = obj(state=state) - print(f"{step.completed=}") - if not step.completed: + print(f"{step.label}: {step.should_run=} {step.completed=}") + if step.should_run and not step.completed: step.execute() diff --git a/geoprob_pipe/workflow/questions/__init__.py b/geoprob_pipe/workflow/questions/__init__.py index e69de29b..cde1384a 100644 --- a/geoprob_pipe/workflow/questions/__init__.py +++ b/geoprob_pipe/workflow/questions/__init__.py @@ -0,0 +1,2 @@ +from geoprob_pipe.workflow.questions.import_hrd import QuestionImportHRD +from geoprob_pipe.workflow.questions.dir_hydranl_db import QuestionDirHydraNLDatabase diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py index e69de29b..2a6e3d72 100644 --- a/geoprob_pipe/workflow/questions/dir_hydranl_db.py +++ b/geoprob_pipe/workflow/questions/dir_hydranl_db.py @@ -0,0 +1,36 @@ +from geoprob_pipe.workflow.base_objects import Question, ValidationResult +from typing import Optional +from InquirerPy import inquirer +import os + + +class QuestionDirHydraNLDatabase(Question): + label = "dir_hydranl_db" + + def ask(self) -> str: + return inquirer.text( + message="Specificeer het volledige pad naar de bestandsmap met de Hydra-NL database. " + "Dat zijn de hlcd, config en het database .sqlite-bestand zelf.", + ).execute() + + def validate(self, answer) -> ValidationResult: + manipulated_answer = answer.replace('"', '') + if not os.path.isdir(manipulated_answer): + return ValidationResult(False, "The provided path is not a directory.") + # TODO: Validate contains correct files in dir + # TODO: Validate HRD-point locations are valid + return ValidationResult(True, manipulated_answer=manipulated_answer) + + @property + def should_run(self) -> bool: + answer: Optional[str] = self.state.retrieve_question_answer("import_hrd") + if answer != "Hydra-NL database": + return False + return True + + @property + def completed(self) -> bool: + answer: Optional[str] = self.state.retrieve_question_answer(self.label) + if answer is None: + return False + return True diff --git a/geoprob_pipe/workflow/questions/import_hrd.py b/geoprob_pipe/workflow/questions/import_hrd.py index a9fbf66b..4c781c28 100644 --- a/geoprob_pipe/workflow/questions/import_hrd.py +++ b/geoprob_pipe/workflow/questions/import_hrd.py @@ -11,7 +11,7 @@ class QuestionImportHRD(Question): - question_label = "import_hrd" + label = "import_hrd" def ask(self) -> str: return inquirer.select( @@ -27,9 +27,13 @@ def validate(self, answer): return ValidationResult(False, f"Kies één van de volgende opties: {CHOICES}.") return ValidationResult(True) + @property + def should_run(self) -> bool: + return True # Always ask if HRD-data should be imported. + @property def completed(self) -> bool: - import_hrd: Optional[str] = self.state.retrieve_question_answer(self.question_label) - if import_hrd is None or import_hrd == CHOICES[2]: + answer: Optional[str] = self.state.retrieve_question_answer(self.label) + if answer is None or answer == CHOICES[2]: return False return True diff --git a/geoprob_pipe/workflow/workflow.py b/geoprob_pipe/workflow/workflow.py index c68668b5..1a4d9b0d 100644 --- a/geoprob_pipe/workflow/workflow.py +++ b/geoprob_pipe/workflow/workflow.py @@ -1,13 +1,13 @@ from geoprob_pipe.workflow.base_objects import Step from typing import List, Type -from geoprob_pipe.workflow.questions.import_hrd import QuestionImportHRD +from geoprob_pipe.workflow.questions import QuestionImportHRD, QuestionDirHydraNLDatabase steps: List[Type[Step]] = [ # HRD QuestionImportHRD, - # QuestionDirectoryPathHydraNLDatabase, + QuestionDirHydraNLDatabase, # ActionImportHRDFromHydraNLDatabase, # ActionImportTrajectParametersFromHydraNLDatabase, # QuestionFilePathGeoProbPipeFileWithHRD, From 1a486707fb77883299eae4b5b1df000967025a25 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 18:16:15 +0200 Subject: [PATCH 14/23] Added test --- geoprob_pipe/workflow/base_objects.py | 3 +- .../workflow/questions/dir_hydranl_db.py | 41 ++++++++++++++++++- geoprob_pipe/workflow/questions/import_hrd.py | 3 +- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/geoprob_pipe/workflow/base_objects.py b/geoprob_pipe/workflow/base_objects.py index a47f48f4..9ddc00c7 100644 --- a/geoprob_pipe/workflow/base_objects.py +++ b/geoprob_pipe/workflow/base_objects.py @@ -47,8 +47,9 @@ def execute(self): answer = self.ask_until_valid() self.state.store_question_answer(question_label=self.label, answer=answer) + @staticmethod @abstractmethod - def validate(self, answer) -> ValidationResult: + def validate(answer) -> ValidationResult: raise NotImplementedError() def ask_until_valid(self): diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py index 2a6e3d72..39cfd328 100644 --- a/geoprob_pipe/workflow/questions/dir_hydranl_db.py +++ b/geoprob_pipe/workflow/questions/dir_hydranl_db.py @@ -4,6 +4,25 @@ import os +def _folder_contains_hrd_db(hrd_dir: str) -> bool: + cnt_sql_files = 0 + cnt_config_files = 0 + cnt_hlcd_files = 0 + + + for file in os.listdir(hrd_dir): + filename = os.fsdecode(file) + if filename.endswith(".sqlite"): + cnt_sql_files += 1 + if filename.endswith(".config.sqlite"): + cnt_config_files += 1 + if filename.endswith("hlcd.sqlite"): + cnt_hlcd_files += 1 + + if cnt_sql_files == 3 and cnt_config_files == 1 and cnt_hlcd_files == 1: + return True + return False + class QuestionDirHydraNLDatabase(Question): label = "dir_hydranl_db" @@ -13,11 +32,14 @@ def ask(self) -> str: "Dat zijn de hlcd, config en het database .sqlite-bestand zelf.", ).execute() - def validate(self, answer) -> ValidationResult: + @staticmethod + def validate(answer) -> ValidationResult: manipulated_answer = answer.replace('"', '') if not os.path.isdir(manipulated_answer): return ValidationResult(False, "The provided path is not a directory.") - # TODO: Validate contains correct files in dir + if not _folder_contains_hrd_db(hrd_dir=manipulated_answer): + return ValidationResult( + False, "The provided directory does not contain the necessary Hydra-NL database-files.") # TODO: Validate HRD-point locations are valid return ValidationResult(True, manipulated_answer=manipulated_answer) @@ -34,3 +56,18 @@ def completed(self) -> bool: if answer is None: return False return True + + +def test_question_dir_hydra_nl_database(): + test_answers = [ + r"C:\Users\CP\Downloads\false_fix\Analyse16-2_V3.2.geoprob_pipe.gpkg", + r"C:\Users\CP\Downloads\false_fix\exports", + r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04", + ] + test_results = [ + False, + False, + True, + ] + for test_answer, test_result in zip(test_answers, test_results): + assert QuestionDirHydraNLDatabase.validate(answer=test_answer).is_valid == test_result diff --git a/geoprob_pipe/workflow/questions/import_hrd.py b/geoprob_pipe/workflow/questions/import_hrd.py index 4c781c28..9e3105fc 100644 --- a/geoprob_pipe/workflow/questions/import_hrd.py +++ b/geoprob_pipe/workflow/questions/import_hrd.py @@ -22,7 +22,8 @@ def ask(self) -> str: default=CHOICES[0], ).execute() - def validate(self, answer): + @staticmethod + def validate(answer): if answer not in CHOICES: return ValidationResult(False, f"Kies één van de volgende opties: {CHOICES}.") return ValidationResult(True) From 8471a4844207448a67363fad0855be2cc031819a Mon Sep 17 00:00:00 2001 From: chrispijo Date: Sat, 22 Aug 2026 15:30:58 +0200 Subject: [PATCH 15/23] Added first action --- geoprob_pipe/workflow/actions/__init__.py | 1 + .../actions/import_hrd_from_hydranl_db.py | 32 +++++ geoprob_pipe/workflow/base_objects.py | 14 ++- geoprob_pipe/workflow/cmd.py | 2 +- .../workflow/questions/dir_hydranl_db.py | 5 +- geoprob_pipe/workflow/questions/import_hrd.py | 2 +- geoprob_pipe/workflow/state.py | 119 ++++++++++++++---- geoprob_pipe/workflow/workflow.py | 4 +- 8 files changed, 142 insertions(+), 37 deletions(-) create mode 100644 geoprob_pipe/workflow/actions/__init__.py create mode 100644 geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py diff --git a/geoprob_pipe/workflow/actions/__init__.py b/geoprob_pipe/workflow/actions/__init__.py new file mode 100644 index 00000000..3e2a1a92 --- /dev/null +++ b/geoprob_pipe/workflow/actions/__init__.py @@ -0,0 +1 @@ +from geoprob_pipe.workflow.actions.import_hrd_from_hydranl_db import ActionImportHRDLocationsFromHydraNLDatabase \ No newline at end of file diff --git a/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py b/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py new file mode 100644 index 00000000..090fd89f --- /dev/null +++ b/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py @@ -0,0 +1,32 @@ +from geoprob_pipe.workflow.base_objects import Action +from geoprob_pipe.cmd_app.spatial_layers.hrd.import_from_hrd import hrd_file_path +import sqlite3 +from pandas import read_sql +from geopandas import GeoDataFrame, points_from_xy + + +class ActionImportHRDLocationsFromHydraNLDatabase(Action): + + def execute(self): + dir_to_db = self.state.question_answer.retrieve(question="dir_hydranl_db") + path_to_db = hrd_file_path(hrd_dir=dir_to_db) + conn = sqlite3.connect(path_to_db) + df = read_sql("SELECT Name, XCoordinate, YCoordinate FROM HRDLocations", conn) + gdf = GeoDataFrame(df, geometry=points_from_xy(df["XCoordinate"], df["YCoordinate"]), crs="EPSG:28992") + conn.close() + gdf = gdf.drop(columns=["XCoordinate", "YCoordinate"]) + gdf = gdf.rename(columns={"Name": "location_name"}) + self.state.gdf.store(gdf=gdf, layer_name="hrd_locations") + + @property + def should_run(self) -> bool: + if not (self.state.question_answer.retrieve("import_hrd") == "Hydra-NL database" and + self.state.question_answer.retrieve("dir_hydranl_db") is not None): + return False + return True + + @property + def completed(self) -> bool: + if self.state.gdf.hrd_locations is None: + return False + return True diff --git a/geoprob_pipe/workflow/base_objects.py b/geoprob_pipe/workflow/base_objects.py index 9ddc00c7..3596716c 100644 --- a/geoprob_pipe/workflow/base_objects.py +++ b/geoprob_pipe/workflow/base_objects.py @@ -45,7 +45,7 @@ def ask(self): def execute(self): answer = self.ask_until_valid() - self.state.store_question_answer(question_label=self.label, answer=answer) + self.state.question_answer.store(question_label=self.label, answer=answer) @staticmethod @abstractmethod @@ -58,13 +58,15 @@ def ask_until_valid(self): # Validate result: ValidationResult = self.validate(answer) - if not result.is_valid: - print(f"{BColors.WARNING}{result.message}{BColors.ENDC}") # Is valid - if result.manipulated_answer: - return result.manipulated_answer - return answer + if result.is_valid: + if result.manipulated_answer: + return result.manipulated_answer + return answer + + # Is not valid + print(f"{BColors.WARNING}{result.message}{BColors.ENDC}") class Action(Step): diff --git a/geoprob_pipe/workflow/cmd.py b/geoprob_pipe/workflow/cmd.py index 09768caa..621ac20d 100644 --- a/geoprob_pipe/workflow/cmd.py +++ b/geoprob_pipe/workflow/cmd.py @@ -2,7 +2,7 @@ import typer -state = State(geoprob_pipe_file_path=r"C:\Users\CP\Downloads\false_fix\tmp.gpkg") +state = State(file_path=r"C:\Users\CP\Downloads\false_fix\tmp.gpkg") app = typer.Typer(help="GeoProb-Pipe - CLI applicatie voor probabilistische piping berekeningen.", add_completion=False) diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py index 39cfd328..e0879c6d 100644 --- a/geoprob_pipe/workflow/questions/dir_hydranl_db.py +++ b/geoprob_pipe/workflow/questions/dir_hydranl_db.py @@ -9,7 +9,6 @@ def _folder_contains_hrd_db(hrd_dir: str) -> bool: cnt_config_files = 0 cnt_hlcd_files = 0 - for file in os.listdir(hrd_dir): filename = os.fsdecode(file) if filename.endswith(".sqlite"): @@ -45,14 +44,14 @@ def validate(answer) -> ValidationResult: @property def should_run(self) -> bool: - answer: Optional[str] = self.state.retrieve_question_answer("import_hrd") + answer: Optional[str] = self.state.question_answer.retrieve("import_hrd") if answer != "Hydra-NL database": return False return True @property def completed(self) -> bool: - answer: Optional[str] = self.state.retrieve_question_answer(self.label) + answer: Optional[str] = self.state.question_answer.retrieve(self.label) if answer is None: return False return True diff --git a/geoprob_pipe/workflow/questions/import_hrd.py b/geoprob_pipe/workflow/questions/import_hrd.py index 9e3105fc..a7b4db5f 100644 --- a/geoprob_pipe/workflow/questions/import_hrd.py +++ b/geoprob_pipe/workflow/questions/import_hrd.py @@ -34,7 +34,7 @@ def should_run(self) -> bool: @property def completed(self) -> bool: - answer: Optional[str] = self.state.retrieve_question_answer(self.label) + answer: Optional[str] = self.state.question_answer.retrieve(self.label) if answer is None or answer == CHOICES[2]: return False return True diff --git a/geoprob_pipe/workflow/state.py b/geoprob_pipe/workflow/state.py index 6edb7a82..ffb05a6b 100644 --- a/geoprob_pipe/workflow/state.py +++ b/geoprob_pipe/workflow/state.py @@ -1,33 +1,25 @@ -from typing import Optional +import os +from typing import Optional, Dict import sqlite3 +from geopandas import GeoDataFrame, read_file +from pathlib import Path +from uuid import uuid4 +from geoprob_pipe.utils.validation_messages import BColors +import fiona -class State: +class QuestionAnswer: + """ The terminal user interface has a workflow of questions that the users answers. Based on this, and imported data + GeoProb-Pipe determines the state of the application. The answers to these questions (this part of the state) are + stored in the geopackage in table 'workflow_questions'. """ - def __init__(self, geoprob_pipe_file_path: str): - self.geoprob_pipe_file_path: str = geoprob_pipe_file_path + def __init__(self, file_path: str): + self.file_path: str = file_path - def retrieve_question_answer(self, question: str) -> Optional[str]: - """ The terminal user interface has a workflow of questions that the users answers. This - method retrieves the answer to a question (if already stored). """ - conn = sqlite3.connect(self.geoprob_pipe_file_path) - cursor = conn.cursor() - try: - cursor.execute(f""" - SELECT answer - FROM workflow_questions - WHERE question_label = '{question}' - LIMIT 1; - """) - except sqlite3.OperationalError: # table does not exist - return None - result = cursor.fetchone() - if not result: - return None - return result[0] + def store(self, question_label: str, answer: str): + """ This method stores the answer to a question. """ - def store_question_answer(self, question_label: str, answer: str): - conn = sqlite3.connect(self.geoprob_pipe_file_path) + conn = sqlite3.connect(self.file_path) cursor = conn.cursor() cursor.execute(""" @@ -49,3 +41,82 @@ def store_question_answer(self, question_label: str, answer: str): conn.commit() conn.close() + + def retrieve(self, question: str) -> Optional[str]: + """ This method retrieves the answer to a question (if already stored). """ + conn = sqlite3.connect(self.file_path) + cursor = conn.cursor() + try: + cursor.execute(f""" + SELECT answer + FROM workflow_questions + WHERE question_label = '{question}' + LIMIT 1; + """) + except sqlite3.OperationalError: # table does not exist + return None + result = cursor.fetchone() + if not result: + return None + return result[0] + + +class GeoDataFrames: + + def __init__(self, file_path: Optional[str] = None): + self.file_path: str = file_path + self._read_geodataframes: Dict[str, GeoDataFrame] = {} + + def store(self, gdf: GeoDataFrame, layer_name: str): + gdf.to_file(Path(self.file_path), layer=layer_name, driver="GPKG") + print(f"{BColors.OKBLUE}" + f"✅ Geografische tabel '{layer_name}' toegevoegd aan het GeoProb-Pipe GeoPackage.{BColors.ENDC}") + + def retrieve(self, layer_name: str) -> Optional[GeoDataFrame]: + # First check if already retrieved before + if layer_name in self._read_geodataframes.keys(): + return self._read_geodataframes[layer_name] + + # Otherwise retrieve from GeoProb-Pipe-file + layers = fiona.listlayers(self.file_path) + if layer_name not in layers: + return None + return read_file(self.file_path, layer=layer_name) + + @property + def hrd_locations(self) -> Optional[GeoDataFrame]: + return self.retrieve(layer_name="hrd_locations") + + +class DataFrames: + + def __init__(self): + pass + + +def _if_needed_create_dummy_gpkg(file_path: Optional[str] = None, file_dir: Optional[str] = None) -> str: + if (file_path is None and file_dir is None) or (file_path is not None and file_dir is not None): + raise ValueError(f"Specify either the path to the GeoProb-Pipe-file, or a directory where a dummy can " + f"can be created.") + elif file_dir is not None: + import geopandas as gpd + gdf = gpd.GeoDataFrame(geometry=[]) + path_to_gpkg = os.path.join(file_dir, f"dummy_{uuid4().__str__()}.geoprob_pipe.gpkg") + gdf.to_file(Path(path_to_gpkg, driver="GPKG")) + file_path = path_to_gpkg + return file_path + + +class State: + """ Through the terminal user interface the user selects choices/preferences (questions and answers) and imports + data. The state of the application is the state of these answers and imported data. """ + + def __init__(self, file_path: Optional[str] = None, file_dir: Optional[str] = None): + """ + + :param file_path: Path to the GeoProb-Pipe file. + """ + self.file_path: str = _if_needed_create_dummy_gpkg(file_path, file_dir) + self.question_answer = QuestionAnswer(self.file_path) + self.gdf = GeoDataFrames(self.file_path) + self.df = DataFrames() diff --git a/geoprob_pipe/workflow/workflow.py b/geoprob_pipe/workflow/workflow.py index 1a4d9b0d..d6daee35 100644 --- a/geoprob_pipe/workflow/workflow.py +++ b/geoprob_pipe/workflow/workflow.py @@ -1,14 +1,14 @@ from geoprob_pipe.workflow.base_objects import Step from typing import List, Type from geoprob_pipe.workflow.questions import QuestionImportHRD, QuestionDirHydraNLDatabase - +from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase steps: List[Type[Step]] = [ # HRD QuestionImportHRD, QuestionDirHydraNLDatabase, - # ActionImportHRDFromHydraNLDatabase, + ActionImportHRDLocationsFromHydraNLDatabase, # ActionImportTrajectParametersFromHydraNLDatabase, # QuestionFilePathGeoProbPipeFileWithHRD, # ActionImportHRDFromOtherGeoProbPipeFile, From 2bd8ada5ed788f69cdf7f3a23c6b80bf642392d3 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Sat, 22 Aug 2026 20:32:15 +0200 Subject: [PATCH 16/23] Unit test Action done --- geoprob_pipe/workflow/state.py | 2 +- geoprob_pipe/workflow/tests/__init__.py | 0 geoprob_pipe/workflow/tests/actions/__init__.py | 0 .../tests/actions/import_hrd_from_hydranl_db.py | 13 +++++++++++++ 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 geoprob_pipe/workflow/tests/__init__.py create mode 100644 geoprob_pipe/workflow/tests/actions/__init__.py create mode 100644 geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py diff --git a/geoprob_pipe/workflow/state.py b/geoprob_pipe/workflow/state.py index ffb05a6b..34346307 100644 --- a/geoprob_pipe/workflow/state.py +++ b/geoprob_pipe/workflow/state.py @@ -111,7 +111,7 @@ class State: """ Through the terminal user interface the user selects choices/preferences (questions and answers) and imports data. The state of the application is the state of these answers and imported data. """ - def __init__(self, file_path: Optional[str] = None, file_dir: Optional[str] = None): + def __init__(self, file_path: Optional[str] = None, file_dir: Optional[str | Path] = None): """ :param file_path: Path to the GeoProb-Pipe file. diff --git a/geoprob_pipe/workflow/tests/__init__.py b/geoprob_pipe/workflow/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/geoprob_pipe/workflow/tests/actions/__init__.py b/geoprob_pipe/workflow/tests/actions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py b/geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py new file mode 100644 index 00000000..dd77f470 --- /dev/null +++ b/geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py @@ -0,0 +1,13 @@ +from geoprob_pipe.workflow import State +from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase + + +def test_action_import_hrd_locations_from_hydranl_db(tmp_path): + state = State(file_dir=tmp_path) + state.question_answer.store( + question_label="dir_hydranl_db", + answer=r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04") + action = ActionImportHRDLocationsFromHydraNLDatabase(state=state) + action.execute() + assert state.gdf.hrd_locations is not None + assert state.gdf.hrd_locations.__len__() == 400 From 1f195090d0ab5a18ed4a78a0333837ffdf9a8a08 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Sat, 22 Aug 2026 20:39:48 +0200 Subject: [PATCH 17/23] Unit test Action done --- .../workflow/questions/dir_hydranl_db.py | 15 --------------- .../tests => tests/workflow}/__init__.py | 0 .../tests => tests/workflow}/actions/__init__.py | 0 .../actions/test_import_hrd_from_hydranl_db.py | 0 tests/workflow/questions/__init__.py | 0 tests/workflow/questions/test_dir_hydranl_db.py | 16 ++++++++++++++++ 6 files changed, 16 insertions(+), 15 deletions(-) rename {geoprob_pipe/workflow/tests => tests/workflow}/__init__.py (100%) rename {geoprob_pipe/workflow/tests => tests/workflow}/actions/__init__.py (100%) rename geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py => tests/workflow/actions/test_import_hrd_from_hydranl_db.py (100%) create mode 100644 tests/workflow/questions/__init__.py create mode 100644 tests/workflow/questions/test_dir_hydranl_db.py diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py index e0879c6d..471e36a2 100644 --- a/geoprob_pipe/workflow/questions/dir_hydranl_db.py +++ b/geoprob_pipe/workflow/questions/dir_hydranl_db.py @@ -55,18 +55,3 @@ def completed(self) -> bool: if answer is None: return False return True - - -def test_question_dir_hydra_nl_database(): - test_answers = [ - r"C:\Users\CP\Downloads\false_fix\Analyse16-2_V3.2.geoprob_pipe.gpkg", - r"C:\Users\CP\Downloads\false_fix\exports", - r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04", - ] - test_results = [ - False, - False, - True, - ] - for test_answer, test_result in zip(test_answers, test_results): - assert QuestionDirHydraNLDatabase.validate(answer=test_answer).is_valid == test_result diff --git a/geoprob_pipe/workflow/tests/__init__.py b/tests/workflow/__init__.py similarity index 100% rename from geoprob_pipe/workflow/tests/__init__.py rename to tests/workflow/__init__.py diff --git a/geoprob_pipe/workflow/tests/actions/__init__.py b/tests/workflow/actions/__init__.py similarity index 100% rename from geoprob_pipe/workflow/tests/actions/__init__.py rename to tests/workflow/actions/__init__.py diff --git a/geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py similarity index 100% rename from geoprob_pipe/workflow/tests/actions/import_hrd_from_hydranl_db.py rename to tests/workflow/actions/test_import_hrd_from_hydranl_db.py diff --git a/tests/workflow/questions/__init__.py b/tests/workflow/questions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/workflow/questions/test_dir_hydranl_db.py b/tests/workflow/questions/test_dir_hydranl_db.py new file mode 100644 index 00000000..55d48a7b --- /dev/null +++ b/tests/workflow/questions/test_dir_hydranl_db.py @@ -0,0 +1,16 @@ +from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase + + +def test_question_dir_hydra_nl_database(): + test_answers = [ + r"C:\Users\CP\Downloads\false_fix\Analyse16-2_V3.2.geoprob_pipe.gpkg", + r"C:\Users\CP\Downloads\false_fix\exports", + r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04", + ] + test_results = [ + False, + False, + True, + ] + for test_answer, test_result in zip(test_answers, test_results): + assert QuestionDirHydraNLDatabase.validate(answer=test_answer).is_valid == test_result From 1c3e3d77be25b8e28aa5c1563080f0d62c0e67ab Mon Sep 17 00:00:00 2001 From: chrispijo Date: Sat, 22 Aug 2026 21:12:55 +0200 Subject: [PATCH 18/23] Small changes --- geoprob_pipe/workflow/__init__.py | 2 +- .../workflow/actions/import_hrd_from_hydranl_db.py | 7 ++++--- geoprob_pipe/workflow/base_objects.py | 7 ++++++- geoprob_pipe/workflow/questions/dir_hydranl_db.py | 4 ++-- geoprob_pipe/workflow/questions/import_hrd.py | 1 - geoprob_pipe/workflow/{workflow.py => steps.py} | 0 tests/workflow/test_workflow.py | 14 ++++++++++++++ 7 files changed, 27 insertions(+), 8 deletions(-) rename geoprob_pipe/workflow/{workflow.py => steps.py} (100%) create mode 100644 tests/workflow/test_workflow.py diff --git a/geoprob_pipe/workflow/__init__.py b/geoprob_pipe/workflow/__init__.py index f163e28d..59ab628d 100644 --- a/geoprob_pipe/workflow/__init__.py +++ b/geoprob_pipe/workflow/__init__.py @@ -1,2 +1,2 @@ -from geoprob_pipe.workflow.workflow import steps +from geoprob_pipe.workflow.steps import steps from geoprob_pipe.workflow.state import State diff --git a/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py b/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py index 090fd89f..9ccd8c00 100644 --- a/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py +++ b/geoprob_pipe/workflow/actions/import_hrd_from_hydranl_db.py @@ -1,4 +1,5 @@ from geoprob_pipe.workflow.base_objects import Action +from geoprob_pipe.workflow.questions import QuestionImportHRD, QuestionDirHydraNLDatabase from geoprob_pipe.cmd_app.spatial_layers.hrd.import_from_hrd import hrd_file_path import sqlite3 from pandas import read_sql @@ -8,7 +9,7 @@ class ActionImportHRDLocationsFromHydraNLDatabase(Action): def execute(self): - dir_to_db = self.state.question_answer.retrieve(question="dir_hydranl_db") + dir_to_db = self.state.question_answer.retrieve(question=QuestionDirHydraNLDatabase.label) path_to_db = hrd_file_path(hrd_dir=dir_to_db) conn = sqlite3.connect(path_to_db) df = read_sql("SELECT Name, XCoordinate, YCoordinate FROM HRDLocations", conn) @@ -20,8 +21,8 @@ def execute(self): @property def should_run(self) -> bool: - if not (self.state.question_answer.retrieve("import_hrd") == "Hydra-NL database" and - self.state.question_answer.retrieve("dir_hydranl_db") is not None): + if not (self.state.question_answer.retrieve(QuestionImportHRD.label) == "Hydra-NL database" and + self.state.question_answer.retrieve(QuestionDirHydraNLDatabase.label) is not None): return False return True diff --git a/geoprob_pipe/workflow/base_objects.py b/geoprob_pipe/workflow/base_objects.py index 3596716c..ddf8a037 100644 --- a/geoprob_pipe/workflow/base_objects.py +++ b/geoprob_pipe/workflow/base_objects.py @@ -7,8 +7,13 @@ from geoprob_pipe.workflow.state import State +class ClassName: + def __get__(self, obj, cls): + return cls.__name__ + + class Step(ABC): - label: str = None + label: str = ClassName() def __init__(self, state: State): self.state = state diff --git a/geoprob_pipe/workflow/questions/dir_hydranl_db.py b/geoprob_pipe/workflow/questions/dir_hydranl_db.py index 471e36a2..95051e32 100644 --- a/geoprob_pipe/workflow/questions/dir_hydranl_db.py +++ b/geoprob_pipe/workflow/questions/dir_hydranl_db.py @@ -1,4 +1,5 @@ from geoprob_pipe.workflow.base_objects import Question, ValidationResult +from geoprob_pipe.workflow.questions import QuestionImportHRD from typing import Optional from InquirerPy import inquirer import os @@ -23,7 +24,6 @@ def _folder_contains_hrd_db(hrd_dir: str) -> bool: return False class QuestionDirHydraNLDatabase(Question): - label = "dir_hydranl_db" def ask(self) -> str: return inquirer.text( @@ -44,7 +44,7 @@ def validate(answer) -> ValidationResult: @property def should_run(self) -> bool: - answer: Optional[str] = self.state.question_answer.retrieve("import_hrd") + answer: Optional[str] = self.state.question_answer.retrieve(QuestionImportHRD.label) if answer != "Hydra-NL database": return False return True diff --git a/geoprob_pipe/workflow/questions/import_hrd.py b/geoprob_pipe/workflow/questions/import_hrd.py index a7b4db5f..ac84648a 100644 --- a/geoprob_pipe/workflow/questions/import_hrd.py +++ b/geoprob_pipe/workflow/questions/import_hrd.py @@ -11,7 +11,6 @@ class QuestionImportHRD(Question): - label = "import_hrd" def ask(self) -> str: return inquirer.select( diff --git a/geoprob_pipe/workflow/workflow.py b/geoprob_pipe/workflow/steps.py similarity index 100% rename from geoprob_pipe/workflow/workflow.py rename to geoprob_pipe/workflow/steps.py diff --git a/tests/workflow/test_workflow.py b/tests/workflow/test_workflow.py new file mode 100644 index 00000000..364432b7 --- /dev/null +++ b/tests/workflow/test_workflow.py @@ -0,0 +1,14 @@ +from geoprob_pipe.workflow import State +from geoprob_pipe.workflow import steps + + + + + +def test_workflow(tmp_path): + state = State(file_dir=tmp_path) + for obj in steps: + step = obj(state=state) + print(f"{step.__name__}: {step.should_run=} {step.completed=}") + if step.should_run and not step.completed: + step.execute() From 770ac27c015f477edb7989bcc444676e137ccd44 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Sat, 22 Aug 2026 21:26:23 +0200 Subject: [PATCH 19/23] Unit test with popping answers to questions --- tests/workflow/test_workflow.py | 38 +++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/workflow/test_workflow.py b/tests/workflow/test_workflow.py index 364432b7..14475472 100644 --- a/tests/workflow/test_workflow.py +++ b/tests/workflow/test_workflow.py @@ -1,14 +1,40 @@ -from geoprob_pipe.workflow import State -from geoprob_pipe.workflow import steps - +from geoprob_pipe.workflow import State, steps +from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase, QuestionImportHRD +from geoprob_pipe.workflow.questions.import_hrd import CHOICES +from geoprob_pipe.workflow.base_objects import Question +import os +import random +# GeoProb-Pipe/tests/systeem_testen/224/hrd_files def test_workflow(tmp_path): + valid_answers = { + QuestionImportHRD.label: CHOICES, + QuestionDirHydraNLDatabase.label: [r"C:\Users\CP\git_clones\GeoProb-Pipe\GeoProb-PipeV4\GeoProb-Pipe\tests\systeem_testen\224\hrd_files"], + } + # TODO: Hoe specificeer ik een relatief pad wat werkt in Ubuntu GitHub en lokaal bij ons. + # TODO: Sowieso test bestanden opzetten en kijken of ik nog ergens lokale paden gebruik. + state = State(file_dir=tmp_path) for obj in steps: step = obj(state=state) - print(f"{step.__name__}: {step.should_run=} {step.completed=}") - if step.should_run and not step.completed: - step.execute() + print(f"\n{step.label}: {step.should_run=} {step.completed=}") + + # Check + if not step.should_run: + continue + if step.completed: + continue + + # Fake 'execute' question + if issubclass(obj, Question): + answer = random.choice(valid_answers[obj.label]) + print(f"Now popping '{answer}' to '{obj.label}'.") + state.question_answer.store(question_label=obj.label, answer=answer) + assert step.completed == True + continue + + # Thus Action + step.execute() From d96604a614de5118c6526dca17a19943cf5edd21 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Mon, 24 Aug 2026 06:13:24 +0200 Subject: [PATCH 20/23] Local paths verwijderd --- geoprob_pipe/workflow/cmd.py | 2 +- .../workflow/actions/test_import_hrd_from_hydranl_db.py | 5 ++++- tests/workflow/questions/test_dir_hydranl_db.py | 9 ++++++--- tests/workflow/test_workflow.py | 7 +++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/geoprob_pipe/workflow/cmd.py b/geoprob_pipe/workflow/cmd.py index 621ac20d..8478a976 100644 --- a/geoprob_pipe/workflow/cmd.py +++ b/geoprob_pipe/workflow/cmd.py @@ -2,7 +2,7 @@ import typer -state = State(file_path=r"C:\Users\CP\Downloads\false_fix\tmp.gpkg") +state = State(file_path=r"path\to\tmp.gpkg") app = typer.Typer(help="GeoProb-Pipe - CLI applicatie voor probabilistische piping berekeningen.", add_completion=False) diff --git a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py index dd77f470..9ed2703a 100644 --- a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py +++ b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py @@ -1,12 +1,15 @@ from geoprob_pipe.workflow import State from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase +from repo_utils.utils import repository_root_path +import os def test_action_import_hrd_locations_from_hydranl_db(tmp_path): state = State(file_dir=tmp_path) + repo_root = repository_root_path() state.question_answer.store( question_label="dir_hydranl_db", - answer=r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04") + answer=os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files")) action = ActionImportHRDLocationsFromHydraNLDatabase(state=state) action.execute() assert state.gdf.hrd_locations is not None diff --git a/tests/workflow/questions/test_dir_hydranl_db.py b/tests/workflow/questions/test_dir_hydranl_db.py index 55d48a7b..c3b48705 100644 --- a/tests/workflow/questions/test_dir_hydranl_db.py +++ b/tests/workflow/questions/test_dir_hydranl_db.py @@ -1,11 +1,14 @@ from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase +from repo_utils.utils import repository_root_path +import os def test_question_dir_hydra_nl_database(): + repo_root = repository_root_path() test_answers = [ - r"C:\Users\CP\Downloads\false_fix\Analyse16-2_V3.2.geoprob_pipe.gpkg", - r"C:\Users\CP\Downloads\false_fix\exports", - r"C:\Users\CP\Downloads\issue_invoer_wshd\alpha_versie_vincent\WBI2017_Benedenrijn_21-2_v04", + os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files", "WBI2017_Bovenrijn_224_v04.sqlite"), + os.path.join(repo_root, "tests", "systeem_testen", "224"), + os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files"), ] test_results = [ False, diff --git a/tests/workflow/test_workflow.py b/tests/workflow/test_workflow.py index 14475472..bf7303d1 100644 --- a/tests/workflow/test_workflow.py +++ b/tests/workflow/test_workflow.py @@ -1,8 +1,10 @@ +import os.path + from geoprob_pipe.workflow import State, steps from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase, QuestionImportHRD from geoprob_pipe.workflow.questions.import_hrd import CHOICES from geoprob_pipe.workflow.base_objects import Question -import os +from repo_utils.utils import repository_root_path import random @@ -10,9 +12,10 @@ def test_workflow(tmp_path): + repo_root = repository_root_path() valid_answers = { QuestionImportHRD.label: CHOICES, - QuestionDirHydraNLDatabase.label: [r"C:\Users\CP\git_clones\GeoProb-Pipe\GeoProb-PipeV4\GeoProb-Pipe\tests\systeem_testen\224\hrd_files"], + QuestionDirHydraNLDatabase.label: [os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files")], } # TODO: Hoe specificeer ik een relatief pad wat werkt in Ubuntu GitHub en lokaal bij ons. # TODO: Sowieso test bestanden opzetten en kijken of ik nog ergens lokale paden gebruik. From c713033f9a439e641f652bf1036045915537fab9 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Mon, 24 Aug 2026 06:43:20 +0200 Subject: [PATCH 21/23] Andere manier om repo root te vinden --- repo_utils/utils.py | 10 +++++++++ .../test_import_hrd_from_hydranl_db.py | 12 ++++++----- .../workflow/questions/test_dir_hydranl_db.py | 12 +++++++---- tests/workflow/test_workflow.py | 21 ++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/repo_utils/utils.py b/repo_utils/utils.py index 20c0aa67..99e32144 100644 --- a/repo_utils/utils.py +++ b/repo_utils/utils.py @@ -3,12 +3,22 @@ from typing import Optional +def find_repo_root(): + """ Alternative version to repository_root_path, which does not always seem to work in PyTest. """ + from pathlib import Path + import subprocess + return Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()) + + def repository_root_path() -> Optional[str]: # Bold guess, it is the execution path base_dir = os.getcwd() + print(f"{base_dir=}") try: + print(f"Now trying Repo-class") repo_root = Repo(base_dir, search_parent_directories=False).working_tree_dir + print(f"From Repo-class {repo_root=}") return repo_root except InvalidGitRepositoryError: pass diff --git a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py index 9ed2703a..5d9a31c3 100644 --- a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py +++ b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py @@ -1,12 +1,14 @@ -from geoprob_pipe.workflow import State -from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase -from repo_utils.utils import repository_root_path -import os def test_action_import_hrd_locations_from_hydranl_db(tmp_path): + from geoprob_pipe.workflow import State + from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase + from repo_utils.utils import find_repo_root + import os + + repo_root = find_repo_root() + state = State(file_dir=tmp_path) - repo_root = repository_root_path() state.question_answer.store( question_label="dir_hydranl_db", answer=os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files")) diff --git a/tests/workflow/questions/test_dir_hydranl_db.py b/tests/workflow/questions/test_dir_hydranl_db.py index c3b48705..26719e24 100644 --- a/tests/workflow/questions/test_dir_hydranl_db.py +++ b/tests/workflow/questions/test_dir_hydranl_db.py @@ -1,15 +1,18 @@ -from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase -from repo_utils.utils import repository_root_path -import os + def test_question_dir_hydra_nl_database(): - repo_root = repository_root_path() + ## + from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase + from repo_utils.utils import find_repo_root + import os + repo_root = find_repo_root() test_answers = [ os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files", "WBI2017_Bovenrijn_224_v04.sqlite"), os.path.join(repo_root, "tests", "systeem_testen", "224"), os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files"), ] + print(f"{test_answers[2]=}") test_results = [ False, False, @@ -17,3 +20,4 @@ def test_question_dir_hydra_nl_database(): ] for test_answer, test_result in zip(test_answers, test_results): assert QuestionDirHydraNLDatabase.validate(answer=test_answer).is_valid == test_result + ## \ No newline at end of file diff --git a/tests/workflow/test_workflow.py b/tests/workflow/test_workflow.py index bf7303d1..0f85bc2a 100644 --- a/tests/workflow/test_workflow.py +++ b/tests/workflow/test_workflow.py @@ -1,18 +1,15 @@ -import os.path - -from geoprob_pipe.workflow import State, steps -from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase, QuestionImportHRD -from geoprob_pipe.workflow.questions.import_hrd import CHOICES -from geoprob_pipe.workflow.base_objects import Question -from repo_utils.utils import repository_root_path -import random - - -# GeoProb-Pipe/tests/systeem_testen/224/hrd_files def test_workflow(tmp_path): - repo_root = repository_root_path() + import os.path + from geoprob_pipe.workflow import State, steps + from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase, QuestionImportHRD + from geoprob_pipe.workflow.questions.import_hrd import CHOICES + from geoprob_pipe.workflow.base_objects import Question + from repo_utils.utils import find_repo_root + import random + + repo_root = find_repo_root() valid_answers = { QuestionImportHRD.label: CHOICES, QuestionDirHydraNLDatabase.label: [os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files")], From 90b002b6ac4ba64d3fa83076a7fa3808dc3da6fe Mon Sep 17 00:00:00 2001 From: chrispijo Date: Mon, 24 Aug 2026 06:56:12 +0200 Subject: [PATCH 22/23] Removed some old code that caused issues --- .../actions/test_import_hrd_from_hydranl_db.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py index 5d9a31c3..db5f5802 100644 --- a/tests/workflow/actions/test_import_hrd_from_hydranl_db.py +++ b/tests/workflow/actions/test_import_hrd_from_hydranl_db.py @@ -1,18 +1,21 @@ def test_action_import_hrd_locations_from_hydranl_db(tmp_path): + ## from geoprob_pipe.workflow import State from geoprob_pipe.workflow.actions import ActionImportHRDLocationsFromHydraNLDatabase + from geoprob_pipe.workflow.questions import QuestionDirHydraNLDatabase from repo_utils.utils import find_repo_root import os - repo_root = find_repo_root() - state = State(file_dir=tmp_path) + repo_root = find_repo_root() + dir_to_db = os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files") state.question_answer.store( - question_label="dir_hydranl_db", - answer=os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files")) + question_label=QuestionDirHydraNLDatabase.label, + answer=dir_to_db) action = ActionImportHRDLocationsFromHydraNLDatabase(state=state) action.execute() assert state.gdf.hrd_locations is not None - assert state.gdf.hrd_locations.__len__() == 400 + assert state.gdf.hrd_locations.__len__() == 13 + ## From 2b9660700293b085a828199ff637985ba0f857ec Mon Sep 17 00:00:00 2001 From: chrispijo Date: Mon, 24 Aug 2026 08:46:18 +0200 Subject: [PATCH 23/23] New test --- tests/workflow/questions/test_dir_hydranl_db.py | 1 - tests/workflow/questions/test_import_hrd.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/workflow/questions/test_import_hrd.py diff --git a/tests/workflow/questions/test_dir_hydranl_db.py b/tests/workflow/questions/test_dir_hydranl_db.py index 26719e24..b90571bf 100644 --- a/tests/workflow/questions/test_dir_hydranl_db.py +++ b/tests/workflow/questions/test_dir_hydranl_db.py @@ -12,7 +12,6 @@ def test_question_dir_hydra_nl_database(): os.path.join(repo_root, "tests", "systeem_testen", "224"), os.path.join(repo_root, "tests", "systeem_testen", "224", "hrd_files"), ] - print(f"{test_answers[2]=}") test_results = [ False, False, diff --git a/tests/workflow/questions/test_import_hrd.py b/tests/workflow/questions/test_import_hrd.py new file mode 100644 index 00000000..e96b2432 --- /dev/null +++ b/tests/workflow/questions/test_import_hrd.py @@ -0,0 +1,13 @@ + + +def test_question_import_hrd(): + from geoprob_pipe.workflow.questions import QuestionImportHRD + from geoprob_pipe.workflow.questions.import_hrd import CHOICES + import numpy as np + + test_answers = CHOICES.copy() # Force copy, otherwise you change CHOICES with .extend() on the next line. + test_answers.extend(["Ja", None, 1, np.nan]) + test_results = [True, True, True, False, False, False, False] + for test_answer, test_result in zip(test_answers, test_results): + assert QuestionImportHRD.validate(answer=test_answer).is_valid == test_result, \ + f"Failed on {test_answer}, expected {test_result}."