From 1d3533cb10df2ef4be83a3d3c6f2a278dd1d1a81 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Tue, 18 Aug 2026 14:19:12 +0200 Subject: [PATCH 01/10] debug log for each utp with values from worker --- .../calculations/systems/build_and_run.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 085970c9..4bb661ff 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -95,16 +95,27 @@ def _worker(row_unique: dict): # noinspection PyBroadException try: with redirect_stdout(log_buffer), redirect_stderr(log_buffer): + logger = logging.getLogger(__name__) + logger.debug("Start berekening voor %s", row_unique) # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() - + logger.debug("SystemCalculation voltooid:") + # Collect results df_limit_state = collect_df_beta_limit_state(calc) + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") df_scenario_rp = collect_df_beta_scenario_rp(calc) + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast}") df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) @@ -115,7 +126,7 @@ def _worker(row_unique: dict): df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, validation_message=calc.validation_messages - ), None, None + ), log_buffer.getvalue(), row_unique except Exception: tb = traceback.format_exc() From 0fc6d2c1b21f01a7d467f8c66a35d891d9a69e65 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 09:19:21 +0200 Subject: [PATCH 02/10] worker error log renamed and only messeges on actual errors --- .../calculations/systems/build_and_run.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 4bb661ff..e4b4687e 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -100,8 +100,9 @@ def _worker(row_unique: dict): # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() - logger.debug("SystemCalculation voltooid:") - + logger.debug("SystemCalculation voltooid.") + logger.debug("Validation messeges:") + logger.debug(f"\n{calc.validation_messages.df}") # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") @@ -176,24 +177,27 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes last_report = time.time() done = 0 + log_errors = 0 results: List[CalcResult] = [] pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) # Multiprocessing setup - error_rows = [] + log_rows = [] with Pool(processes=pool_size, initializer=_init_worker, initargs=( geohydrologisch_model, geopackage_filepath, to_run_vakken_ids)) as pool: - for res, error_logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): + for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): if isinstance(res, CalcResult): results.append(res) - if isinstance(error_logs, str): - error_rows.append({ + if isinstance(logs, str): + log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], "ondergrondscenario_naam": row["ondergrondscenario_naam"], "vak_id": row["vak_id"], - "error_logs": error_logs, + "logs": logs, }) + if "ERROR" in logs: + log_errors += 1 done += 1 # Alleen kijken of er gelogd moet worden bij de laatste @@ -208,25 +212,21 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes # Log error_count_append = "" - if error_rows.__len__() > 0: - error_count_append = f" (of which {error_rows.__len__()} failed calculations)" + + if log_errors > 0: + error_count_append = f" (of which {log_errors} failed calculations)" logger.info(f"Progress: {done:>{char_len_total}} / {n_calc_totaal} calculations{error_count_append}.") last_report = now # Push errors to database (if any) conn = sqlite3.connect(geoprob_pipe.input_data.app_settings.geopackage_filepath) - table_name = "calculation_error_logs" - if error_rows.__len__() > 0: - df_errors = DataFrame(data=error_rows) - df_errors.to_sql(table_name, conn, if_exists="replace", index=False) - conn.close() - logger.error(f"There are {error_rows.__len__()} failed calculations. Error logs are stored inside the " + table_name = "calculation_logs" + df_logs = DataFrame(data=log_rows) + df_logs.to_sql(table_name, conn, if_exists="replace", index=False) + conn.commit() + conn.close() + if log_errors > 0: + logger.error(f"There are {log_errors} failed calculations. Error logs are stored inside the " f"GeoPacakge in table '{table_name}'.") - else: - # Remove old table (if exists) - cur = conn.cursor() - cur.execute(f"DROP TABLE IF EXISTS {table_name};") - conn.commit() - conn.close() - + return results From d5effc79430f694af3cf4734ca8a5ef807213985 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 10:07:26 +0200 Subject: [PATCH 03/10] 0 total model runs warning --- geoprob_pipe/calculations/systems/build_and_run.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index e4b4687e..645b54ee 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -101,12 +101,14 @@ def _worker(row_unique: dict): calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() logger.debug("SystemCalculation voltooid.") - logger.debug("Validation messeges:") + logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") logger.debug(f"\n{df_limit_state}") + if any(r == 0 for r in df_limit_state.total_model_runs): + logger.warning("Limit state with 0 total model runs encountered.") df_scenario_rp = collect_df_beta_scenario_rp(calc) logger.debug("df_scenario_rp:") logger.debug(f"\n{df_scenario_rp}") @@ -178,6 +180,7 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes last_report = time.time() done = 0 log_errors = 0 + error_rows = [] results: List[CalcResult] = [] pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) @@ -196,8 +199,9 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes "vak_id": row["vak_id"], "logs": logs, }) - if "ERROR" in logs: + if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): log_errors += 1 + error_rows.append(row) done += 1 # Alleen kijken of er gelogd moet worden bij de laatste @@ -226,7 +230,8 @@ def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcRes conn.commit() conn.close() if log_errors > 0: - logger.error(f"There are {log_errors} failed calculations. Error logs are stored inside the " - f"GeoPacakge in table '{table_name}'.") + logger.error(f"There are {log_errors} failed calculations. The calculation logs are stored inside the " + f"GeoPackage in table '{table_name}'. The following rows are marked:" + f"\n{error_rows}") return results From bdb16a14c4fe12aa99cf445ea172274e30e326cc Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 11:50:40 +0200 Subject: [PATCH 04/10] Form settings test --- .../systems/base_objects/system_calculation.py | 2 ++ geoprob_pipe/calculations/systems/build_and_run.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 112135d0..c7d66583 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,7 +13,9 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, + # "minimum_iterations": 10, "relaxation_factor": 0.4, + "reuse_calculations": False, } # DEFAULT_RELIABILITY_SETTINGS: Dict[str, Union[str, float, int]] = { diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 645b54ee..635e2ca1 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -97,12 +97,25 @@ def _worker(row_unique: dict): with redirect_stdout(log_buffer), redirect_stderr(log_buffer): logger = logging.getLogger(__name__) logger.debug("Start berekening voor %s", row_unique) + # Build and run calculations calc = _BUILDER.build_instance(row_unique=row_unique) calc.run() + logger.debug("SystemCalculation voltooid.") logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") + + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() + + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() + # Collect results df_limit_state = collect_df_beta_limit_state(calc) logger.debug("df_limit_state:") From f7df84ccec898a5d71e42e639d6ee32d71fe6384 Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Wed, 19 Aug 2026 14:08:45 +0200 Subject: [PATCH 05/10] test setting update --- .../calculations/systems/base_objects/system_calculation.py | 3 ++- geoprob_pipe/calculations/systems/build_and_run.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index c7d66583..95abe9f1 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,7 +13,8 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, - # "minimum_iterations": 10, + # "minimum_iterations": 5, # Don't allow zero iterations + # "minimum_directions": 2, # Prevent one direction solutions (single dominant alpha) "relaxation_factor": 0.4, "reuse_calculations": False, } diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 635e2ca1..f8320caa 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -131,7 +131,11 @@ def _worker(row_unique: dict): df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast}") + logger.debug(f"\n{df_stochast.to_string()}") + if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") + if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) From 9657ebe0e7fc257a73dca96de1f737db646a27ed Mon Sep 17 00:00:00 2001 From: Vincent Jilesen Date: Thu, 20 Aug 2026 10:55:12 +0200 Subject: [PATCH 06/10] debug only tabel in calculation logs --- .../calculations/systems/build_and_run.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index f8320caa..6691a69f 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -17,7 +17,7 @@ collect_stochast_values, calculate_derived_values) import logging from pandas import DataFrame - +import os if TYPE_CHECKING: from geoprob_pipe import GeoProbPipe from geoprob_pipe.calculations.systems.base_objects\ @@ -103,35 +103,40 @@ def _worker(row_unique: dict): calc.run() logger.debug("SystemCalculation voltooid.") + logger.debug("Validation messages:") logger.debug(f"\n{calc.validation_messages.df}") - - logger.debug("Limit states print:") - for lm in calc.results.dps_limit_states: - lm.print() - - logger.debug("Combine project print:") - calc.results.combine_project.design_point.print() + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() - logger.debug("Reliability project print:") - calc.results.reliability_project.design_point.print() + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() # Collect results df_limit_state = collect_df_beta_limit_state(calc) - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") if any(r == 0 for r in df_limit_state.total_model_runs): logger.warning("Limit state with 0 total model runs encountered.") df_scenario_rp = collect_df_beta_scenario_rp(calc) - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast.to_string()}") if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): From 675d8f3d3e4f8f7731e5cdc9fd213c197a27d788 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Thu, 20 Aug 2026 16:13:12 +0200 Subject: [PATCH 07/10] Herindeling build_and_run.py --- geoprob_pipe/app_object.py | 4 +- .../base_objects/system_calculation.py | 9 - .../calculations/systems/build_and_run.py | 351 ++++++++++-------- 3 files changed, 198 insertions(+), 166 deletions(-) diff --git a/geoprob_pipe/app_object.py b/geoprob_pipe/app_object.py index 549c6692..32716b57 100644 --- a/geoprob_pipe/app_object.py +++ b/geoprob_pipe/app_object.py @@ -21,7 +21,7 @@ from geoprob_pipe.results import Results from geoprob_pipe.spatial import Spatial from geoprob_pipe.visualizations import Visualizations -from geoprob_pipe.calculations.systems.build_and_run import build_and_run_system_calculations +from geoprob_pipe.calculations.systems.build_and_run import BuildAndRunCalculations from geoprob_pipe.utils.update_metadata import update_metadata import logging if TYPE_CHECKING: @@ -57,7 +57,7 @@ def __init__(self, app_settings: ApplicationSettings) -> None: # self._read_calculation_settings() # TODO: Not part of new version # TODO: Unsure if the single statement belongs here. Wouldn't it be part of input data? - self.calc_results: List[CalcResult] = build_and_run_system_calculations(self) + self.calc_results: List[CalcResult] = BuildAndRunCalculations(self).run() self.results = Results(self) # Log finish diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 95abe9f1..85c68fce 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -19,15 +19,6 @@ "reuse_calculations": False, } -# DEFAULT_RELIABILITY_SETTINGS: Dict[str, Union[str, float, int]] = { -# "reliability_method": ReliabilityMethod.form.__str__(), -# "variation_coefficient": 0.02, -# "maximum_iterations": 10_000, -# "relaxation_factor": 0.4, -# "epsilon_beta": 0.05, -# "relaxation_loops": 10, -# } - class SystemSetup: diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 6691a69f..325ec027 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -19,7 +19,7 @@ from pandas import DataFrame import os if TYPE_CHECKING: - from geoprob_pipe import GeoProbPipe + from geoprob_pipe import GeoProbPipe, SystemCalculation from geoprob_pipe.calculations.systems.base_objects\ .base_system_build import BaseSystemBuilder from geoprob_pipe.utils.validation_messages import ValidationMessages @@ -70,13 +70,7 @@ def _init_worker( to_run_vakken_ids=to_run_vakken_ids)) -def _worker(row_unique: dict): - """ De worker functie die op de parallelle rekenkernen wordt gedraaid. - - :param row_unique: Identificatie naar unieke berekening, bijvoorbeeld - {'uittredepunt_id': 1, 'ondergrondscenario_naam': 'scenario1', 'vak_id': 4}. - :return: Tuple[Optional[CalcResult], Optional[str], Optional[dict]] - """ +def _logging_code(): log_buffer = StringIO() buffer_handler = logging.StreamHandler(log_buffer) buffer_handler.setLevel(logging.DEBUG) @@ -92,72 +86,96 @@ def _worker(row_unique: dict): root.addHandler(buffer_handler) logging.captureWarnings(True) + return log_buffer, buffer_handler, root, prev_level + + +def _run_calculation(row_unique: dict) -> SystemCalculation: + logger.debug("Start berekening voor %s", row_unique) + calc: SystemCalculation = _BUILDER.build_instance(row_unique=row_unique) + calc.run() + logger.debug("SystemCalculation voltooid.") # TODO: Skip if no issue? + + logger.debug("Validation messages:") + logger.debug(f"\n{calc.validation_messages.df}") # TODO: Only if there are any? + + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("Limit states print:") + for lm in calc.results.dps_limit_states: + lm.print() + + logger.debug("Combine project print:") + calc.results.combine_project.design_point.print() + + logger.debug("Reliability project print:") + calc.results.reliability_project.design_point.print() + + return calc + + +def _collect_results(calc: SystemCalculation) -> CalcResult: + df_limit_state = collect_df_beta_limit_state(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_limit_state:") + logger.debug(f"\n{df_limit_state}") + if any(r == 0 for r in df_limit_state.total_model_runs): + logger.warning("Limit state with 0 total model runs encountered. " + "Notify developer and re-run calculations. ") + + df_scenario_rp = collect_df_beta_scenario_rp(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_rp:") + logger.debug(f"\n{df_scenario_rp}") + + df_scenario_cp = collect_df_beta_scenario_cp(calc) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_scenario_cp:") + logger.debug(f"\n{df_scenario_cp}") + + df_scenario_final = collect_df_beta_scenario_final(calc) + + df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) + if os.environ.get("GEOPROB_DEBUG", False): + logger.debug("df_stochast:") + logger.debug(f"\n{df_stochast.to_string()}") + + if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") + if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): + logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") + df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) + df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) + df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) + df_scenario_final = df_scenario_final.drop(columns=["system_calculation"]) + + return CalcResult( + df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, + df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, + validation_message=calc.validation_messages) + + +def _worker(row_unique: dict): + """ De worker functie die op de parallelle rekenkernen wordt gedraaid. + + :param row_unique: Identificatie naar unieke berekening, bijvoorbeeld + {'uittredepunt_id': 1, 'ondergrondscenario_naam': 'scenario1', 'vak_id': 4}. + :return: Tuple[Optional[CalcResult], Optional[str], Optional[dict]] + """ + log_buffer, buffer_handler, root, prev_level = _logging_code() + # noinspection PyBroadException try: with redirect_stdout(log_buffer), redirect_stderr(log_buffer): - logger = logging.getLogger(__name__) - logger.debug("Start berekening voor %s", row_unique) - - # Build and run calculations - calc = _BUILDER.build_instance(row_unique=row_unique) - calc.run() - - logger.debug("SystemCalculation voltooid.") - - logger.debug("Validation messages:") - logger.debug(f"\n{calc.validation_messages.df}") - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("Limit states print:") - for lm in calc.results.dps_limit_states: - lm.print() - - logger.debug("Combine project print:") - calc.results.combine_project.design_point.print() - - logger.debug("Reliability project print:") - calc.results.reliability_project.design_point.print() - - # Collect results - df_limit_state = collect_df_beta_limit_state(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") - if any(r == 0 for r in df_limit_state.total_model_runs): - logger.warning("Limit state with 0 total model runs encountered.") - df_scenario_rp = collect_df_beta_scenario_rp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") - df_scenario_cp = collect_df_beta_scenario_cp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") - df_scenario_final = collect_df_beta_scenario_final(calc) - df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") - if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): - logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") - if df_scenario_rp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): - logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in reliability project.") - df_derived = calculate_derived_values(df_scenarios_final=df_scenario_final, geohydrologisch_model=_MODEL) - df_scenario_rp = df_scenario_rp.drop(columns=["system_calculation"]) - df_scenario_cp = df_scenario_cp.drop(columns=["system_calculation"]) - df_scenario_final = df_scenario_final.drop(columns=["system_calculation"]) - - # Return results (without calculation object) - return CalcResult( - df_limit_state=df_limit_state, df_scenario_rp=df_scenario_rp, df_scenario_cp=df_scenario_cp, - df_scenario_final=df_scenario_final, df_stochast=df_stochast, df_derived=df_derived, - validation_message=calc.validation_messages - ), log_buffer.getvalue(), row_unique + # logger = logging.getLogger(__name__) + calc = _run_calculation(row_unique) + result = _collect_results(calc) + return result, log_buffer.getvalue(), row_unique except Exception: tb = traceback.format_exc() log_buffer.write(tb) buffer_handler.flush() return None, log_buffer.getvalue(), row_unique + finally: # Handler altijd verwijderen logging.captureWarnings(False) @@ -166,94 +184,117 @@ def _worker(row_unique: dict): buffer_handler.close() -def build_and_run_system_calculations(geoprob_pipe: GeoProbPipe) -> List[CalcResult]: - """ In deze functie worden de parameters voor de berekeningen verzamelt, - aan de workers gegeven en vervolgens de resultaten verzameld. - """ - geohydrologisch_model = geoprob_pipe.input_data.geohydrologisch_model - geopackage_filepath = ( - geoprob_pipe.input_data.app_settings.geopackage_filepath) - to_run_vakken_ids = geoprob_pipe.input_data.app_settings.to_run_vakken_ids - system_builder: BaseSystemBuilder = ( - CALCULATION_MAPPER[geohydrologisch_model]['system_builder']( - geopackage_filepath=geopackage_filepath, - to_run_vakken_ids=to_run_vakken_ids)) +class BuildAndRunCalculations: + """ In dit object worden de parameters voor de berekeningen verzamelt, aan de workers gegeven en vervolgens de + resultaten verzameld. """ + + def __init__(self, geoprob_pipe: GeoProbPipe): + """ Init zet dit object op, maar het uitvoeren van de berekeningen gaat via de method .run(). """ + + self.geoprob_pipe: GeoProbPipe = geoprob_pipe + self.geohydrologisch_model: str = geoprob_pipe.input_data.geohydrologisch_model + self.geopackage_filepath: str = geoprob_pipe.input_data.app_settings.geopackage_filepath + self.to_run_vakken_ids: str = geoprob_pipe.input_data.app_settings.to_run_vakken_ids + + self._construct_run_settings() + self._setup_progress_variables() + + # Run logic with method .run() + + def _construct_run_settings(self): + """ Opzetten van de system builder en andere settings voor de berekeningen. """ + + logger.info("Now preparing for calculations...") + self.system_builder: BaseSystemBuilder = CALCULATION_MAPPER[self.geohydrologisch_model]['system_builder']( + geopackage_filepath=self.geopackage_filepath, + to_run_vakken_ids=self.to_run_vakken_ids) + self.df_unique_combos = self.system_builder.setup_iteration_df() + + # Bepaal de parameters voor de multiprocessing setup en de logger + self.n_threads: int = cpu_count() - 1 + self.n_calc_totaal: int = len(self.df_unique_combos) + + # Minimaal 5 berekeningen per chunk en grootte van chunk beperken zodat er gelogd kan worden. + self.chunk_size: int = max(math.ceil(self.n_calc_totaal / (self.n_threads * 10)), 5) + + def _setup_progress_variables(self): + """ Simpel initiƫren van een aantal variabelen die nodig zijn tijdens de berekeningen. """ + + self.last_report: float = time.time() + self.done = 0 + self.log_errors = 0 + self.error_rows = [] + self.char_len_total: int = str(self.n_calc_totaal).__len__() + self.log_rows = [] + + def _report_calculation_progress_to_user(self): + """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ + + # Alleen kijken of er gelogd moet worden bij de laatste + # berekening die uit de chunk komt. + if self.done % self.chunk_size != 0: + return + + # Alleen loggen wanneer 30 seconden is gepasseerd + now = time.time() + if now - self.last_report < 30.0: + return + + # Log + error_count_append = "" + if self.log_errors > 0: + error_count_append = f" (of which {self.log_errors} failed calculations)" + logger.info( + f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations{error_count_append}.") + self.last_report = now + + def _push_errors_to_database(self): + """ Aan eind van run pushed deze method de errors (if any) naar de database. """ + + conn = sqlite3.connect(self.geopackage_filepath) + table_name = "calculation_logs" + df_logs = DataFrame(data=self.log_rows) + df_logs.to_sql(table_name, conn, if_exists="replace", index=False) + conn.commit() + conn.close() + if self.log_errors > 0: + logger.error(f"There are {self.log_errors} failed calculations. The calculation logs are stored inside the " + f"GeoPackage in table '{table_name}'. The following rows are marked:" + f"\n{self.error_rows}") + + def run(self) -> List[CalcResult]: + + logger.info( + f"Running {self.n_calc_totaal} calculations in chunks of {self.chunk_size}" + f" with {self.n_threads} parallel threads.") + logger.info( + f"Progress: {0:>{self.char_len_total}} / {self.n_calc_totaal} calculations.") + + rows = [dict(zip(self.df_unique_combos.columns, r)) + for r in self.df_unique_combos.itertuples(index=False, name=None)] + results: List[CalcResult] = [] + pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) + + # Multiprocessing setup + with Pool(processes=pool_size, initializer=_init_worker, initargs=( + self.geohydrologisch_model, self.geopackage_filepath, self.to_run_vakken_ids)) as pool: + + for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): + if isinstance(res, CalcResult): + results.append(res) + if isinstance(logs, str): + self.log_rows.append({ + "uittredepunt_id": row["uittredepunt_id"], + "ondergrondscenario_naam": row["ondergrondscenario_naam"], + "vak_id": row["vak_id"], + "logs": logs, + }) + if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): + self.log_errors += 1 + self.error_rows.append(row) + self.done += 1 + + self._report_calculation_progress_to_user() - logger.info("Now building and running calculations...") - df_unique_combos = system_builder.setup_iteration_df() - # Bepaal de parameters voor de multiprocessing setup en de logger - n_threads: int = cpu_count() - 1 - n_calc_totaal: int = len(df_unique_combos) - # Minimaal 5 berekeningen per chunk en grootte van chunk beperken - # zodat er gelogd kan worden. - chunk_size: int = max(math.ceil(n_calc_totaal / (n_threads * 10)), 5) - logger.info( - f"Running {n_calc_totaal} calculations in chunks of {chunk_size}" - f" with {n_threads} parallel threads.") - char_len_total = str(n_calc_totaal).__len__() - logger.info( - f"Progress: {0:>{char_len_total}} / {n_calc_totaal} calculations.") - - # Dicts zijn gemakkelijker te pickelen en daardoor sneller te - # verwerken dan pandas series. - rows = [dict(zip(df_unique_combos.columns, r)) - for r in df_unique_combos.itertuples(index=False, name=None)] - - last_report = time.time() - done = 0 - log_errors = 0 - error_rows = [] - results: List[CalcResult] = [] - pool_size = max(min(math.floor(n_calc_totaal / chunk_size), n_threads), 1) - - # Multiprocessing setup - log_rows = [] - with Pool(processes=pool_size, initializer=_init_worker, initargs=( - geohydrologisch_model, geopackage_filepath, to_run_vakken_ids)) as pool: - - for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=chunk_size): - if isinstance(res, CalcResult): - results.append(res) - if isinstance(logs, str): - log_rows.append({ - "uittredepunt_id": row["uittredepunt_id"], - "ondergrondscenario_naam": row["ondergrondscenario_naam"], - "vak_id": row["vak_id"], - "logs": logs, - }) - if any(level in logs for level in ("WARNING", "ERROR", "CRITICAL")): - log_errors += 1 - error_rows.append(row) - done += 1 - - # Alleen kijken of er gelogd moet worden bij de laatste - # berekening die uit de chunk komt. - if done % chunk_size != 0: - continue - - # Alleen loggen wanneer 30 seconden is gepasseerd - now = time.time() - if now - last_report < 30.0: - continue - - # Log - error_count_append = "" - - if log_errors > 0: - error_count_append = f" (of which {log_errors} failed calculations)" - logger.info(f"Progress: {done:>{char_len_total}} / {n_calc_totaal} calculations{error_count_append}.") - last_report = now - - # Push errors to database (if any) - conn = sqlite3.connect(geoprob_pipe.input_data.app_settings.geopackage_filepath) - table_name = "calculation_logs" - df_logs = DataFrame(data=log_rows) - df_logs.to_sql(table_name, conn, if_exists="replace", index=False) - conn.commit() - conn.close() - if log_errors > 0: - logger.error(f"There are {log_errors} failed calculations. The calculation logs are stored inside the " - f"GeoPackage in table '{table_name}'. The following rows are marked:" - f"\n{error_rows}") - - return results + self._push_errors_to_database() + return results From 99c537bcca6af219dc091ea62155983c9905fee0 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 08:57:52 +0200 Subject: [PATCH 08/10] Small change --- .../calculations/systems/base_objects/system_calculation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py index 85c68fce..73618798 100644 --- a/geoprob_pipe/calculations/systems/base_objects/system_calculation.py +++ b/geoprob_pipe/calculations/systems/base_objects/system_calculation.py @@ -13,8 +13,6 @@ "reliability_method": ReliabilityMethod.form.__str__(), "variation_coefficient": 0.02, "maximum_iterations": 1000, - # "minimum_iterations": 5, # Don't allow zero iterations - # "minimum_directions": 2, # Prevent one direction solutions (single dominant alpha) "relaxation_factor": 0.4, "reuse_calculations": False, } From 541f0cc49b0d7ab29ca90fe5996c02fe9e79d597 Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 12:24:39 +0200 Subject: [PATCH 09/10] Afgerond --- .../calculations/systems/build_and_run.py | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 325ec027..1a2146a6 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -90,15 +90,18 @@ def _logging_code(): def _run_calculation(row_unique: dict) -> SystemCalculation: - logger.debug("Start berekening voor %s", row_unique) + debug: bool = os.environ.get("GEOPROB_DEBUG", False) + if debug: logger.debug("Start berekening voor %s", row_unique) + + # Run calculation calc: SystemCalculation = _BUILDER.build_instance(row_unique=row_unique) calc.run() - logger.debug("SystemCalculation voltooid.") # TODO: Skip if no issue? + if debug: logger.debug("SystemCalculation voltooid.") - logger.debug("Validation messages:") - logger.debug(f"\n{calc.validation_messages.df}") # TODO: Only if there are any? + # Remainder is logging in case of validation messages or debug modus + if calc.validation_messages.df: logger.debug(f"Validation messages:\n{calc.validation_messages.df}") - if os.environ.get("GEOPROB_DEBUG", False): + if debug: logger.debug("Limit states print:") for lm in calc.results.dps_limit_states: lm.print() @@ -113,30 +116,22 @@ def _run_calculation(row_unique: dict) -> SystemCalculation: def _collect_results(calc: SystemCalculation) -> CalcResult: + debug: bool = os.environ.get("GEOPROB_DEBUG", False) df_limit_state = collect_df_beta_limit_state(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_limit_state:") - logger.debug(f"\n{df_limit_state}") + if debug: logger.debug(f"df_limit_state:\n{df_limit_state}") if any(r == 0 for r in df_limit_state.total_model_runs): - logger.warning("Limit state with 0 total model runs encountered. " - "Notify developer and re-run calculations. ") + logger.warning("Limit state with 0 total model runs encountered. Notify developer and re-run calculations. ") df_scenario_rp = collect_df_beta_scenario_rp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_rp:") - logger.debug(f"\n{df_scenario_rp}") + if debug: logger.debug(f"df_scenario_rp:\n{df_scenario_rp}") df_scenario_cp = collect_df_beta_scenario_cp(calc) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_scenario_cp:") - logger.debug(f"\n{df_scenario_cp}") + if debug: logger.debug(f"df_scenario_cp:\n{df_scenario_cp}") df_scenario_final = collect_df_beta_scenario_final(calc) df_stochast = collect_stochast_values(calc, df_scenario_final=df_scenario_final) - if os.environ.get("GEOPROB_DEBUG", False): - logger.debug("df_stochast:") - logger.debug(f"\n{df_stochast.to_string()}") + if debug: logger.debug(f"df_stochast:\n{df_stochast.to_string()}") if df_scenario_cp.converged is True and any(a >= 0.99 for a in df_stochast.alpha): logger.warning("Unrealistically dominant (alpha >= 0.99) parameter found in combined project.") @@ -196,12 +191,12 @@ def __init__(self, geoprob_pipe: GeoProbPipe): self.geopackage_filepath: str = geoprob_pipe.input_data.app_settings.geopackage_filepath self.to_run_vakken_ids: str = geoprob_pipe.input_data.app_settings.to_run_vakken_ids - self._construct_run_settings() - self._setup_progress_variables() + self._construct_system_builder_and_settings() + self._setup_calculation_progress_variables() # Run logic with method .run() - def _construct_run_settings(self): + def _construct_system_builder_and_settings(self): """ Opzetten van de system builder en andere settings voor de berekeningen. """ logger.info("Now preparing for calculations...") @@ -217,7 +212,7 @@ def _construct_run_settings(self): # Minimaal 5 berekeningen per chunk en grootte van chunk beperken zodat er gelogd kan worden. self.chunk_size: int = max(math.ceil(self.n_calc_totaal / (self.n_threads * 10)), 5) - def _setup_progress_variables(self): + def _setup_calculation_progress_variables(self): """ Simpel initiƫren van een aantal variabelen die nodig zijn tijdens de berekeningen. """ self.last_report: float = time.time() @@ -226,12 +221,20 @@ def _setup_progress_variables(self): self.error_rows = [] self.char_len_total: int = str(self.n_calc_totaal).__len__() self.log_rows = [] + self.results: List[CalcResult] = [] def _report_calculation_progress_to_user(self): """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ - # Alleen kijken of er gelogd moet worden bij de laatste - # berekening die uit de chunk komt. + # If finished + error_count_append = "" + if self.log_errors > 0: + error_count_append = f" (of which {self.log_errors} failed calculations)" + if self.n_calc_totaal == self.done: + logger.info(f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations" + f"{error_count_append}.") + + # Alleen kijken of er gelogd moet worden bij de laatste berekening die uit de chunk komt. if self.done % self.chunk_size != 0: return @@ -240,15 +243,11 @@ def _report_calculation_progress_to_user(self): if now - self.last_report < 30.0: return - # Log - error_count_append = "" - if self.log_errors > 0: - error_count_append = f" (of which {self.log_errors} failed calculations)" logger.info( f"Progress: {self.done:>{self.char_len_total}} / {self.n_calc_totaal} calculations{error_count_append}.") self.last_report = now - def _push_errors_to_database(self): + def _push_resulting_error_messages_to_database(self): """ Aan eind van run pushed deze method de errors (if any) naar de database. """ conn = sqlite3.connect(self.geopackage_filepath) @@ -272,7 +271,6 @@ def run(self) -> List[CalcResult]: rows = [dict(zip(self.df_unique_combos.columns, r)) for r in self.df_unique_combos.itertuples(index=False, name=None)] - results: List[CalcResult] = [] pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) # Multiprocessing setup @@ -281,7 +279,7 @@ def run(self) -> List[CalcResult]: for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): if isinstance(res, CalcResult): - results.append(res) + self.results.append(res) if isinstance(logs, str): self.log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], @@ -296,5 +294,5 @@ def run(self) -> List[CalcResult]: self._report_calculation_progress_to_user() - self._push_errors_to_database() - return results + self._push_resulting_error_messages_to_database() + return self.results From 4f08f996d32fc78d6ba05c9fa3816b58cb22389b Mon Sep 17 00:00:00 2001 From: chrispijo Date: Fri, 21 Aug 2026 12:30:33 +0200 Subject: [PATCH 10/10] Tijdelijke aanpassing teruggedraaid --- geoprob_pipe/calculations/systems/build_and_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geoprob_pipe/calculations/systems/build_and_run.py b/geoprob_pipe/calculations/systems/build_and_run.py index 1a2146a6..e46354a5 100644 --- a/geoprob_pipe/calculations/systems/build_and_run.py +++ b/geoprob_pipe/calculations/systems/build_and_run.py @@ -221,7 +221,6 @@ def _setup_calculation_progress_variables(self): self.error_rows = [] self.char_len_total: int = str(self.n_calc_totaal).__len__() self.log_rows = [] - self.results: List[CalcResult] = [] def _report_calculation_progress_to_user(self): """ Gedurende het uitvoeren van de berekening koppelt deze method terug wat de progressie is. """ @@ -274,12 +273,13 @@ def run(self) -> List[CalcResult]: pool_size = max(min(math.floor(self.n_calc_totaal / self.chunk_size), self.n_threads), 1) # Multiprocessing setup + results: List[CalcResult] = [] with Pool(processes=pool_size, initializer=_init_worker, initargs=( self.geohydrologisch_model, self.geopackage_filepath, self.to_run_vakken_ids)) as pool: for res, logs, row in pool.imap_unordered(_worker, rows, chunksize=self.chunk_size): if isinstance(res, CalcResult): - self.results.append(res) + results.append(res) if isinstance(logs, str): self.log_rows.append({ "uittredepunt_id": row["uittredepunt_id"], @@ -295,4 +295,4 @@ def run(self) -> List[CalcResult]: self._report_calculation_progress_to_user() self._push_resulting_error_messages_to_database() - return self.results + return results