diff --git a/example_files/Template_PVtool5_0.xlsx b/example_files/Template_PVtool5_0.xlsx deleted file mode 100644 index 386b4b8..0000000 Binary files a/example_files/Template_PVtool5_0.xlsx and /dev/null differ diff --git a/example_files/Template_PVtool5_0_23ZP0747_STOWA.xlsx b/example_files/Template_PVtool5_0_23ZP0747_STOWA.xlsx new file mode 100644 index 0000000..455166c Binary files /dev/null and b/example_files/Template_PVtool5_0_23ZP0747_STOWA.xlsx differ diff --git a/example_files/Template_PVtool5_0_SAFE_2022_PV.xlsx b/example_files/Template_PVtool5_0_SAFE_2022_PV.xlsx new file mode 100644 index 0000000..f579cd0 Binary files /dev/null and b/example_files/Template_PVtool5_0_SAFE_2022_PV.xlsx differ diff --git a/example_files/Template_PVtool5_0_SAFE_2022_PV_handmatig.xlsx b/example_files/Template_PVtool5_0_SAFE_2022_PV_handmatig.xlsx new file mode 100644 index 0000000..2820610 Binary files /dev/null and b/example_files/Template_PVtool5_0_SAFE_2022_PV_handmatig.xlsx differ diff --git a/example_files/Template_PVtool5_0_SAFE_2022_PV_zonder_resultaten.xlsx b/example_files/Template_PVtool5_0_SAFE_2022_PV_zonder_resultaten.xlsx new file mode 100644 index 0000000..f579cd0 Binary files /dev/null and b/example_files/Template_PVtool5_0_SAFE_2022_PV_zonder_resultaten.xlsx differ diff --git a/example_files/Template_PVtool5_0_aanpassingen_handmatig.xlsx b/example_files/Template_PVtool5_0_aanpassingen_handmatig.xlsx deleted file mode 100644 index 2aead13..0000000 Binary files a/example_files/Template_PVtool5_0_aanpassingen_handmatig.xlsx and /dev/null differ diff --git a/example_files/Template_PVtool5_0_backup.xlsx b/example_files/Template_PVtool5_0_backup.xlsx deleted file mode 100644 index dcae356..0000000 Binary files a/example_files/Template_PVtool5_0_backup.xlsx and /dev/null differ diff --git a/example_files/Template_PVtool5_0_output_van_pvtool_import.xlsx b/example_files/Template_PVtool5_0_output_van_pvtool_import.xlsx deleted file mode 100644 index 58f9cbb..0000000 Binary files a/example_files/Template_PVtool5_0_output_van_pvtool_import.xlsx and /dev/null differ diff --git a/example_files/Template_PVtool5_0_zonder_resultaten.xlsx b/example_files/Template_PVtool5_0_zonder_resultaten.xlsx deleted file mode 100644 index cba1275..0000000 Binary files a/example_files/Template_PVtool5_0_zonder_resultaten.xlsx and /dev/null differ diff --git a/example_files/~$SAFE 2022 Proevenverzameling_tool_v4.2n_test_zonder_functies.xlsm b/example_files/~$SAFE 2022 Proevenverzameling_tool_v4.2n_test_zonder_functies.xlsm new file mode 100644 index 0000000..1349cba Binary files /dev/null and b/example_files/~$SAFE 2022 Proevenverzameling_tool_v4.2n_test_zonder_functies.xlsm differ diff --git a/main_nathan.py b/main_nathan.py index e6604d6..c0b99fb 100644 --- a/main_nathan.py +++ b/main_nathan.py @@ -15,6 +15,7 @@ from pv_tool.imports.import_data import Dbase from pv_tool.cphi_analysis.c_phi_analysis import CPhiAnalyse from pv_tool.imports.import_options import * +from typing import Literal def get_repo_root(root_search_dir: Optional[str] = None) -> str: @@ -56,25 +57,42 @@ def get_repo_root(root_search_dir: Optional[str] = None) -> str: return repo.working_tree_dir -def test_database_import(): +def test_database_import(source: Literal['Stowa', 'PV-tool', 'Dbase'], + file_name_import: str, file_name_export: str = 'Template_PVtool5_0.xlsx', short=False, validate=False, export=False): """Test de database import en validatie functionaliteit.""" repo_root = Path(get_repo_root()) - path_to_data = repo_root / "example_files" / "Template_PVtool5_0.xlsx" + path_to_data = repo_root / "example_files" / file_name_import save_test = Path(r"c:\Users\gebraadn0645\ARCADIS\103076457 - STOWA PV Tool - 05 Project execution\Deliverables\2. validatie\Test output") - # Database import - dbase = Dbase() - dbase.import_data(source='Dbase', source_dir=path_to_data) - - # Print unieke verzamelingen - print('\nUnieke verzamelingen:') - for pvnaam in dbase.dbase_df['PV_NAAM'].unique(): - print(pvnaam) - - return dbase - - -def test_cphi_analysis_txt(dbase: Dbase): + if short and not validate: + print("Uitvoeren van korte database import test...") + dbase = Dbase() + dbase.import_dbase_short(source=source, source_dir=path_to_data) + if export: + dbase.export_dbase_to_excel(export_dir = save_test, filename=file_name_export) + return dbase + elif validate and not short: + print("Uitvoeren van database import en validatie test...") + dbase = Dbase() + dbase.import_data(source=source, source_dir=path_to_data) + dbase.validate_data(export_path=save_test) + if export: + dbase.export_dbase_to_excel(export_dir = save_test, filename=file_name_export) + return dbase + elif short and validate: + print("Korte import en validatie kan niet samen worden uitgevoerd. Kies één optie.") + return None + else: + print("Uitvoeren van volledige database import test...") + # Database import + dbase = Dbase() + dbase.import_data(source=source, source_dir=path_to_data) + if export: + dbase.export_dbase_to_excel(export_dir = save_test, filename=file_name_export) + return dbase + + +def test_cphi_analysis_txt(dbase: Dbase, file_name: str = 'Template_PVtool5_0.xlsx'): """ Test een TXT C-phi analyse. @@ -95,12 +113,12 @@ def test_cphi_analysis_txt(dbase: Dbase): # Pas instellingen toe analyse.apply_settings(alpha=0.75) - analyse.apply_parameters(cohesie_kar=0) + analyse.apply_parameters(cohesie_kar=0.0) # Print en exporteer resultaten print('\nResultaten TXT C-phi analyse:') print(analyse.print_short_results()) - analyse.add_results_to_dbase(path=str(save_test)) + analyse.add_results_to_dbase(path=str(save_test), file_name=file_name) # Visualisatie analyse.show_figure() @@ -205,13 +223,19 @@ def test_cphi_analysis_dss_sh(dbase: Dbase): if __name__ == "__main__": # Test database import - dbase = test_database_import() + source = 'Dbase' # Opties: 'Stowa', 'PV-tool', 'Dbase' + import_name = 'Template_PVtool5_0_SAFE_2022_PV_zonder_resultaten.xlsx' + export_name = 'Template_PVtool5_0_SAFE_2022_PV.xlsx' + import_name2 = export_name # TODO eventueel: import naam kan nu niet export naam zijn, dan kan die niet de layout aanpassen. Moet nog worden aangepast in de toekomst. + print("Start van de tests...\n") + dbase = test_database_import(source=source, file_name_import=import_name, file_name_export=export_name, short=True, validate=False, export=True) # Test verschillende analyses print("\nUitvoeren van verschillende test cases...") print("\n1. TXT C-phi analyse test") - test_cphi_analysis_txt(dbase) + file_name = 'Template_PVtool5_0_SAFE_2022_PV.xlsx' + test_cphi_analysis_txt(dbase, file_name=file_name) # print("\n2. DSS C-phi analyse test") # test_cphi_analysis_dss(dbase) diff --git a/pv_tool/cphi_analysis/c_phi_analysis.py b/pv_tool/cphi_analysis/c_phi_analysis.py index a5ac8ee..a6cafbb 100644 --- a/pv_tool/cphi_analysis/c_phi_analysis.py +++ b/pv_tool/cphi_analysis/c_phi_analysis.py @@ -555,7 +555,7 @@ def print_short_results(self): analyse_output_df['cohesie [kPa]'] = [self.c_gem, self.c_kar, self.c_d, self.st_dev_c] return analyse_output_df - def add_results_to_dbase(self, path): + def add_results_to_dbase(self, path, file_name: str = 'Template_PVtool5_0.xlsx'): """ Voegt analyseresultaten toe aan de database export. @@ -566,13 +566,14 @@ def add_results_to_dbase(self, path): ---------- path : str Map locatie waar het Excel-bestand staat of moet komen + file_name : str, optioneel + Naam van het Excel-bestand (standaard 'Template_PVtool5_0.xlsx') Returns ------- DataFrame DataFrame met alle resultaten in het tabblad """ - file_name = 'Template_PVtool5_0.xlsx' file_path = f"{path}/{file_name}" if self.analysis_type in ['TXT_SH', 'DSS_SH']: @@ -584,7 +585,7 @@ def add_results_to_dbase(self, path): with open(file_path, 'r'): pass except FileNotFoundError: - raise FileNotFoundError("Er is geen dbase aanwezig onder de naam Template_PVtool5_0.xlsx") + raise FileNotFoundError(f"Er is geen dbase aanwezig onder de naam {file_name}") expected_columns = [ 'PV_RESULTAAT_ID', 'PVNAAM', 'PV_REK', 'PV_TYPE_PROEF', 'PV_ANALYSE', diff --git a/pv_tool/imports/add_ana_columns.py b/pv_tool/imports/add_ana_columns.py index 258c5e6..39b612f 100644 --- a/pv_tool/imports/add_ana_columns.py +++ b/pv_tool/imports/add_ana_columns.py @@ -15,8 +15,9 @@ def add_columns(self: Dbase): analysis_columns = [ 'ANA_TERREINSPANNING', 'ANA_TXT_MAX_VERTICALE_CONSOLIDATIE_SPANNING', 'ANA_DSS_MAX_CONSOLIDATIE_SPANNING', 'ANA_TXT_CONSOLIDATIE_TYPE_VOORSTEL', - 'ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG', 'ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL', - 'ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG', 'ANA_GRENSSPANNING_PROEF', 'ANA_POP_VELD', + 'ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG', 'ANA_TXT_CONSOLIDATIE_TYPE_REKEN', + 'ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL', 'ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG', + 'ANA_DSS_CONSOLIDATIE_TYPE_REKEN', 'ANA_GRENSSPANNING_PROEF', 'ANA_POP_VELD', 'ANA_POP_VELD_GEMIDDELD', 'ANA_GRENSSPANNING_VOORSTEL', 'ANA_GRENSSPANNING_HANDMATIG', 'ANA_GRENSSPANNING_REKEN', 'OCR_TXT', 'OCR_DSS' ] @@ -31,30 +32,32 @@ def add_columns(self: Dbase): preserved_data = {} for col in preserve_cols: if col in df.columns: - if col.endswith('_HANDMATIG') and 'CONSOLIDATIE_TYPE' in col: - # For consolidation type columns, preserve as string - preserved_data[col] = df[col].astype(str).where(df[col].notna(), None) - elif col == 'ANA_GRENSSPANNING_HANDMATIG': - # For numerical columns, convert to float + if col == 'ANA_GRENSSPANNING_HANDMATIG': + # Ensure numeric type for grensspanning preserved_data[col] = pd.to_numeric(df[col], errors='coerce') + elif 'CONSOLIDATIE_TYPE' in col: + # Ensure string type for consolidation types + preserved_data[col] = df[col].astype(str).where(df[col].notna(), None) else: - preserved_data[col] = df[col].copy() + preserved_data[col] = df[col] # Get non-analysis columns other_columns = [col for col in df.columns if col not in analysis_columns] - # Create new DataFrame with correct order + # Create new DataFrame with non-analysis columns new_df = df[other_columns].copy() - # Add analysis columns with proper types + # Add each analysis column in the specified order for col in analysis_columns: - if col in preserve_cols and col in preserved_data: + if col in preserved_data: + # If we have preserved data for this column, use it new_df[col] = preserved_data[col] else: + # Otherwise initialize with appropriate type if 'CONSOLIDATIE_TYPE' in col: - new_df[col] = None # Will be filled with 'OC' or 'NC' later + new_df[col] = pd.Series(dtype='object') else: - new_df[col] = pd.Series(dtype='float64') # For numerical columns + new_df[col] = pd.Series(dtype='float64') self.dbase_df = new_df @@ -90,7 +93,6 @@ def add_txt_consol_type(self: Dbase): else 'NC', axis=1 ) - def add_dss_consol_type(self: Dbase): """Geeft een voorstel voor het consolidatietype van de DSS-proef. Indien de maximale consolidatiespanning niet meer dan 30% afwijkt van de terreinspanning wordt het consolidatietype OC aangenomen, anders wordt het @@ -101,10 +103,29 @@ def add_dss_consol_type(self: Dbase): else 'NC', axis=1 ) +def add_txt_consol_type_reken(self: Dbase): + """vult de kolom rekenwaarde van consolidatie type: als er een handmatige waarde is ingevuld, wordt deze overgenomen, + anders wordt de voorgestelde waarde overgenomen.""" + def calculate_row(row): + if row['ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG'] and row['ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG'] is not None and not pd.isna(row['ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG']): + return row['ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG'] + else: + return row['ANA_TXT_CONSOLIDATIE_TYPE_VOORSTEL'] + self.dbase_df['ANA_TXT_CONSOLIDATIE_TYPE_REKEN'] = self.dbase_df.apply(calculate_row, axis=1) + +def add_dss_consol_type_reken(self: Dbase): + """vult de kolom rekenwaarde van consolidatie type: als er een handmatige waarde is ingevuld, wordt deze overgenomen, + anders wordt de voorgestelde waarde overgenomen.""" + def calculate_row(row): + if row['ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG'] and row['ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG'] is not None and not pd.isna(row['ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG']): + return row['ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG'] + else: + return row['ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL'] + self.dbase_df['ANA_DSS_CONSOLIDATIE_TYPE_REKEN'] = self.dbase_df.apply(calculate_row, axis=1) def add_grensspanning_proef(self: Dbase): """Deze functie bepaalt de grensspanning.""" - columns = ['CRS_GRENSSPANNING_A', 'SD_ISOTACHE_GRENSSPANNING_A', 'ANA_GRENSSPANNING_HANDMATIG'] + columns = ['CRS_GRENSSPANNING_A', 'SD_ISOTACHE_GRENSSPANNING_A'] grens_values = self.dbase_df[columns].max(axis=1) self.dbase_df['ANA_GRENSSPANNING_PROEF'] = grens_values @@ -125,19 +146,30 @@ def add_grensspanning_proef(self: Dbase): def calc_pop_veld(self): - """Berekend de POP in het veld""" + """Berekent de POP in het veld""" self.dbase_df['ANA_POP_VELD'] = self.dbase_df['ANA_GRENSSPANNING_PROEF'] - self.dbase_df['ANA_TERREINSPANNING'] def calc_pop_average(self): - """Berekend de gemiddelde POP van een monster. Aangenomen wordt dat de POP gelijk blijft in de diepte.""" + """Berekent de gemiddelde POP van een monster. Aangenomen wordt dat de POP gelijk blijft in de diepte.""" self.dbase_df['ANA_POP_VELD_GEMIDDELD'] = self.dbase_df.groupby('BORING_NUMMER')['ANA_POP_VELD'].transform( 'mean') def add_grensspanning_voorstel(self: Dbase): - self.dbase_df['ANA_GRENSSPANNING_VOORSTEL'] = (self.dbase_df['ANA_TERREINSPANNING'] + - self.dbase_df['ANA_POP_VELD_GEMIDDELD']) + """Bepaalt de voorgestelde grensspanning alleen wanneer er geen proefwaarde is. + + Als 'ANA_GRENSSPANNING_PROEF' leeg of NaN is, wordt 'ANA_GRENSSPANNING_VOORSTEL' + gelijk aan 'ANA_TERREINSPANNING' + 'ANA_POP_VELD_GEMIDDELD'. + In alle andere gevallen wordt 'ANA_GRENSSPANNING_VOORSTEL' op None gezet. + """ + mask = self.dbase_df['ANA_GRENSSPANNING_PROEF'].isna() + # Standaard None, alleen vullen waar geen proefwaarde is + self.dbase_df['ANA_GRENSSPANNING_VOORSTEL'] = None + self.dbase_df.loc[mask, 'ANA_GRENSSPANNING_VOORSTEL'] = ( + self.dbase_df.loc[mask, 'ANA_TERREINSPANNING'] + + self.dbase_df.loc[mask, 'ANA_POP_VELD_GEMIDDELD'] + ) def calc_grensspanning_reken(self: Dbase): # klopt @@ -145,12 +177,15 @@ def calc_grensspanning_reken(self: Dbase): # klopt Berekent de rekenwaarde van de grensspanning per rij. """ def calculate_row(row): - if 'ANA_GRENSSPANNING_HANDMATIG' in row and row['ANA_GRENSSPANNING_HANDMATIG'] is not None: + if 'ANA_GRENSSPANNING_HANDMATIG' in row and row['ANA_GRENSSPANNING_HANDMATIG'] is not None and not pd.isna(row['ANA_GRENSSPANNING_HANDMATIG']): return row['ANA_GRENSSPANNING_HANDMATIG'] - elif 'ANA_GRENSSPANNING_VOORSTEL' in row: + elif 'ANA_GRENSSPANNING_VOORSTEL' in row and row['ANA_GRENSSPANNING_VOORSTEL'] is not None and not pd.isna(row['ANA_GRENSSPANNING_VOORSTEL']): return row['ANA_GRENSSPANNING_VOORSTEL'] + elif 'ANA_GRENSSPANNING_PROEF' in row: + return row['ANA_GRENSSPANNING_PROEF'] + # TODO check of dit goed gaat return None - if 'ANA_GRENSSPANNING_HANDMATIG' in self.dbase_df.columns or 'ANA_GRENSSPANNING_VOORSTEL' in self.dbase_df.columns: + if 'ANA_GRENSSPANNING_HANDMATIG' in self.dbase_df.columns or 'ANA_GRENSSPANNING_VOORSTEL' in self.dbase_df.columns or 'ANA_GRENSSPANNING_PROEF' in self.dbase_df.columns: self.dbase_df['ANA_GRENSSPANNING_REKEN'] = self.dbase_df.apply(calculate_row, axis=1) else: self.dbase_df['ANA_GRENSSPANNING_REKEN'] = None @@ -164,10 +199,10 @@ def calculate_row(row): terreinspanning = row['ANA_TERREINSPANNING'] if grensspanning_reken is not None and terreinspanning is not None: - if row['ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG'] == 'OC': - return grensspanning_reken / terreinspanning - elif row['ANA_TXT_CONSOLIDATIE_TYPE_VOORSTEL'] == 'OC': + if row['ANA_TXT_CONSOLIDATIE_TYPE_REKEN'] == 'OC': return grensspanning_reken / terreinspanning + # elif row['ANA_TXT_CONSOLIDATIE_TYPE_VOORSTEL'] == 'OC': + # return grensspanning_reken / terreinspanning else: return 1.0 return None @@ -187,10 +222,10 @@ def calculate_row(row): terreinspanning = row['ANA_TERREINSPANNING'] if grensspanning_reken is not None and terreinspanning is not None: - if row['ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG'] == 'OC': - return grensspanning_reken / terreinspanning - elif row['ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL'] == 'OC': + if row['ANA_DSS_CONSOLIDATIE_TYPE_REKEN'] == 'OC': return grensspanning_reken / terreinspanning + # elif row['ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL'] == 'OC': + # return grensspanning_reken / terreinspanning else: return 1.0 else: diff --git a/pv_tool/imports/create_dbase.py b/pv_tool/imports/create_dbase.py index 2163522..f6daa60 100644 --- a/pv_tool/imports/create_dbase.py +++ b/pv_tool/imports/create_dbase.py @@ -1,6 +1,8 @@ from __future__ import annotations import pandas as pd import numpy as np + +from imports.add_ana_columns import add_txt_consol_type_reken, add_dss_consol_type_reken from pv_tool.imports.globals import (PV_TOOL_DBASE_COLUMNS, CLAS_COLUMNS, CRS_COLUMNS, SD_COLUMNS, DSS_COLUMNS, TXT_COLUMNS) from pv_tool.imports.add_ana_columns import (add_columns, add_terreinspanning, add_txt_max_vert_consol_sp, @@ -57,8 +59,12 @@ def add_ana_columns(self): add_txt_consol_type(self) add_dss_consol_type(self) + # Add rekenwaarde consolidation types + add_txt_consol_type_reken(self) + add_dss_consol_type_reken(self) + # Now that preserved values are in place from add_columns, calculate dependent values - add_grensspanning_proef(self) # Uses preserved ANA_GRENSSPANNING_HANDMATIG + add_grensspanning_proef(self) calc_pop_veld(self) # Depends on grensspanning_proef calc_pop_average(self) add_grensspanning_voorstel(self) diff --git a/pv_tool/imports/import_data.py b/pv_tool/imports/import_data.py index 1987e74..da86b60 100644 --- a/pv_tool/imports/import_data.py +++ b/pv_tool/imports/import_data.py @@ -64,59 +64,82 @@ def validate_data(self, export_path: Path): def export_dbase_to_excel(self, export_dir: Path, filename: str = 'Template_PVtool5_0.xlsx'): """ Exports the Dbase DataFrame to an Excel file, maintaining the correct column order - and preserving specified columns if they exist. + and preserving specified columns from the current DataFrame. """ export_path = export_dir / filename sheet_name = 'Dbase5_0' - preserve_cols = [ - 'ANA_GRENSSPANNING_HANDMATIG', - 'ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG', - 'ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG' - ] # Ensure the export directory exists export_dir.mkdir(parents=True, exist_ok=True) - # Try to preserve columns if the file & sheet exist - preserved_data = {} - if export_path.exists(): - try: - existing_df = read_excel(export_path, sheet_name=sheet_name) - for col in preserve_cols: - if col in existing_df.columns and col in self.dbase_df.columns: - preserved_data[col] = existing_df[col] - except Exception: - pass - - # Restore preserved columns - for col, data in preserved_data.items(): - self.dbase_df[col] = data - - # Ensure correct column order based on PV_TOOL_DBASE_COLUMNS + # Define analysis columns in their correct order + analysis_columns = [ + 'ANA_TERREINSPANNING', 'ANA_TXT_MAX_VERTICALE_CONSOLIDATIE_SPANNING', + 'ANA_DSS_MAX_CONSOLIDATIE_SPANNING', 'ANA_TXT_CONSOLIDATIE_TYPE_VOORSTEL', + 'ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG', 'ANA_TXT_CONSOLIDATIE_TYPE_REKEN', + 'ANA_DSS_CONSOLIDATIE_TYPE_VOORSTEL', 'ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG', + 'ANA_DSS_CONSOLIDATIE_TYPE_REKEN', 'ANA_GRENSSPANNING_PROEF', 'ANA_POP_VELD', + 'ANA_POP_VELD_GEMIDDELD', 'ANA_GRENSSPANNING_VOORSTEL', 'ANA_GRENSSPANNING_HANDMATIG', + 'ANA_GRENSSPANNING_REKEN', 'OCR_TXT', 'OCR_DSS' + ] + + # Get base columns from PV_TOOL_DBASE_COLUMNS from pv_tool.imports.globals import PV_TOOL_DBASE_COLUMNS - ordered_columns = [col for col in PV_TOOL_DBASE_COLUMNS if col in self.dbase_df.columns] - extra_columns = [col for col in self.dbase_df.columns if col not in PV_TOOL_DBASE_COLUMNS] - final_columns = ordered_columns + extra_columns + base_columns = [col for col in PV_TOOL_DBASE_COLUMNS if col in self.dbase_df.columns] + + # Remove any analysis columns that might be in base_columns to prevent duplication + base_columns = [col for col in base_columns if col not in analysis_columns] + + # Get analysis columns that exist in the DataFrame + ana_columns = [col for col in analysis_columns if col in self.dbase_df.columns] + + # Get any remaining columns that aren't in either list, excluding duplicates + used_columns = set(base_columns + ana_columns) + other_columns = [col for col in self.dbase_df.columns if col not in used_columns] + + # Combine all columns in the correct order + final_columns = base_columns + ana_columns + other_columns + + # Create a copy of the DataFrame with reordered columns, ensuring no duplicates + export_df = self.dbase_df[final_columns].copy() - # Reorder columns - self.dbase_df = self.dbase_df[final_columns] + # rond waarden af voor consistentie + cols_to_round = ['ANA_TERREINSPANNING', 'ANA_TXT_MAX_VERTICALE_CONSOLIDATIE_SPANNING', + 'ANA_DSS_MAX_CONSOLIDATIE_SPANNING', 'ANA_GRENSSPANNING_PROEF', + 'ANA_POP_VELD', 'ANA_POP_VELD_GEMIDDELD', 'ANA_GRENSSPANNING_VOORSTEL', + 'ANA_GRENSSPANNING_REKEN', 'OCR_TXT', 'OCR_DSS'] + for col in cols_to_round: + if col in export_df.columns: + export_df[col] = export_df[col].round(2) timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - print(f"Excel sheet Dbase5_0 wordt overschreven met een nieuwe database op {timestamp}") + print(f"Excel sheet Dbase5_0 wordt weggeschreven op {timestamp}") - # Write the DataFrame to Excel + # Write the DataFrame to Excel with improved settings to prevent corruption if export_path.exists(): with ExcelWriter(export_path, engine='openpyxl', mode='a', if_sheet_exists='replace') as writer: - self.dbase_df.to_excel(writer, sheet_name=sheet_name, index=True) + export_df.to_excel( + writer, + sheet_name=sheet_name, + index=True, + engine='openpyxl', + float_format="%.6f" # Use consistent float format + ) else: with ExcelWriter(export_path, engine='openpyxl', mode='w') as writer: - self.dbase_df.to_excel(writer, sheet_name=sheet_name, index=True) + export_df.to_excel( + writer, + sheet_name=sheet_name, + index=True, + engine='openpyxl', + float_format="%.6f" # Use consistent float format + ) print(f"Excel file exported to: {export_path}") # Formatting - num_columns = self.dbase_df.shape[1] - num_rows = self.dbase_df.shape[0] + num_columns = export_df.shape[1] + num_rows = export_df.shape[0] format_excel_sheet( file_path=str(export_path), sheet_name='Dbase5_0', diff --git a/pv_tool/imports/import_options.py b/pv_tool/imports/import_options.py index 421ab77..a46144d 100644 --- a/pv_tool/imports/import_options.py +++ b/pv_tool/imports/import_options.py @@ -15,11 +15,29 @@ def import_dbase(self: Dbase, dbase_dir: Path): def import_pv_tool(self: Dbase, pv_dir: Path): """Importeert data uit de oude pv-tool (Excel-versie).""" + preserve_cols = [ + 'ANA_GRENSSPANNING_HANDMATIG', + 'ANA_TXT_CONSOLIDATIE_TYPE_HANDMATIG', + 'ANA_DSS_CONSOLIDATIE_TYPE_HANDMATIG' + ] + + # Read the PV-tool file pv = pd.read_excel(pv_dir, skiprows=47, sheet_name='Dbase2') pv = pv.dropna(subset=['ALG__BORING_MONSTERNR_ID']) + + # Create the ID column pv[['ALG__REGEL', 'BORING_NUMMER', 'MONSTER_ID']] = pv[['ALG__REGEL', 'BORING_NUMMER', 'MONSTER_ID']].fillna( '').astype(str) pv['ALG__BORING_MONSTERNR_ID'] = pv[['ALG__REGEL', 'BORING_NUMMER', 'MONSTER_ID']].apply('_'.join, axis=1) + + # Ensure preserved columns maintain their data types + for col in preserve_cols: + if col in pv.columns: + if col.endswith('_HANDMATIG') and 'CONSOLIDATIE_TYPE' in col: + pv[col] = pv[col].astype(str) + elif col == 'ANA_GRENSSPANNING_HANDMATIG': + pv[col] = pd.to_numeric(pv[col], errors='coerce') + pv = pv.set_index('ALG__BORING_MONSTERNR_ID') self.pv_tool = pv return self.pv_tool