From f022dadf0ea38e448995d022257425bbe21095ae Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Fri, 15 May 2026 16:22:30 +0200 Subject: [PATCH 01/11] feat (uploadtasks): update and delete on succesfull tasks --- api/bro_import/object_import.py | 6 + api/signals.py | 28 +- api/tests/fixtures.py | 44 +++ api/tests/test_signals.py | 442 ++++++++++++++++++++++- api/utils.py | 605 +++++++++++++++++++++++++++++++- uv.lock | 2 +- 6 files changed, 1108 insertions(+), 19 deletions(-) diff --git a/api/bro_import/object_import.py b/api/bro_import/object_import.py index c517f8b..683928a 100644 --- a/api/bro_import/object_import.py +++ b/api/bro_import/object_import.py @@ -48,6 +48,7 @@ def check_dates( last_correction_date: str | None, last_addition_date: str | None, registration_completion_time: str | None, + object_registration_time: str | None, ) -> bool: """Check if any of the date fields is more recent than the last_import_date""" dates = [ @@ -56,6 +57,7 @@ def check_dates( last_correction_date, last_addition_date, registration_completion_time, + object_registration_time, ] if d is not None ] @@ -170,12 +172,16 @@ def should_import(self) -> bool: registration_completion_time = properties.get( "registration_completion_time", None ) # ISO String or None + object_registration_time = properties.get( + "object_registration_time", None + ) # ISO String or None if check_dates( last_import_date, last_correction_date, last_addition_date, registration_completion_time, + object_registration_time, ): logger.info( f"Data is more recent in PDOK for {self.bro_domain} and ID {self.bro_id} - Should import - dates: {last_correction_date}, {last_addition_date}, {registration_completion_time}" diff --git a/api/signals.py b/api/signals.py index e529bfa..c291cfc 100644 --- a/api/signals.py +++ b/api/signals.py @@ -7,7 +7,7 @@ from . import tasks from .models import ImportTask, InviteUser, UploadTask, UserProfile -from .utils import create_objects +from .utils import create_objects, delete_objects, update_objects logger = logging.getLogger(__name__) @@ -117,7 +117,7 @@ def post_save_upload_task(sender, instance: UploadTask, created, **kwargs): if ( instance.status == "COMPLETED" - and instance.request_type == "registration" + and instance.request_type in ["registration", "insert"] and instance.data_owner ): create_objects( @@ -127,6 +127,30 @@ def post_save_upload_task(sender, instance: UploadTask, created, **kwargs): instance.sourcedocument_data, instance.data_owner, ) + elif ( + instance.status == "COMPLETED" + and instance.request_type in ["replace", "move"] + and instance.data_owner + ): + update_objects( + instance.registration_type, + instance.bro_id, + instance.metadata, + instance.sourcedocument_data, + instance.data_owner, + ) + elif ( + instance.status == "COMPLETED" + and instance.request_type == "delete" + and instance.data_owner + ): + delete_objects( + instance.registration_type, + instance.bro_id, + instance.metadata, + instance.sourcedocument_data, + instance.data_owner, + ) @receiver(post_save, sender=ImportTask) diff --git a/api/tests/fixtures.py b/api/tests/fixtures.py index 68109f6..eb1cd01 100644 --- a/api/tests/fixtures.py +++ b/api/tests/fixtures.py @@ -5,6 +5,7 @@ from django.contrib.auth.models import User from api import models as api_models +from frd import models as frd_models from gar import models as gar_models from gld import models as gld_models from gmn import models as gmn_models @@ -205,3 +206,46 @@ def bulk_upload(organisation): log="", progress=20.0, ) + + +@pytest.fixture +def frd(organisation): + return frd_models.FRD.objects.create( + data_owner=organisation, + bro_id="FRD123456789", + gmw_bro_id="GMW123456789", + tube_number="1", + quality_regime="IMBRO/A", + ) + + +@pytest.fixture +def measurement_configuration(organisation, frd): + return frd_models.MeasurementConfiguration.objects.create( + frd=frd, + data_owner=organisation, + measurement_configuration_id="MC001", + ) + + +@pytest.fixture +def geo_electric_measurement(organisation, frd): + return frd_models.GeoElectricMeasurement.objects.create( + frd=frd, + data_owner=organisation, + measurement_date=datetime.date(2024, 1, 1), + ) + + +@pytest.fixture +def observation(organisation, gld): + return gld_models.Observation.objects.create( + gld=gld, + data_owner=organisation, + observation_id="_obs1", + begin_position=datetime.date(2024, 1, 1), + end_position=datetime.date(2024, 12, 31), + result_time=datetime.datetime(2024, 12, 31, 12, 0, 0, tzinfo=pytz.UTC), + observation_type="reguliereMeting", + investigator_kvk="12345678", + ) diff --git a/api/tests/test_signals.py b/api/tests/test_signals.py index 9ddd378..3842f89 100644 --- a/api/tests/test_signals.py +++ b/api/tests/test_signals.py @@ -2,10 +2,22 @@ import pytest from api.models import UploadTask -from api.tests.fixtures import gld, gmn, gmw, organisation # noqa: F401 -from frd.models import FRD +from api.tests.fixtures import ( # noqa: F401 + frd, + geo_electric_measurement, + gld, + gmn, + gmw, + intermediate_event, + measurement_configuration, + measuringpoint, + observation, + organisation, + tube, +) +from frd.models import FRD, GeoElectricMeasurement, MeasurementConfiguration from gld.models import GLD, Observation # Example model to check side effects -from gmn.models import GMN, Measuringpoint +from gmn.models import GMN, IntermediateEvent, Measuringpoint from gmw.models import GMW, Event, MonitoringTube # Example model to check side effects from ..models import Organisation @@ -283,16 +295,16 @@ def test_post_save_uploadtask_triggers_create_objects_gld_observation( assert GLD.objects.filter(bro_id=gld.bro_id).exists() assert Observation.objects.filter(gld=gld).exists() - observation = Observation.objects.get(gld=gld) + observation_object = Observation.objects.get(gld=gld) - assert observation.observation_id == "_1ae0ebbf-4845-4237-aa68-f6a0c3b7e6bc" - assert observation.begin_position.isoformat() == "2021-08-23" - assert observation.end_position.isoformat() == "2025-05-08" - assert observation.result_time.isoformat() == "2025-05-08T08:16:40+00:00" - assert observation.investigator_kvk == "08213234" - assert observation.observation_type == "controlemeting" - assert observation.process_reference == "vitensMeetprotocolGrondwater" - assert observation.measurement_instrument_type == "elektronischPeilklokje" + assert observation_object.observation_id == "_1ae0ebbf-4845-4237-aa68-f6a0c3b7e6bc" + assert observation_object.begin_position.isoformat() == "2021-08-23" + assert observation_object.end_position.isoformat() == "2025-05-08" + assert observation_object.result_time.isoformat() == "2025-05-08T08:16:40+00:00" + assert observation_object.investigator_kvk == "08213234" + assert observation_object.observation_type == "controlemeting" + assert observation_object.process_reference == "vitensMeetprotocolGrondwater" + assert observation_object.measurement_instrument_type == "elektronischPeilklokje" @pytest.mark.django_db @@ -371,8 +383,8 @@ def test_post_save_uploadtask_triggers_create_objects_frd(organisation: Organisa # For example, if create_objects makes DB entries: assert FRD.objects.filter(bro_id="BRO123").exists() - frd = FRD.objects.get(bro_id="BRO123") - assert frd.internal_id == "test_upload" + frd_object = FRD.objects.get(bro_id="BRO123") + assert frd_object.internal_id == "test_upload" GMW_EVENT_TYPES = [ @@ -595,3 +607,405 @@ def test_pre_save_uploadtask_triggers_insert( assert task.request_type == "insert" assert task.bro_errors == "" assert task.status == "PROCESSING" + + +# ── Update tests ────────────────────────────────────────────────────────────── + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmw( + organisation: Organisation, # noqa: F811 + gmw: GMW, # noqa: F811 +): + """Saving a replace UploadTask for GMW_Construction updates GMW fields and upserts MonitoringTubes.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMW", + registration_type="GMW_Construction", + request_type="replace", + status="COMPLETED", + bro_id=gmw.bro_id, + metadata={ + "qualityRegime": "IMBRO", + "deliveryAccountableParty": "99999999", + }, + sourcedocument_data={ + "nitgCode": "B09A0001", + "owner": "11111111", + "deliveredLocation": "250000 450000", + "objectIdAccountableParty": "updated_id", + "monitoringTubes": [ + { + "tubeNumber": 1, + "tubeType": "standaardbuis", + "tubeStatus": "gebruiksklaar", + "tubeMaterial": "pvc", + "tubeTopPosition": "1.0", + "plainTubePartLength": "2.0", + "screenLength": "1.0", + "sedimentSumpPresent": "nee", + "artesianWellCapPresent": "nee", + "variableDiameter": "nee", + "numberOfGeoOhmCables": 0, + "geoOhmCables": [], + } + ], + }, + ) + + gmw.refresh_from_db() + assert gmw.nitg_code == "B09A0001" + assert gmw.owner == "11111111" + assert gmw.quality_regime == "IMBRO" + assert gmw.internal_id == "updated_id" + assert gmw.standardized_location is not None + + assert MonitoringTube.objects.filter(gmw=gmw, tube_number=1).exists() + tube_obj = MonitoringTube.objects.get(gmw=gmw, tube_number=1) + assert tube_obj.tube_material == "pvc" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmw_event( + organisation: Organisation, # noqa: F811 + gmw: GMW, # noqa: F811 +): + """Replacing a GMW event updates its metadata and sourcedocument_data.""" + Event.objects.create( + gmw=gmw, + event_name="GMW_Positions", + event_date="2025-01-10", + data_owner=organisation, + metadata={"qualityRegime": "IMBRO/A"}, + sourcedocument_data={"eventDate": "2025-01-10"}, + ) + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMW", + registration_type="GMW_Positions", + request_type="replace", + status="COMPLETED", + bro_id=gmw.bro_id, + metadata={"qualityRegime": "IMBRO", "deliveryAccountableParty": "88888888"}, + sourcedocument_data={ + "eventDate": "2025-01-10", + "wellHeadProtector": "kokerPlastic", + }, + ) + + event = Event.objects.get(gmw=gmw, event_name="GMW_Positions") + assert event.metadata["qualityRegime"] == "IMBRO" + assert event.sourcedocument_data["wellHeadProtector"] == "kokerPlastic" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmw_event_move( + organisation: Organisation, # noqa: F811 + gmw: GMW, # noqa: F811 +): + """Moving a GMW event (dateToBeCorrected) updates its event_date.""" + Event.objects.create( + gmw=gmw, + event_name="GMW_Owner", + event_date="2025-01-10", + data_owner=organisation, + metadata={"qualityRegime": "IMBRO/A"}, + sourcedocument_data={"eventDate": "2025-01-10"}, + ) + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMW", + registration_type="GMW_Owner", + request_type="move", + status="COMPLETED", + bro_id=gmw.bro_id, + metadata={"qualityRegime": "IMBRO/A"}, + sourcedocument_data={ + "dateToBeCorrected": "2025-01-10", + "eventDate": "2025-03-01", + }, + ) + + event = Event.objects.get(gmw=gmw, event_name="GMW_Owner") + assert str(event.event_date) == "2025-03-01" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmw_removal( + organisation: Organisation, # noqa: F811 + gmw: GMW, # noqa: F811 +): + """Replacing a GMW_Removal event updates its metadata and sourcedocument_data.""" + Event.objects.create( + gmw=gmw, + event_name="GMW_Removal", + event_date="2025-06-01", + data_owner=organisation, + metadata={"qualityRegime": "IMBRO/A"}, + sourcedocument_data={"eventDate": "2025-06-01", "removalReason": "old"}, + ) + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMW", + registration_type="GMW_Removal", + request_type="replace", + status="COMPLETED", + bro_id=gmw.bro_id, + metadata={"qualityRegime": "IMBRO"}, + sourcedocument_data={"eventDate": "2025-06-01", "removalReason": "updated"}, + ) + + event = Event.objects.get(gmw=gmw, event_name="GMW_Removal") + assert event.sourcedocument_data["removalReason"] == "updated" + assert event.metadata["qualityRegime"] == "IMBRO" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gld( + organisation: Organisation, # noqa: F811 + gld: GLD, # noqa: F811 +): + """Replacing a GLD_StartRegistration updates GLD fields.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GLD", + registration_type="GLD_StartRegistration", + request_type="replace", + status="COMPLETED", + bro_id=gld.bro_id, + metadata={ + "qualityRegime": "IMBRO", + "deliveryAccountableParty": "77777777", + }, + sourcedocument_data={ + "objectIdAccountableParty": "updated_gld_id", + "gmwBroId": "GMW999999999", + "tubeNumber": 3, + }, + ) + + gld.refresh_from_db() + assert gld.internal_id == "updated_gld_id" + assert gld.gmw_bro_id == "GMW999999999" + assert str(gld.tube_number) == "3" + assert gld.quality_regime == "IMBRO" + assert gld.delivery_accountable_party == "77777777" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gld_observation( + organisation: Organisation, # noqa: F811 + gld: GLD, # noqa: F811 + observation, # noqa: F811 +): + """Replacing a GLD_Addition updates the matching Observation fields.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GLD", + registration_type="GLD_Addition", + request_type="replace", + status="COMPLETED", + bro_id=gld.bro_id, + metadata={"qualityRegime": "IMBRO"}, + sourcedocument_data={ + "dateToBeCorrected": "2024-01-01", + "beginPosition": "2024-01-01", + "endPosition": "2024-12-31", + "observationType": "controlemeting", + "investigatorKvk": "99999999", + "processReference": "updatedProcess", + "evaluationProcedure": "updatedProcedure", + "measurementInstrumentType": "elektronischPeilklokje", + }, + ) + + observation.refresh_from_db() + assert observation.observation_type == "controlemeting" + assert observation.investigator_kvk == "99999999" + assert observation.process_reference == "updatedProcess" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmn( + organisation: Organisation, # noqa: F811 + gmn: GMN, # noqa: F811 +): + """Replacing a GMN_StartRegistration updates GMN fields.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMN", + registration_type="GMN_StartRegistration", + request_type="replace", + status="COMPLETED", + bro_id=gmn.bro_id, + metadata={"qualityRegime": "IMBRO"}, + sourcedocument_data={ + "name": "Updated GMN name", + "monitoringPurpose": "strategischBeheerKwaliteitRegionaal", + "groundwaterAspect": "kwantiteit", + "deliveryContext": "waterwetPeilbeheer", + "startDateMonitoring": "2025-01-01", + }, + ) + + gmn.refresh_from_db() + assert gmn.name == "Updated GMN name" + assert gmn.monitoring_purpose == "strategischBeheerKwaliteitRegionaal" + assert gmn.groundwater_aspect == "kwantiteit" + assert gmn.quality_regime == "IMBRO" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_gmn_measuringpoint( + organisation: Organisation, # noqa: F811 + gmn: GMN, # noqa: F811 + measuringpoint, # noqa: F811 +): + """Replacing a GMN_MeasuringPoint updates the matching Measuringpoint fields.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMN", + registration_type="GMN_MeasuringPoint", + request_type="replace", + status="COMPLETED", + bro_id=gmn.bro_id, + metadata={"qualityRegime": "IMBRO/A"}, + sourcedocument_data={ + "measuringPointCode": "MP123456", + "broId": "GMW999999999", + "tubeNumber": 2, + "eventDate": "2025-06-01", + }, + ) + + measuringpoint.refresh_from_db() + assert measuringpoint.gmw_bro_id == "GMW999999999" + assert str(measuringpoint.tube_number) == "2" + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_update_objects_frd( + organisation: Organisation, # noqa: F811 + frd, # noqa: F811 +): + """Replacing a FRD_StartRegistration updates FRD fields.""" + UploadTask.objects.create( + data_owner=organisation, + bro_domain="FRD", + registration_type="FRD_StartRegistration", + request_type="replace", + status="COMPLETED", + bro_id=frd.bro_id, + metadata={"qualityRegime": "IMBRO"}, + sourcedocument_data={ + "objectIdAccountableParty": "updated_frd_id", + "gmwBroId": "GMW888888888", + "tubeNumber": 4, + "deliveryAccountableParty": "55555555", + }, + ) + + frd.refresh_from_db() + assert frd.internal_id == "updated_frd_id" + assert frd.gmw_bro_id == "GMW888888888" + assert str(frd.tube_number) == "4" + assert frd.quality_regime == "IMBRO" + assert frd.delivery_accountable_party == "55555555" + + +# ── Delete tests ────────────────────────────────────────────────────────────── + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_delete_objects_gld_closure( + organisation: Organisation, # noqa: F811 + gld: GLD, # noqa: F811 + observation, # noqa: F811 +): + """A delete UploadTask for GLD_Closure removes the matching Observation.""" + assert Observation.objects.filter(gld=gld).count() == 1 + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GLD", + registration_type="GLD_Closure", + request_type="delete", + status="COMPLETED", + bro_id=gld.bro_id, + metadata={}, + sourcedocument_data={"dateToBeCorrected": "2024-01-01"}, + ) + + assert Observation.objects.filter(gld=gld).count() == 0 + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_delete_objects_gmn_closure( + organisation: Organisation, # noqa: F811 + gmn: GMN, # noqa: F811 + intermediate_event, # noqa: F811 +): + """A delete UploadTask for GMN_Closure removes the matching IntermediateEvent.""" + assert IntermediateEvent.objects.filter(gmn=gmn).count() == 1 + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="GMN", + registration_type="GMN_Closure", + request_type="delete", + status="COMPLETED", + bro_id=gmn.bro_id, + metadata={}, + sourcedocument_data={"dateToBeCorrected": "2024-01-01"}, + ) + + assert IntermediateEvent.objects.filter(gmn=gmn).count() == 0 + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_delete_objects_frd_gem_measurement_configuration( + organisation: Organisation, # noqa: F811 + frd, # noqa: F811 + measurement_configuration, # noqa: F811 +): + """A delete UploadTask for FRD_GEM_MeasurementConfiguration removes the matching MeasurementConfiguration.""" + assert MeasurementConfiguration.objects.filter(frd=frd).count() == 1 + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="FRD", + registration_type="FRD_GEM_MeasurementConfiguration", + request_type="delete", + status="COMPLETED", + bro_id=frd.bro_id, + metadata={}, + sourcedocument_data={"measurementConfigurationId": "MC001"}, + ) + + assert MeasurementConfiguration.objects.filter(frd=frd).count() == 0 + + +@pytest.mark.django_db +def test_post_save_uploadtask_triggers_delete_objects_frd_gem_measurement( + organisation: Organisation, # noqa: F811 + frd, # noqa: F811 + geo_electric_measurement, # noqa: F811 +): + """A delete UploadTask for FRD_GEM_Measurement removes the matching GeoElectricMeasurement.""" + assert GeoElectricMeasurement.objects.filter(frd=frd).count() == 1 + + UploadTask.objects.create( + data_owner=organisation, + bro_domain="FRD", + registration_type="FRD_GEM_Measurement", + request_type="delete", + status="COMPLETED", + bro_id=frd.bro_id, + metadata={}, + sourcedocument_data={"dateToBeCorrected": "2024-01-01"}, + ) + + assert GeoElectricMeasurement.objects.filter(frd=frd).count() == 0 diff --git a/api/utils.py b/api/utils.py index 88bc938..aa1709b 100644 --- a/api/utils.py +++ b/api/utils.py @@ -4,9 +4,9 @@ from pyproj import Transformer -from frd.models import FRD +from frd.models import FRD, GeoElectricMeasurement, MeasurementConfiguration from gld.models import GLD, Observation -from gmn.models import GMN, Measuringpoint +from gmn.models import GMN, IntermediateEvent, Measuringpoint from gmw.models import GMW, Event, MonitoringTube logger = logging.getLogger(__name__) @@ -390,6 +390,565 @@ def create_frd( } +# ── Update functions ────────────────────────────────────────────────────────── + + +def update_gmw( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "well_construction_date": "wellConstructionDate", + "delivery_context": "deliveryContext", + "construction_standard": "constructionStandard", + "initial_function": "initialFunction", + "ground_level_stable": "groundLevelStable", + "well_stability": "wellStability", + "nitg_code": "nitgCode", + "well_code": "wellCode", + "owner": "owner", + "well_head_protector": "wellHeadProtector", + "delivered_location": "deliveredLocation", + "horizontal_positioning_method": "horizontalPositioningMethod", + "local_vertical_reference_point": "localVerticalReferencePoint", + "offset": "offset", + "vertical_datum": "verticalDatum", + "ground_level_position": "groundLevelPosition", + "ground_level_positioning_method": "groundLevelPositioningMethod", + } + meta_field_map = { + "delivery_accountable_party": "deliveryAccountableParty", + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + if "deliveredLocation" in sourcedocument_data: + delivered_location = sourcedocument_data.get("deliveredLocation", "0 0").split( + " " + ) + if len(delivered_location) == 2: + rd_x, rd_y = map(float, delivered_location) + lon, lat = transformer.transform(rd_x, rd_y) + updates["standardized_location"] = f"{lat} {lon}" + + for field, value in updates.items(): + setattr(gmw, field, value) + if updates: + gmw.save(update_fields=list(updates.keys())) + logger.info(f"Successfully updated {gmw}.") + + monitoring_tubes_data = sourcedocument_data.get("monitoringTubes", []) + for tube in monitoring_tubes_data: + tube_number = tube.get("tubeNumber", 1) + + MonitoringTube.objects.update_or_create( + gmw=gmw, + tube_number=tube_number, + data_owner=data_owner, + defaults={ + "tube_type": tube.get("tubeType"), + "artesian_well_cap_present": tube.get("artesianWellCapPresent"), + "sediment_sump_present": tube.get("sedimentSumpPresent"), + "sediment_sump_length": tube.get("sedimentSumpLength"), + "number_of_geo_ohm_cables": tube.get("numberOfGeoOhmCables", 0), + "geo_ohm_cables": tube.get("geoOhmCables", []), + "tube_top_diameter": tube.get("tubeTopDiameter"), + "variable_diameter": tube.get("variableDiameter"), + "tube_status": tube.get("tubeStatus"), + "tube_top_position": tube.get("tubeTopPosition"), + "tube_top_positioning_method": tube.get("tubeTopPositioningMethod"), + "tube_part_inserted": tube.get("tubePartInserted"), + "tube_in_use": tube.get("tubeInUse"), + "tube_packing_material": tube.get("tubePackingMaterial"), + "tube_material": tube.get("tubeMaterial"), + "glue": tube.get("glue"), + "screen_length": tube.get("screenLength"), + "screen_protection": tube.get("screenProtection"), + "sock_material": tube.get("sockMaterial"), + "plain_tube_part_length": tube.get("plainTubePartLength"), + }, + ) + + +def update_gmw_event( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + request_type = "move" if date_to_correct else "replace" + + event_qs = Event.objects.filter( + gmw=gmw, event_name=event_type, data_owner=data_owner + ) + if request_type == "move": + event_qs = event_qs.filter(event_date=date_to_correct) + else: + event_qs = event_qs.filter(event_date=sourcedocument_data.get("eventDate")) + + if event_qs.count() != 1: + logger.warning( + f"Expected to find exactly 1 event for bro_id={bro_id}, event_type={event_type}, date={date_to_correct}, but found {event_qs.count()}. Skipping update." + ) + return + + event = event_qs.first() + if not event: + logger.info( + f"Event {event_type} not found for bro_id={bro_id}, date={date_to_correct}. Creating new event." + ) + + ## FUTURE: Make sure that the sourcedocument and metadata are formatted correctly + Event.objects.create( + gmw=gmw, + event_name=event_type, + event_date=sourcedocument_data.get("eventDate"), + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + return + + update_fields = ["metadata", "sourcedocument_data"] + event.metadata = metadata + event.sourcedocument_data = sourcedocument_data + new_event_date = sourcedocument_data.get("eventDate") + if new_event_date: + event.event_date = new_event_date + update_fields.append("event_date") + event.save(update_fields=update_fields) + + +def update_gmw_removal( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + event_qs = Event.objects.filter( + gmw=gmw, event_name="GMW_Removal", data_owner=data_owner + ) + if date_to_correct: + event_qs = event_qs.filter(event_date=date_to_correct) + + event = event_qs.order_by("-event_date").first() + if not event: + logger.info( + f"GMW_Removal event not found for bro_id={bro_id}, date={date_to_correct}." + ) + return + + update_fields = ["metadata", "sourcedocument_data"] + event.metadata = metadata + event.sourcedocument_data = sourcedocument_data + new_event_date = sourcedocument_data.get("eventDate") + if new_event_date: + event.event_date = new_event_date + update_fields.append("event_date") + event.save(update_fields=update_fields) + + +def update_gld( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "linked_gmns": "linkedGmns", + "gmw_bro_id": "gmwBroId", + "tube_number": "tubeNumber", + } + meta_field_map = { + "delivery_accountable_party": "deliveryAccountableParty", + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(gld, field, value) + if updates: + gld.save(update_fields=list(updates.keys())) + + +def update_gld_observation( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) + if date_to_correct: + obs_qs = obs_qs.filter(begin_position=date_to_correct) + + observation = obs_qs.order_by("-begin_position").first() + if not observation: + logger.info( + f"Observation not found for bro_id={bro_id}, date={date_to_correct}." + ) + return + + source_field_map = { + "begin_position": "beginPosition", + "end_position": "endPosition", + "result_time": "resultTime", + "validation_status": "validationStatus", + "investigator_kvk": "investigatorKvk", + "observation_type": "observationType", + "process_reference": "processReference", + "air_pressure_compensation_type": "airPressureCompensationType", + "evaluation_procedure": "evaluationProcedure", + "measurement_instrument_type": "measurementInstrumentType", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + for field, value in updates.items(): + setattr(observation, field, value) + if updates: + observation.save(update_fields=list(updates.keys())) + + +def update_gmn( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "name": "name", + "delivery_context": "deliveryContext", + "monitoring_purpose": "monitoringPurpose", + "groundwater_aspect": "groundwaterAspect", + "start_date_monitoring": "startDateMonitoring", + } + meta_field_map = { + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(gmn, field, value) + if updates: + gmn.save(update_fields=list(updates.keys())) + + +def update_gmn_measuringpoint( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + mp_qs = Measuringpoint.objects.filter( + gmn=gmn, + measuringpoint_code=sourcedocument_data.get("measuringPointCode"), + data_owner=data_owner, + event_type=event_type, + ) + if date_to_correct: + mp_qs = mp_qs.filter(tube_start_date=date_to_correct) + + measuringpoint = mp_qs.order_by("-tube_start_date").first() + if not measuringpoint: + logger.info( + f"Measuringpoint not found for bro_id={bro_id}, " + f"code={sourcedocument_data.get('measuringPointCode')}, date={date_to_correct}." + ) + return + + source_field_map = { + "gmw_bro_id": "broId", + "tube_number": "tubeNumber", + "tube_start_date": "eventDate", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + for field, value in updates.items(): + setattr(measuringpoint, field, value) + if updates: + measuringpoint.save(update_fields=list(updates.keys())) + + +def update_frd( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "delivery_accountable_party": "deliveryAccountableParty", + "gmw_bro_id": "gmwBroId", + "tube_number": "tubeNumber", + } + meta_field_map = { + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(frd, field, value) + if updates: + frd.save(update_fields=list(updates.keys())) + + +UPDATE_FUNCTION_MAPPING = { + "GMW_Construction": update_gmw, + "GMW_Positions": partial(update_gmw_event, event_type="GMW_Positions"), + "GMW_PositionsMeasuring": partial( + update_gmw_event, event_type="GMW_PositionsMeasuring" + ), + "GMW_WellHeadProtector": partial( + update_gmw_event, event_type="GMW_WellHeadProtector" + ), + "GMW_Owner": partial(update_gmw_event, event_type="GMW_Owner"), + "GMW_Shift": partial(update_gmw_event, event_type="GMW_Shift"), + "GMW_GroundLevel": partial(update_gmw_event, event_type="GMW_GroundLevel"), + "GMW_GroundLevelMeasuring": partial( + update_gmw_event, event_type="GMW_GroundLevelMeasuring" + ), + "GMW_Insertion": partial(update_gmw_event, event_type="GMW_Insertion"), + "GMW_TubeStatus": partial(update_gmw_event, event_type="GMW_TubeStatus"), + "GMW_Lengthening": partial(update_gmw_event, event_type="GMW_Lengthening"), + "GMW_Shortening": partial(update_gmw_event, event_type="GMW_Shortening"), + "GMW_ElectrodeStatus": partial(update_gmw_event, event_type="GMW_ElectrodeStatus"), + "GMW_Maintainer": partial(update_gmw_event, event_type="GMW_Maintainer"), + "GMW_Removal": update_gmw_removal, + "GLD_StartRegistration": update_gld, + "GLD_Addition": update_gld_observation, + "GMN_StartRegistration": update_gmn, + "GMN_MeasuringPoint": partial( + update_gmn_measuringpoint, event_type="GMN_MeasuringPoint" + ), + "GMN_MeasuringPointEndDate": partial( + update_gmn_measuringpoint, event_type="GMN_MeasuringPointEndDate" + ), + "GMN_TubeReference": partial( + update_gmn_measuringpoint, event_type="GMN_TubeReference" + ), + "FRD_StartRegistration": update_frd, +} + + +# ── Delete functions ────────────────────────────────────────────────────────── + + +def delete_gld_observation( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) + if date_to_correct: + obs_qs = obs_qs.filter(begin_position=date_to_correct) + + deleted_count, _ = obs_qs.delete() + logger.info( + f"Deleted {deleted_count} observation(s) for bro_id={bro_id}, date={date_to_correct}." + ) + + +def delete_gmn_intermediate_event( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + event_qs = IntermediateEvent.objects.filter(gmn=gmn, data_owner=data_owner) + if date_to_correct: + event_qs = event_qs.filter(event_date=date_to_correct) + + deleted_count, _ = event_qs.delete() + logger.info( + f"Deleted {deleted_count} intermediate event(s) for bro_id={bro_id}, date={date_to_correct}." + ) + + +def delete_frd_measurement_configuration( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + config_id = sourcedocument_data.get("measurementConfigurationId") + config_qs = MeasurementConfiguration.objects.filter(frd=frd, data_owner=data_owner) + if config_id: + config_qs = config_qs.filter(measurement_configuration_id=config_id) + + deleted_count, _ = config_qs.delete() + logger.info( + f"Deleted {deleted_count} measurement configuration(s) for bro_id={bro_id}." + ) + + +def delete_frd_measurement( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + measurement_qs = GeoElectricMeasurement.objects.filter( + frd=frd, data_owner=data_owner + ) + if date_to_correct: + measurement_qs = measurement_qs.filter(measurement_date=date_to_correct) + + deleted_count, _ = measurement_qs.delete() + logger.info( + f"Deleted {deleted_count} geo-electric measurement(s) for bro_id={bro_id}, date={date_to_correct}." + ) + + +DELETE_FUNCTION_MAPPING = { + "GLD_Closure": delete_gld_observation, + "GMN_Closure": delete_gmn_intermediate_event, + "FRD_GEM_MeasurementConfiguration": delete_frd_measurement_configuration, + "FRD_GEM_Measurement": delete_frd_measurement, + "FRD_EMM_InstrumentConfiguration": delete_frd_measurement_configuration, + "FRD_EMM_Measurement": delete_frd_measurement, +} + + def create_objects( registration_type: str, bro_id: str, @@ -411,6 +970,48 @@ def create_objects( return +def update_objects( + registration_type: str, + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + UPDATE_FUNCTION_MAPPING[registration_type]( + bro_id=bro_id, + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + except KeyError: + logger.info( + f"Unable to update as there is no function mapped for registration type: {registration_type}." + ) + return + + +def delete_objects( + registration_type: str, + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + DELETE_FUNCTION_MAPPING[registration_type]( + bro_id=bro_id, + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + except KeyError: + logger.info( + f"Unable to delete as there is no function mapped for registration type: {registration_type}." + ) + return + + def empty_strings_to_none(d: dict) -> dict: for key, value in d.items(): if isinstance(value, str) and value.strip() == "": diff --git a/uv.lock b/uv.lock index a5a18c2..7b21092 100644 --- a/uv.lock +++ b/uv.lock @@ -99,7 +99,7 @@ wheels = [ [[package]] name = "brostar-api" -version = "1.70.dev0" +version = "1.71.dev0" source = { editable = "." } dependencies = [ { name = "celery", extra = ["redis"] }, From 068f2a79eb5e282da173118ae83be35eeb8f9fb7 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Tue, 19 May 2026 10:42:48 +0200 Subject: [PATCH 02/11] add removal as intermediate event, better checks for update/delete --- api/bro_import/object_import.py | 66 ++++++++++++++++++++++++--------- api/utils.py | 5 ++- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/api/bro_import/object_import.py b/api/bro_import/object_import.py index 683928a..f350d8d 100644 --- a/api/bro_import/object_import.py +++ b/api/bro_import/object_import.py @@ -370,6 +370,9 @@ def _save_measuringpoint_data( if not event.is_empty(): event_name = event.item(0, 0) event_type = GMN_EVENT_MAPPING[event_name] + + # FUTURE: This is tricky, as MoveRequests might make this not true + # If a MeasuringPoint is moved, it's date changes while it might already be in the database. IntermediateEvent.objects.update_or_create( gmn=self.gmn_obj, data_owner=self.gmn_obj.data_owner, @@ -384,17 +387,19 @@ def _save_measuringpoint_data( defaults = { "measuringpoint_start_date": mp_start_date, "measuringpoint_end_date": mp_end_date, + "gmw_bro_id": bro_id, "tube_number": tube_nr, "tube_start_date": event_date, "tube_end_date": end_date, "event_type": event_type, } + # There can only be one measuring point with the same code active at one point + # GMW-ID is not a contributing factor Measuringpoint.objects.update_or_create( gmn=self.gmn_obj, data_owner=self.data_owner, measuringpoint_code=mp_code, - gmw_bro_id=bro_id, defaults=defaults, ) @@ -477,7 +482,11 @@ def _save_gmw_data(self, gmw_data: dict[str, Any]) -> None: well_construction_date: dict = gmw_data.get("wellHistory", {}).get( "wellConstructionDate", {} ) + well_removal_date: dict = gmw_data.get("wellHistory", {}).get( + "wellRemovalDate", {} + ) internal_id = self.retrieve_internal_id(gmw_data.get("brocom:broId", "")) + removed = gmw_data.get("removed", None) self.gmw_obj = GMW.objects.update_or_create( bro_id=gmw_data.get("brocom:broId", None), data_owner=self.data_owner, @@ -500,7 +509,7 @@ def _save_gmw_data(self, gmw_data: dict[str, Any]) -> None: "initial_function": gmw_data.get("initialFunction", {}).get( "#text", None ), - "removed": gmw_data.get("removed", None), + "removed": removed, "ground_level_stable": gmw_data.get("groundLevelStable", None), "well_stability": gmw_data.get("wellStability", {}).get("#text", None), "nitg_code": gmw_data.get("nitgCode", None), @@ -546,6 +555,24 @@ def _save_gmw_data(self, gmw_data: dict[str, Any]) -> None: }, )[0] + if removed: + Event.objects.update_or_create( + gmw=self.gmw_obj, + data_owner=self.data_owner, + event_type="GMW_Removal", + defaults={ + "event_date": well_removal_date, + "metadata": { + "broId": self.gmw_obj.bro_id, + "qualityRegime": self.gmw_obj.quality_regime, + "deliveryAccountableParty": self.gmw_obj.delivery_accountable_party, + }, + "sourcedocument_data": { + "eventDate": well_removal_date, + }, + }, + ) + def _save_monitoringtubes_data( self, monitoringtubes_data: list[dict[str, Any]] | dict[str, Any], @@ -732,6 +759,7 @@ def _save_events_data(self, event_data: list[dict[str, any]]): intermediate_event ) + # FUTURE: This is tricky, as MoveRequests might make this not true. The same event can be in the history multiple times with different dates, but only the most recent one is relevant. Currently, we create an event for each of them, which can lead to duplicates. A solution could be to check if an event with the same name already exists for this GMW, and if so, only update it if the date is more recent. For now, we just create an event for each entry in the history, which can lead to duplicates. Event.objects.update_or_create( gmw=self.gmw_obj, data_owner=self.data_owner, @@ -942,15 +970,17 @@ def _save_field_measurements( for measurement in measurements: measurement_value = measurement.get("garcommon:fieldMeasurementValue", None) - FieldMeasurement.objects.create( + FieldMeasurement.objects.update_or_create( gar=gar, parameter=int(measurement.get("garcommon:parameter")), - unit=self._attr_or_none(measurement_value, "uom"), - field_measurement_value=self._text_or_none(measurement_value), - quality_control_status=self._text_or_none( - measurement.get("garcommon:qualityControlStatus", None) - ), data_owner=self.data_owner, + defaults={ + "unit": self._attr_or_none(measurement_value, "uom"), + "field_measurement_value": self._text_or_none(measurement_value), + "quality_control_status": self._text_or_none( + measurement.get("garcommon:qualityControlStatus", None) + ), + }, ) def _save_laboratory_researches(self, gar: GAR, lab_analyses: list) -> None: @@ -986,7 +1016,7 @@ def _save_analysis_processes( analyses_date = process.get("garcommon:analysisDate", {}).get( "brocom:date", None ) - analysis_process = AnalysisProcess.objects.create( + analysis_process = AnalysisProcess.objects.update_or_create( laboratory_research=lab_research, analyses_date=analyses_date, analytical_technique=self._text_or_none( @@ -1021,17 +1051,19 @@ def _save_analyses( reporting_limit = value value = None - Analysis.objects.create( + Analysis.objects.update_or_create( analysis_process=analysis_process, parameter=int(analysis.get("garcommon:parameter")), - value=value, - unit=self._attr_or_none(analysis_value, "uom"), - reporting_limit=reporting_limit, - limit_symbol=limit_symbol, - status_quality_control=self._text_or_none( - analysis.get("garcommon:qualityControlStatus", None) - ), data_owner=self.data_owner, + defaults={ + "value": value, + "unit": self._attr_or_none(analysis_value, "uom"), + "reporting_limit": reporting_limit, + "limit_symbol": limit_symbol, + "status_quality_control": self._text_or_none( + analysis.get("garcommon:qualityControlStatus", None) + ), + }, ) diff --git a/api/utils.py b/api/utils.py index aa1709b..f670774 100644 --- a/api/utils.py +++ b/api/utils.py @@ -169,9 +169,9 @@ def create_gmw_removal( Event.objects.update_or_create( gmw=gmw, event_name="GMW_Removal", - event_date=sourcedocument_data.get("eventDate"), data_owner=data_owner, defaults={ + "event_date": sourcedocument_data.get("eventDate"), "metadata": metadata, "sourcedocument_data": sourcedocument_data, }, @@ -536,7 +536,8 @@ def update_gmw_event( ) ## FUTURE: Make sure that the sourcedocument and metadata are formatted correctly - Event.objects.create( + # Currently this creates, not updates the right event. + Event.objects.update( gmw=gmw, event_name=event_type, event_date=sourcedocument_data.get("eventDate"), From d7050c365c7b01c47aed585181f49f30ad69179f Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Sat, 23 May 2026 21:53:48 +0200 Subject: [PATCH 03/11] split utils --- api/utils.py | 1066 ---------------------------------------- api/utils/__init__.py | 270 ++++++++++ api/utils/frd_utils.py | 298 +++++++++++ api/utils/gar_utils.py | 111 +++++ api/utils/gld_utils.py | 172 +++++++ api/utils/gmn_utils.py | 244 +++++++++ api/utils/gmw_utils.py | 382 ++++++++++++++ api/utils/gpd_utils.py | 73 +++ api/utils/guf_utils.py | 183 +++++++ api/utils/helpers.py | 61 +++ 10 files changed, 1794 insertions(+), 1066 deletions(-) delete mode 100644 api/utils.py create mode 100644 api/utils/__init__.py create mode 100644 api/utils/frd_utils.py create mode 100644 api/utils/gar_utils.py create mode 100644 api/utils/gld_utils.py create mode 100644 api/utils/gmn_utils.py create mode 100644 api/utils/gmw_utils.py create mode 100644 api/utils/gpd_utils.py create mode 100644 api/utils/guf_utils.py create mode 100644 api/utils/helpers.py diff --git a/api/utils.py b/api/utils.py deleted file mode 100644 index f670774..0000000 --- a/api/utils.py +++ /dev/null @@ -1,1066 +0,0 @@ -import datetime -import logging -from functools import partial - -from pyproj import Transformer - -from frd.models import FRD, GeoElectricMeasurement, MeasurementConfiguration -from gld.models import GLD, Observation -from gmn.models import GMN, IntermediateEvent, Measuringpoint -from gmw.models import GMW, Event, MonitoringTube - -logger = logging.getLogger(__name__) - -# Define transformer from RD New (EPSG:28992) to ETRS89 (EPSG:4258 = lat/lon) -transformer = Transformer.from_crs("EPSG:28992", "EPSG:4258", always_xy=True) - - -def create_gmw( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - delivered_location = sourcedocument_data.get("deliveredLocation", "0 0").split(" ") - if delivered_location == "" or len(delivered_location) != 2: - logger.info( - f"Invalid deliveredLocation format for bro_id={bro_id}, owner={data_owner}. Expected 'x y', got: {sourcedocument_data.get('deliveredLocation')}" - ) - rd_x, rd_y = 0.0, 0.0 - else: - rd_x, rd_y = map(float, delivered_location) - - lon, lat = transformer.transform(rd_x, rd_y) - standardized_location = f"{lat} {lon}" - gmw = GMW.objects.update_or_create( - bro_id=bro_id, - data_owner=data_owner, - defaults={ - "internal_id": sourcedocument_data.get("objectIdAccountableParty"), - "delivery_accountable_party": metadata.get("deliveryAccountableParty"), - "quality_regime": metadata.get("qualityRegime"), - "well_construction_date": sourcedocument_data.get("wellConstructionDate"), - "delivery_context": sourcedocument_data.get("deliveryContext"), - "construction_standard": sourcedocument_data.get("constructionStandard"), - "initial_function": sourcedocument_data.get("initialFunction"), - "ground_level_stable": sourcedocument_data.get("groundLevelStable"), - "well_stability": sourcedocument_data.get("wellStability"), - "nitg_code": sourcedocument_data.get("nitgCode"), - "well_code": sourcedocument_data.get("wellCode"), - "owner": sourcedocument_data.get("owner"), - "well_head_protector": sourcedocument_data.get("wellHeadProtector"), - "delivered_location": sourcedocument_data.get("deliveredLocation"), - "horizontal_positioning_method": sourcedocument_data.get( - "horizontalPositioningMethod" - ), - "local_vertical_reference_point": sourcedocument_data.get( - "localVerticalReferencePoint" - ), - "offset": sourcedocument_data.get("offset"), - "vertical_datum": sourcedocument_data.get("verticalDatum"), - "ground_level_position": sourcedocument_data.get("groundLevelPosition"), - "ground_level_positioning_method": sourcedocument_data.get( - "groundLevelPositioningMethod" - ), - "standardized_location": standardized_location, - "registration_status": "geregistreerd", - "removed": "nee", - }, - )[0] - for tube in sourcedocument_data.get("monitoringTubes", []): - position_tube_top = tube.get("tubeTopPosition") - plain_tube_length = tube.get("plainTubePartLength") - screen_top_position = ( - float(position_tube_top) - float(plain_tube_length) - if (plain_tube_length is not None and position_tube_top is not None) - else None - ) - screen_bottom_position = ( - float(screen_top_position) - float(tube.get("screenLength")) - if ( - screen_top_position is not None and tube.get("screenLength") is not None - ) - else None - ) - geo_ohm_cables = tube.get("geoOhmCables", []) - MonitoringTube.objects.update_or_create( - gmw=gmw, - tube_number=tube.get("tubeNumber"), - data_owner=gmw.data_owner, - defaults={ - "tube_type": tube.get("tubeType"), - "artesian_well_cap_present": tube.get("artesianWellCapPresent"), - "sediment_sump_present": tube.get("sedimentSumpPresent"), - "sediment_sump_length": tube.get("sedimentSumpLength"), - "number_of_geo_ohm_cables": tube.get("numberOfGeoOhmCables", 0), - "geo_ohm_cables": geo_ohm_cables if geo_ohm_cables else [], - "tube_top_diameter": tube.get("tubeTopDiameter"), - "variable_diameter": tube.get("variableDiameter"), - "tube_status": tube.get("tubeStatus"), - "tube_top_position": position_tube_top, - "tube_top_positioning_method": tube.get("tubeTopPositioningMethod"), - "tube_part_inserted": tube.get("tubePartInserted"), - "tube_in_use": tube.get("tubeInUse"), - "tube_packing_material": tube.get("tubePackingMaterial"), - "tube_material": tube.get("tubeMaterial"), - "glue": tube.get("glue"), - "screen_length": tube.get("screenLength"), - "screen_protection": tube.get("screenProtection"), - "sock_material": tube.get("sockMaterial"), - "screen_top_position": screen_top_position, - "screen_bottom_position": screen_bottom_position, - "plain_tube_part_length": plain_tube_length, - }, - ) - - logger.info(f"Sucessfully created {gmw} with {gmw.tubes.count()} monitoring tubes.") - return - - -def create_gmw_event( - *, - bro_id: str, - event_type: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - """Generic factory for creating GMW events.""" - try: - gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMW.DoesNotExist: - logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") - return - except GMW.MultipleObjectsReturned: - gmw = ( - GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) - .order_by("created_at") - .first() - ) - - Event.objects.update_or_create( - gmw=gmw, - event_name=event_type, - event_date=sourcedocument_data.get("eventDate"), - data_owner=data_owner, - defaults={ - "metadata": metadata, - "sourcedocument_data": sourcedocument_data, - }, - ) - - -def create_gmw_removal( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - """Generic factory for creating GMW events.""" - try: - gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMW.DoesNotExist: - logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") - return - except GMW.MultipleObjectsReturned: - gmw = ( - GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) - .order_by("created_at") - .first() - ) - - Event.objects.update_or_create( - gmw=gmw, - event_name="GMW_Removal", - data_owner=data_owner, - defaults={ - "event_date": sourcedocument_data.get("eventDate"), - "metadata": metadata, - "sourcedocument_data": sourcedocument_data, - }, - ) - - gmw.removed = "ja" - gmw.save(update_fields=["removed"]) - - -def find_linked_gmns(gmn_bro_ids: list[str] | str) -> list[GMN]: - if isinstance(gmn_bro_ids, str): - gmn_bro_ids = [gmn_bro_ids] - gmns = GMN.objects.filter(bro_id__in=gmn_bro_ids) - if len(gmns) != len(gmn_bro_ids): - found_bro_ids = {gmn.bro_id for gmn in gmns} - missing = set(gmn_bro_ids) - found_bro_ids - logger.info(f"Could not find GMNs with bro_id(s): {', '.join(missing)}") - return list(gmns) - - -def create_gld( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - GLD.objects.update_or_create( - bro_id=bro_id, - data_owner=data_owner, - defaults={ - "internal_id": sourcedocument_data.get("objectIdAccountableParty"), - "delivery_accountable_party": metadata.get("deliveryAccountableParty"), - "linked_gmns": sourcedocument_data.get("linkedGmns", []), - "quality_regime": metadata.get("qualityRegime"), - "gmw_bro_id": sourcedocument_data.get("gmwBroId"), - "tube_number": sourcedocument_data.get("tubeNumber"), - }, - ) - return - - -def create_gld_addition( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - """Generic factory for creating GLD Observations.""" - try: - gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) - except GLD.DoesNotExist: - logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") - return - - Observation.objects.update_or_create( - gld=gld, - observation_id=sourcedocument_data.get("observationId"), - data_owner=data_owner, - defaults={ - "begin_position": sourcedocument_data.get("beginPosition"), - "end_position": sourcedocument_data.get("endPosition"), - "result_time": sourcedocument_data.get("resultTime"), - "validation_status": sourcedocument_data.get("validationStatus"), - "investigator_kvk": sourcedocument_data.get("investigatorKvk"), - "observation_type": sourcedocument_data.get("observationType"), - "process_reference": sourcedocument_data.get("processReference"), - "air_pressure_compensation_type": sourcedocument_data.get( - "airPressureCompensationType" - ), - "evaluation_procedure": sourcedocument_data.get("evaluationProcedure"), - "measurement_instrument_type": sourcedocument_data.get( - "measurementInstrumentType" - ), - }, - ) - - -def create_gmn( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - GMN.objects.update_or_create( - bro_id=bro_id, - data_owner=data_owner, - defaults={ - "internal_id": sourcedocument_data.get("objectIdAccountableParty"), - "quality_regime": metadata.get("qualityRegime"), - "name": sourcedocument_data.get("name"), - "delivery_context": sourcedocument_data.get("deliveryContext"), - "monitoring_purpose": sourcedocument_data.get("monitoringPurpose"), - "groundwater_aspect": sourcedocument_data.get("groundwaterAspect"), - "start_date_monitoring": sourcedocument_data.get("startDateMonitoring"), - }, - ) - return - - -def create_gmn_measuringpoint( - *, - bro_id: str, - event_type: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - """Generic factory for creating GMN Measuring Points.""" - try: - gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMN.DoesNotExist: - logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") - return - - last_measuring_point = ( - Measuringpoint.objects.filter( - gmn=gmn, - measuringpoint_code=sourcedocument_data.get("measuringPointCode"), - data_owner=data_owner, - tube_start_date__lt=sourcedocument_data.get("eventDate"), - ) - .order_by("-tube_start_date") - .first() - ) - - Measuringpoint.objects.update_or_create( - gmn=gmn, - data_owner=data_owner, - measuringpoint_code=sourcedocument_data.get("measuringPointCode"), - gmw_bro_id=sourcedocument_data.get( - "broId" - ), # In GMN MeasuringPoint model, broId instead of gmwBroId is used. - tube_number=sourcedocument_data.get("tubeNumber"), - event_type=event_type, - defaults={ - "tube_start_date": sourcedocument_data.get("eventDate"), - "measuringpoint_start_date": last_measuring_point.measuringpoint_start_date - if last_measuring_point - else sourcedocument_data.get("eventDate"), - }, - ) - - if last_measuring_point and event_type == "GMN_MeasuringPointEndDate": - # Set all measuring points that started before this end date to have this end date - Measuringpoint.objects.filter( - gmn=gmn, - measuringpoint_code=sourcedocument_data.get("measuringPointCode"), - data_owner=data_owner, - tube_start_date__lt=datetime.datetime.strptime( - sourcedocument_data.get("eventDate", "1900-01-01"), "%Y-%m-%d" - ), - ).update( - measuringpoint_end_date=sourcedocument_data.get("eventDate"), - ) - - last_measuring_point.refresh_from_db() - # Update the last tube to have the tube_end_date as well - last_measuring_point.tube_end_date = sourcedocument_data.get("eventDate") - last_measuring_point.save(update_fields=["tube_end_date"]) - - elif last_measuring_point and event_type == "GMN_TubeReference": - last_measuring_point.tube_end_date = sourcedocument_data.get("eventDate") - last_measuring_point.save(update_fields=["tube_end_date"]) - - -def create_frd( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - FRD.objects.update_or_create( - bro_id=bro_id, - data_owner=data_owner, - defaults={ - "internal_id": sourcedocument_data.get("objectIdAccountableParty"), - "delivery_accountable_party": sourcedocument_data.get( - "deliveryAccountableParty" - ), - "quality_regime": metadata.get("qualityRegime"), - "gmw_bro_id": sourcedocument_data.get("gmwBroId"), - "tube_number": sourcedocument_data.get("tubeNumber"), - }, - ) - return - - -# Build mapping with pre-bound event types -CREATE_FUNCTION_MAPPING = { - "GMW_Construction": create_gmw, - "GMW_Positions": partial(create_gmw_event, event_type="GMW_Positions"), - "GMW_PositionsMeasuring": partial( - create_gmw_event, event_type="GMW_PositionsMeasuring" - ), - "GMW_WellHeadProtector": partial( - create_gmw_event, event_type="GMW_WellHeadProtector" - ), - "GMW_Owner": partial(create_gmw_event, event_type="GMW_Owner"), - "GMW_Shift": partial(create_gmw_event, event_type="GMW_Shift"), - "GMW_GroundLevel": partial(create_gmw_event, event_type="GMW_GroundLevel"), - "GMW_GroundLevelMeasuring": partial( - create_gmw_event, event_type="GMW_GroundLevelMeasuring" - ), - "GMW_Insertion": partial(create_gmw_event, event_type="GMW_Insertion"), - "GMW_TubeStatus": partial(create_gmw_event, event_type="GMW_TubeStatus"), - "GMW_Lengthening": partial(create_gmw_event, event_type="GMW_Lengthening"), - "GMW_Shortening": partial(create_gmw_event, event_type="GMW_Shortening"), - "GMW_ElectrodeStatus": partial(create_gmw_event, event_type="GMW_ElectrodeStatus"), - "GMW_Maintainer": partial(create_gmw_event, event_type="GMW_Maintainer"), - "GMW_Removal": create_gmw_removal, - "GLD_StartRegistration": create_gld, - "GLD_Addition": create_gld_addition, - "GMN_StartRegistration": create_gmn, - "GMN_MeasuringPoint": partial( - create_gmn_measuringpoint, event_type="GMN_MeasuringPoint" - ), - "GMN_MeasuringPointEndDate": partial( - create_gmn_measuringpoint, event_type="GMN_MeasuringPointEndDate" - ), - "GMN_TubeReference": partial( - create_gmn_measuringpoint, event_type="GMN_TubeReference" - ), - "FRD_StartRegistration": create_frd, - # "GAR": create_gar, # TODO -} - - -# ── Update functions ────────────────────────────────────────────────────────── - - -def update_gmw( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - try: - gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMW.DoesNotExist: - logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") - return - except GMW.MultipleObjectsReturned: - gmw = ( - GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) - .order_by("created_at") - .first() - ) - - source_field_map = { - "internal_id": "objectIdAccountableParty", - "well_construction_date": "wellConstructionDate", - "delivery_context": "deliveryContext", - "construction_standard": "constructionStandard", - "initial_function": "initialFunction", - "ground_level_stable": "groundLevelStable", - "well_stability": "wellStability", - "nitg_code": "nitgCode", - "well_code": "wellCode", - "owner": "owner", - "well_head_protector": "wellHeadProtector", - "delivered_location": "deliveredLocation", - "horizontal_positioning_method": "horizontalPositioningMethod", - "local_vertical_reference_point": "localVerticalReferencePoint", - "offset": "offset", - "vertical_datum": "verticalDatum", - "ground_level_position": "groundLevelPosition", - "ground_level_positioning_method": "groundLevelPositioningMethod", - } - meta_field_map = { - "delivery_accountable_party": "deliveryAccountableParty", - "quality_regime": "qualityRegime", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - updates.update( - { - field: metadata[key] - for field, key in meta_field_map.items() - if key in metadata - } - ) - if "deliveredLocation" in sourcedocument_data: - delivered_location = sourcedocument_data.get("deliveredLocation", "0 0").split( - " " - ) - if len(delivered_location) == 2: - rd_x, rd_y = map(float, delivered_location) - lon, lat = transformer.transform(rd_x, rd_y) - updates["standardized_location"] = f"{lat} {lon}" - - for field, value in updates.items(): - setattr(gmw, field, value) - if updates: - gmw.save(update_fields=list(updates.keys())) - logger.info(f"Successfully updated {gmw}.") - - monitoring_tubes_data = sourcedocument_data.get("monitoringTubes", []) - for tube in monitoring_tubes_data: - tube_number = tube.get("tubeNumber", 1) - - MonitoringTube.objects.update_or_create( - gmw=gmw, - tube_number=tube_number, - data_owner=data_owner, - defaults={ - "tube_type": tube.get("tubeType"), - "artesian_well_cap_present": tube.get("artesianWellCapPresent"), - "sediment_sump_present": tube.get("sedimentSumpPresent"), - "sediment_sump_length": tube.get("sedimentSumpLength"), - "number_of_geo_ohm_cables": tube.get("numberOfGeoOhmCables", 0), - "geo_ohm_cables": tube.get("geoOhmCables", []), - "tube_top_diameter": tube.get("tubeTopDiameter"), - "variable_diameter": tube.get("variableDiameter"), - "tube_status": tube.get("tubeStatus"), - "tube_top_position": tube.get("tubeTopPosition"), - "tube_top_positioning_method": tube.get("tubeTopPositioningMethod"), - "tube_part_inserted": tube.get("tubePartInserted"), - "tube_in_use": tube.get("tubeInUse"), - "tube_packing_material": tube.get("tubePackingMaterial"), - "tube_material": tube.get("tubeMaterial"), - "glue": tube.get("glue"), - "screen_length": tube.get("screenLength"), - "screen_protection": tube.get("screenProtection"), - "sock_material": tube.get("sockMaterial"), - "plain_tube_part_length": tube.get("plainTubePartLength"), - }, - ) - - -def update_gmw_event( - *, - bro_id: str, - event_type: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMW.DoesNotExist: - logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") - return - except GMW.MultipleObjectsReturned: - gmw = ( - GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) - .order_by("created_at") - .first() - ) - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - request_type = "move" if date_to_correct else "replace" - - event_qs = Event.objects.filter( - gmw=gmw, event_name=event_type, data_owner=data_owner - ) - if request_type == "move": - event_qs = event_qs.filter(event_date=date_to_correct) - else: - event_qs = event_qs.filter(event_date=sourcedocument_data.get("eventDate")) - - if event_qs.count() != 1: - logger.warning( - f"Expected to find exactly 1 event for bro_id={bro_id}, event_type={event_type}, date={date_to_correct}, but found {event_qs.count()}. Skipping update." - ) - return - - event = event_qs.first() - if not event: - logger.info( - f"Event {event_type} not found for bro_id={bro_id}, date={date_to_correct}. Creating new event." - ) - - ## FUTURE: Make sure that the sourcedocument and metadata are formatted correctly - # Currently this creates, not updates the right event. - Event.objects.update( - gmw=gmw, - event_name=event_type, - event_date=sourcedocument_data.get("eventDate"), - metadata=metadata, - sourcedocument_data=sourcedocument_data, - data_owner=data_owner, - ) - return - - update_fields = ["metadata", "sourcedocument_data"] - event.metadata = metadata - event.sourcedocument_data = sourcedocument_data - new_event_date = sourcedocument_data.get("eventDate") - if new_event_date: - event.event_date = new_event_date - update_fields.append("event_date") - event.save(update_fields=update_fields) - - -def update_gmw_removal( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMW.DoesNotExist: - logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") - return - except GMW.MultipleObjectsReturned: - gmw = ( - GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) - .order_by("created_at") - .first() - ) - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - event_qs = Event.objects.filter( - gmw=gmw, event_name="GMW_Removal", data_owner=data_owner - ) - if date_to_correct: - event_qs = event_qs.filter(event_date=date_to_correct) - - event = event_qs.order_by("-event_date").first() - if not event: - logger.info( - f"GMW_Removal event not found for bro_id={bro_id}, date={date_to_correct}." - ) - return - - update_fields = ["metadata", "sourcedocument_data"] - event.metadata = metadata - event.sourcedocument_data = sourcedocument_data - new_event_date = sourcedocument_data.get("eventDate") - if new_event_date: - event.event_date = new_event_date - update_fields.append("event_date") - event.save(update_fields=update_fields) - - -def update_gld( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - try: - gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) - except GLD.DoesNotExist: - logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") - return - - source_field_map = { - "internal_id": "objectIdAccountableParty", - "linked_gmns": "linkedGmns", - "gmw_bro_id": "gmwBroId", - "tube_number": "tubeNumber", - } - meta_field_map = { - "delivery_accountable_party": "deliveryAccountableParty", - "quality_regime": "qualityRegime", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - updates.update( - { - field: metadata[key] - for field, key in meta_field_map.items() - if key in metadata - } - ) - for field, value in updates.items(): - setattr(gld, field, value) - if updates: - gld.save(update_fields=list(updates.keys())) - - -def update_gld_observation( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) - except GLD.DoesNotExist: - logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") - return - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) - if date_to_correct: - obs_qs = obs_qs.filter(begin_position=date_to_correct) - - observation = obs_qs.order_by("-begin_position").first() - if not observation: - logger.info( - f"Observation not found for bro_id={bro_id}, date={date_to_correct}." - ) - return - - source_field_map = { - "begin_position": "beginPosition", - "end_position": "endPosition", - "result_time": "resultTime", - "validation_status": "validationStatus", - "investigator_kvk": "investigatorKvk", - "observation_type": "observationType", - "process_reference": "processReference", - "air_pressure_compensation_type": "airPressureCompensationType", - "evaluation_procedure": "evaluationProcedure", - "measurement_instrument_type": "measurementInstrumentType", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - for field, value in updates.items(): - setattr(observation, field, value) - if updates: - observation.save(update_fields=list(updates.keys())) - - -def update_gmn( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - try: - gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMN.DoesNotExist: - logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") - return - - source_field_map = { - "internal_id": "objectIdAccountableParty", - "name": "name", - "delivery_context": "deliveryContext", - "monitoring_purpose": "monitoringPurpose", - "groundwater_aspect": "groundwaterAspect", - "start_date_monitoring": "startDateMonitoring", - } - meta_field_map = { - "quality_regime": "qualityRegime", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - updates.update( - { - field: metadata[key] - for field, key in meta_field_map.items() - if key in metadata - } - ) - for field, value in updates.items(): - setattr(gmn, field, value) - if updates: - gmn.save(update_fields=list(updates.keys())) - - -def update_gmn_measuringpoint( - *, - bro_id: str, - event_type: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMN.DoesNotExist: - logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") - return - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - mp_qs = Measuringpoint.objects.filter( - gmn=gmn, - measuringpoint_code=sourcedocument_data.get("measuringPointCode"), - data_owner=data_owner, - event_type=event_type, - ) - if date_to_correct: - mp_qs = mp_qs.filter(tube_start_date=date_to_correct) - - measuringpoint = mp_qs.order_by("-tube_start_date").first() - if not measuringpoint: - logger.info( - f"Measuringpoint not found for bro_id={bro_id}, " - f"code={sourcedocument_data.get('measuringPointCode')}, date={date_to_correct}." - ) - return - - source_field_map = { - "gmw_bro_id": "broId", - "tube_number": "tubeNumber", - "tube_start_date": "eventDate", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - for field, value in updates.items(): - setattr(measuringpoint, field, value) - if updates: - measuringpoint.save(update_fields=list(updates.keys())) - - -def update_frd( - bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str -) -> None: - try: - frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) - except FRD.DoesNotExist: - logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") - return - - source_field_map = { - "internal_id": "objectIdAccountableParty", - "delivery_accountable_party": "deliveryAccountableParty", - "gmw_bro_id": "gmwBroId", - "tube_number": "tubeNumber", - } - meta_field_map = { - "quality_regime": "qualityRegime", - } - updates = { - field: sourcedocument_data[key] - for field, key in source_field_map.items() - if key in sourcedocument_data - } - updates.update( - { - field: metadata[key] - for field, key in meta_field_map.items() - if key in metadata - } - ) - for field, value in updates.items(): - setattr(frd, field, value) - if updates: - frd.save(update_fields=list(updates.keys())) - - -UPDATE_FUNCTION_MAPPING = { - "GMW_Construction": update_gmw, - "GMW_Positions": partial(update_gmw_event, event_type="GMW_Positions"), - "GMW_PositionsMeasuring": partial( - update_gmw_event, event_type="GMW_PositionsMeasuring" - ), - "GMW_WellHeadProtector": partial( - update_gmw_event, event_type="GMW_WellHeadProtector" - ), - "GMW_Owner": partial(update_gmw_event, event_type="GMW_Owner"), - "GMW_Shift": partial(update_gmw_event, event_type="GMW_Shift"), - "GMW_GroundLevel": partial(update_gmw_event, event_type="GMW_GroundLevel"), - "GMW_GroundLevelMeasuring": partial( - update_gmw_event, event_type="GMW_GroundLevelMeasuring" - ), - "GMW_Insertion": partial(update_gmw_event, event_type="GMW_Insertion"), - "GMW_TubeStatus": partial(update_gmw_event, event_type="GMW_TubeStatus"), - "GMW_Lengthening": partial(update_gmw_event, event_type="GMW_Lengthening"), - "GMW_Shortening": partial(update_gmw_event, event_type="GMW_Shortening"), - "GMW_ElectrodeStatus": partial(update_gmw_event, event_type="GMW_ElectrodeStatus"), - "GMW_Maintainer": partial(update_gmw_event, event_type="GMW_Maintainer"), - "GMW_Removal": update_gmw_removal, - "GLD_StartRegistration": update_gld, - "GLD_Addition": update_gld_observation, - "GMN_StartRegistration": update_gmn, - "GMN_MeasuringPoint": partial( - update_gmn_measuringpoint, event_type="GMN_MeasuringPoint" - ), - "GMN_MeasuringPointEndDate": partial( - update_gmn_measuringpoint, event_type="GMN_MeasuringPointEndDate" - ), - "GMN_TubeReference": partial( - update_gmn_measuringpoint, event_type="GMN_TubeReference" - ), - "FRD_StartRegistration": update_frd, -} - - -# ── Delete functions ────────────────────────────────────────────────────────── - - -def delete_gld_observation( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) - except GLD.DoesNotExist: - logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") - return - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) - if date_to_correct: - obs_qs = obs_qs.filter(begin_position=date_to_correct) - - deleted_count, _ = obs_qs.delete() - logger.info( - f"Deleted {deleted_count} observation(s) for bro_id={bro_id}, date={date_to_correct}." - ) - - -def delete_gmn_intermediate_event( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) - except GMN.DoesNotExist: - logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") - return - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - event_qs = IntermediateEvent.objects.filter(gmn=gmn, data_owner=data_owner) - if date_to_correct: - event_qs = event_qs.filter(event_date=date_to_correct) - - deleted_count, _ = event_qs.delete() - logger.info( - f"Deleted {deleted_count} intermediate event(s) for bro_id={bro_id}, date={date_to_correct}." - ) - - -def delete_frd_measurement_configuration( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) - except FRD.DoesNotExist: - logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") - return - - config_id = sourcedocument_data.get("measurementConfigurationId") - config_qs = MeasurementConfiguration.objects.filter(frd=frd, data_owner=data_owner) - if config_id: - config_qs = config_qs.filter(measurement_configuration_id=config_id) - - deleted_count, _ = config_qs.delete() - logger.info( - f"Deleted {deleted_count} measurement configuration(s) for bro_id={bro_id}." - ) - - -def delete_frd_measurement( - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) - except FRD.DoesNotExist: - logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") - return - - date_to_correct = sourcedocument_data.get("dateToBeCorrected") - measurement_qs = GeoElectricMeasurement.objects.filter( - frd=frd, data_owner=data_owner - ) - if date_to_correct: - measurement_qs = measurement_qs.filter(measurement_date=date_to_correct) - - deleted_count, _ = measurement_qs.delete() - logger.info( - f"Deleted {deleted_count} geo-electric measurement(s) for bro_id={bro_id}, date={date_to_correct}." - ) - - -DELETE_FUNCTION_MAPPING = { - "GLD_Closure": delete_gld_observation, - "GMN_Closure": delete_gmn_intermediate_event, - "FRD_GEM_MeasurementConfiguration": delete_frd_measurement_configuration, - "FRD_GEM_Measurement": delete_frd_measurement, - "FRD_EMM_InstrumentConfiguration": delete_frd_measurement_configuration, - "FRD_EMM_Measurement": delete_frd_measurement, -} - - -def create_objects( - registration_type: str, - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - CREATE_FUNCTION_MAPPING[registration_type]( - bro_id=bro_id, - metadata=metadata, - sourcedocument_data=sourcedocument_data, - data_owner=data_owner, - ) - except KeyError: - logger.info( - f"Unable to create as there is no function mapped for registration type: {registration_type}." - ) - return - - -def update_objects( - registration_type: str, - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - UPDATE_FUNCTION_MAPPING[registration_type]( - bro_id=bro_id, - metadata=metadata, - sourcedocument_data=sourcedocument_data, - data_owner=data_owner, - ) - except KeyError: - logger.info( - f"Unable to update as there is no function mapped for registration type: {registration_type}." - ) - return - - -def delete_objects( - registration_type: str, - bro_id: str, - metadata: dict, - sourcedocument_data: dict, - data_owner: str, -) -> None: - try: - DELETE_FUNCTION_MAPPING[registration_type]( - bro_id=bro_id, - metadata=metadata, - sourcedocument_data=sourcedocument_data, - data_owner=data_owner, - ) - except KeyError: - logger.info( - f"Unable to delete as there is no function mapped for registration type: {registration_type}." - ) - return - - -def empty_strings_to_none(d: dict) -> dict: - for key, value in d.items(): - if isinstance(value, str) and value.strip() == "": - d[key] = None - elif isinstance(value, dict): - d[key] = empty_strings_to_none(value) - elif isinstance(value, list): - d[key] = [ - empty_strings_to_none(v) - if isinstance(v, dict) - else (None if v == "" else v) - for v in value - ] - return d - - -def strip_whitespace(data): - if isinstance(data, dict): - return {k: strip_whitespace(v) for k, v in data.items()} - elif isinstance(data, list): - return [strip_whitespace(item) for item in data] - elif isinstance(data, str): - return data.strip() - return data - - -def drop_empty_strings(d: dict) -> dict: # noqa: C901 - cleaned = {} - for key, value in d.items(): - if isinstance(value, str): - if value.strip() == "": - continue # skip this field entirely - cleaned[key] = value - elif isinstance(value, dict): - nested = drop_empty_strings(value) - if nested: # only keep if not empty - cleaned[key] = nested - elif isinstance(value, list): - cleaned_list = [] - for v in value: - if isinstance(v, dict): - nested = drop_empty_strings(v) - if nested: - cleaned_list.append(nested) - elif not (isinstance(v, str) and v.strip() == ""): - cleaned_list.append(v) - if cleaned_list: - cleaned[key] = cleaned_list - else: - cleaned[key] = value - return cleaned diff --git a/api/utils/__init__.py b/api/utils/__init__.py new file mode 100644 index 0000000..65d463d --- /dev/null +++ b/api/utils/__init__.py @@ -0,0 +1,270 @@ +""" +api.utils package — domain-specific create/update/delete utilities. + +Each sub-module owns one BRO domain. This __init__.py re-exports every +symbol that was previously in the flat api/utils.py so that existing +imports continue to work without change. +""" + +import logging as _logging +from functools import partial + +from .frd_utils import ( + create_frd, + create_frd_emm_instrument_configuration, + create_frd_emm_measurement, + create_frd_gem_measurement, + create_frd_gem_measurement_configuration, + delete_frd_measurement, + delete_frd_measurement_configuration, + update_frd, +) +from .gar_utils import create_gar +from .gld_utils import ( + create_gld, + create_gld_addition, + delete_gld_observation, + update_gld, + update_gld_observation, +) +from .gmn_utils import ( + create_gmn, + create_gmn_measuringpoint, + delete_gmn_intermediate_event, + find_linked_gmns, + remove_enddate_gmn, + update_enddate_gmn, + update_gmn, + update_gmn_measuringpoint, +) +from .gmw_utils import ( + create_gmw, + create_gmw_event, + create_gmw_removal, + update_gmw, + update_gmw_event, + update_gmw_removal, +) +from .gpd_utils import create_gpd, create_gpd_report +from .guf_utils import create_guf +from .helpers import ( + drop_empty_strings, + empty_strings_to_none, + strip_whitespace, + transformer, +) + +__all__ = [ + # helpers + "transformer", + "empty_strings_to_none", + "strip_whitespace", + "drop_empty_strings", + # GMW + "create_gmw", + "create_gmw_event", + "create_gmw_removal", + "update_gmw", + "update_gmw_event", + "update_gmw_removal", + # GLD + "create_gld", + "create_gld_addition", + "update_gld", + "update_gld_observation", + "delete_gld_observation", + # GMN + "find_linked_gmns", + "create_gmn", + "create_gmn_measuringpoint", + "update_gmn", + "update_gmn_measuringpoint", + "delete_gmn_intermediate_event", + # FRD + "create_frd", + "create_frd_gem_measurement_configuration", + "create_frd_emm_instrument_configuration", + "create_frd_gem_measurement", + "create_frd_emm_measurement", + "update_frd", + "delete_frd_measurement_configuration", + "delete_frd_measurement", + # GAR + "create_gar", + # GUF + "create_guf", + # GPD + "create_gpd", + "create_gpd_report", + # mappings + dispatchers + "CREATE_FUNCTION_MAPPING", + "UPDATE_FUNCTION_MAPPING", + "DELETE_FUNCTION_MAPPING", + "create_objects", + "update_objects", + "delete_objects", +] + +# ── Mappings ────────────────────────────────────────────────────────────────── + +CREATE_FUNCTION_MAPPING = { + "GMW_Construction": create_gmw, + "GMW_Positions": partial(create_gmw_event, event_type="GMW_Positions"), + "GMW_PositionsMeasuring": partial( + create_gmw_event, event_type="GMW_PositionsMeasuring" + ), + "GMW_WellHeadProtector": partial( + create_gmw_event, event_type="GMW_WellHeadProtector" + ), + "GMW_Owner": partial(create_gmw_event, event_type="GMW_Owner"), + "GMW_Shift": partial(create_gmw_event, event_type="GMW_Shift"), + "GMW_GroundLevel": partial(create_gmw_event, event_type="GMW_GroundLevel"), + "GMW_GroundLevelMeasuring": partial( + create_gmw_event, event_type="GMW_GroundLevelMeasuring" + ), + "GMW_Insertion": partial(create_gmw_event, event_type="GMW_Insertion"), + "GMW_TubeStatus": partial(create_gmw_event, event_type="GMW_TubeStatus"), + "GMW_Lengthening": partial(create_gmw_event, event_type="GMW_Lengthening"), + "GMW_Shortening": partial(create_gmw_event, event_type="GMW_Shortening"), + "GMW_ElectrodeStatus": partial(create_gmw_event, event_type="GMW_ElectrodeStatus"), + "GMW_Maintainer": partial(create_gmw_event, event_type="GMW_Maintainer"), + "GMW_Removal": create_gmw_removal, + "GLD_StartRegistration": create_gld, + "GLD_Addition": create_gld_addition, + "GMN_StartRegistration": create_gmn, + "GMN_MeasuringPoint": partial( + create_gmn_measuringpoint, event_type="GMN_MeasuringPoint" + ), + "GMN_MeasuringPointEndDate": partial( + create_gmn_measuringpoint, event_type="GMN_MeasuringPointEndDate" + ), + "GMN_TubeReference": partial( + create_gmn_measuringpoint, event_type="GMN_TubeReference" + ), + "GMN_Closure": update_enddate_gmn, + "FRD_StartRegistration": create_frd, + "FRD_GEM_MeasurementConfiguration": create_frd_gem_measurement_configuration, + "FRD_EMM_InstrumentConfiguration": create_frd_emm_instrument_configuration, + "FRD_GEM_Measurement": create_frd_gem_measurement, + "FRD_EMM_Measurement": create_frd_emm_measurement, + "GAR_StartRegistration": create_gar, + "GUF_StartRegistration": create_guf, + "GPD_StartRegistration": create_gpd, + "GPD_AddReport": create_gpd_report, +} + +UPDATE_FUNCTION_MAPPING = { + "GMW_Construction": update_gmw, + "GMW_Positions": partial(update_gmw_event, event_type="GMW_Positions"), + "GMW_PositionsMeasuring": partial( + update_gmw_event, event_type="GMW_PositionsMeasuring" + ), + "GMW_WellHeadProtector": partial( + update_gmw_event, event_type="GMW_WellHeadProtector" + ), + "GMW_Owner": partial(update_gmw_event, event_type="GMW_Owner"), + "GMW_Shift": partial(update_gmw_event, event_type="GMW_Shift"), + "GMW_GroundLevel": partial(update_gmw_event, event_type="GMW_GroundLevel"), + "GMW_GroundLevelMeasuring": partial( + update_gmw_event, event_type="GMW_GroundLevelMeasuring" + ), + "GMW_Insertion": partial(update_gmw_event, event_type="GMW_Insertion"), + "GMW_TubeStatus": partial(update_gmw_event, event_type="GMW_TubeStatus"), + "GMW_Lengthening": partial(update_gmw_event, event_type="GMW_Lengthening"), + "GMW_Shortening": partial(update_gmw_event, event_type="GMW_Shortening"), + "GMW_ElectrodeStatus": partial(update_gmw_event, event_type="GMW_ElectrodeStatus"), + "GMW_Maintainer": partial(update_gmw_event, event_type="GMW_Maintainer"), + "GMW_Removal": update_gmw_removal, + "GLD_StartRegistration": update_gld, + "GLD_Addition": update_gld_observation, + "GMN_StartRegistration": update_gmn, + "GMN_MeasuringPoint": partial( + update_gmn_measuringpoint, event_type="GMN_MeasuringPoint" + ), + "GMN_MeasuringPointEndDate": partial( + update_gmn_measuringpoint, event_type="GMN_MeasuringPointEndDate" + ), + "GMN_TubeReference": partial( + update_gmn_measuringpoint, event_type="GMN_TubeReference" + ), + "FRD_StartRegistration": update_frd, +} + +DELETE_FUNCTION_MAPPING = { + "GLD_Addition": delete_gld_observation, + "GMN_MeasuringPoint": delete_gmn_intermediate_event, + "GMN_MeasuringPointEndDate": delete_gmn_intermediate_event, + "GMN_TubeReference": delete_gmn_intermediate_event, + "GMN_Closure": remove_enddate_gmn, + "FRD_GEM_MeasurementConfiguration": delete_frd_measurement_configuration, + "FRD_GEM_Measurement": delete_frd_measurement, + "FRD_EMM_InstrumentConfiguration": delete_frd_measurement_configuration, + "FRD_EMM_Measurement": delete_frd_measurement, +} + + +# ── Dispatcher functions ────────────────────────────────────────────────────── +_logger = _logging.getLogger(__name__) + + +def create_objects( + registration_type: str, + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + CREATE_FUNCTION_MAPPING[registration_type]( + bro_id=bro_id, + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + except KeyError: + _logger.info( + f"Unable to create as there is no function mapped for registration type: {registration_type}." + ) + return + + +def update_objects( + registration_type: str, + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + UPDATE_FUNCTION_MAPPING[registration_type]( + bro_id=bro_id, + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + except KeyError: + _logger.info( + f"Unable to update as there is no function mapped for registration type: {registration_type}." + ) + return + + +def delete_objects( + registration_type: str, + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + DELETE_FUNCTION_MAPPING[registration_type]( + bro_id=bro_id, + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + except KeyError: + _logger.info( + f"Unable to delete as there is no function mapped for registration type: {registration_type}." + ) + return diff --git a/api/utils/frd_utils.py b/api/utils/frd_utils.py new file mode 100644 index 0000000..911fe39 --- /dev/null +++ b/api/utils/frd_utils.py @@ -0,0 +1,298 @@ +import logging + +from frd.models import ( + FRD, + CalculatedApparentFormationResistance, + GeoElectricMeasure, + GeoElectricMeasurement, + MeasurementConfiguration, +) + +logger = logging.getLogger(__name__) + + +def create_frd( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + FRD.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": sourcedocument_data.get( + "deliveryAccountableParty" + ), + "quality_regime": metadata.get("qualityRegime"), + "gmw_bro_id": sourcedocument_data.get("gmwBroId"), + "tube_number": sourcedocument_data.get("tubeNumber"), + }, + ) + return + + +def create_frd_gem_measurement_configuration( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create GEM MeasurementConfiguration records for an FRD.""" + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + for config in sourcedocument_data.get("measurementConfigurations", []): + measurement_pair = { + "measurementE1CableNumber": config.get("measurementE1CableNumber"), + "measurementE1ElectrodeNumber": config.get("measurementE1ElectrodeNumber"), + "measurementE2CableNumber": config.get("measurementE2CableNumber"), + "measurementE2ElectrodeNumber": config.get("measurementE2ElectrodeNumber"), + } + current_pair = { + "currentE1CableNumber": config.get("currentE1CableNumber"), + "currentE1ElectrodeNumber": config.get("currentE1ElectrodeNumber"), + "currentE2CableNumber": config.get("currentE2CableNumber"), + "currentE2ElectrodeNumber": config.get("currentE2ElectrodeNumber"), + } + MeasurementConfiguration.objects.update_or_create( + frd=frd, + measurement_configuration_id=config.get("measurementConfigurationId"), + data_owner=data_owner, + defaults={ + "measurement_pair": measurement_pair, + "current_pair": current_pair, + }, + ) + logger.info(f"Created/updated measurement configurations for FRD bro_id={bro_id}.") + + +def create_frd_emm_instrument_configuration( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create EMM InstrumentConfiguration record for an FRD. + + EMM instrument configurations are stored in the MeasurementConfiguration model + with instrument-specific fields packed into measurement_pair. + """ + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + instrument_data = { + "relativePositionTransmitterCoil": sourcedocument_data.get( + "relativePositionTransmitterCoil" + ), + "relativePositionPrimaryReceiverCoil": sourcedocument_data.get( + "relativePositionPrimaryReceiverCoil" + ), + "secondaryReceiverCoilAvailable": sourcedocument_data.get( + "secondaryReceiverCoilAvailable" + ), + "relativePositionSecondaryReceiverCoil": sourcedocument_data.get( + "relativePositionSecondaryReceiverCoil" + ), + "coilFrequencyKnown": sourcedocument_data.get("coilFrequencyKnown"), + "coilFrequency": sourcedocument_data.get("coilFrequency"), + "instrumentLength": sourcedocument_data.get("instrumentLength"), + } + MeasurementConfiguration.objects.update_or_create( + frd=frd, + measurement_configuration_id=sourcedocument_data.get( + "instrumentConfigurationId" + ), + data_owner=data_owner, + defaults={ + "measurement_pair": instrument_data, + "current_pair": None, + }, + ) + logger.info( + f"Created/updated EMM instrument configuration for FRD bro_id={bro_id}." + ) + + +def create_frd_gem_measurement( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create a GeoElectricMeasurement with its measures and optional calculated resistance.""" + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + gem, _ = GeoElectricMeasurement.objects.update_or_create( + frd=frd, + measurement_date=sourcedocument_data.get("measurementDate"), + data_owner=data_owner, + defaults={ + "determination_procedure": sourcedocument_data.get( + "determinationProcedure" + ), + "evaluation_procedure": sourcedocument_data.get("evaluationProcedure"), + }, + ) + + # Recreate sub-measures on every registration to keep them in sync + gem.measures.all().delete() + for measure in sourcedocument_data.get("measurements", []): + GeoElectricMeasure.objects.create( + geo_electric_measurement=gem, + resistance=str(measure.get("value")) + if measure.get("value") is not None + else None, + related_measurement_configuration=measure.get("configuration"), + data_owner=data_owner, + ) + + calc_resistance = sourcedocument_data.get( + "relatedCalculatedApparentFormationResistance" + ) + if calc_resistance: + CalculatedApparentFormationResistance.objects.update_or_create( + geo_electric_measurement=gem, + defaults={ + "evaluation_procedure": calc_resistance.get("evaluationProcedure"), + "values": calc_resistance.get("values"), + "data_owner": data_owner, + }, + ) + + logger.info( + f"Created/updated GEM measurement for FRD bro_id={bro_id}, date={sourcedocument_data.get('measurementDate')}." + ) + + +def create_frd_emm_measurement( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create an EMM (Electro Magnetic Method) measurement for an FRD. + + EMM measurements are stored in the same GeoElectricMeasurement model as GEM. + The calculated series values are stored in CalculatedApparentFormationResistance. + """ + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + gem, _ = GeoElectricMeasurement.objects.update_or_create( + frd=frd, + measurement_date=sourcedocument_data.get("measurementDate"), + data_owner=data_owner, + defaults={ + "determination_procedure": sourcedocument_data.get( + "determinationProcedure" + ), + "evaluation_procedure": sourcedocument_data.get( + "measurementEvaluationProcedure" + ), + }, + ) + + CalculatedApparentFormationResistance.objects.update_or_create( + geo_electric_measurement=gem, + defaults={ + "evaluation_procedure": sourcedocument_data.get( + "calculationEvaluationProcedure" + ), + "values": sourcedocument_data.get("calculationValues"), + "data_owner": data_owner, + }, + ) + + logger.info( + f"Created/updated EMM measurement for FRD bro_id={bro_id}, date={sourcedocument_data.get('measurementDate')}." + ) + + +# ── Update functions ────────────────────────────────────────────────────────── + + +def update_frd( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "delivery_accountable_party": "deliveryAccountableParty", + "gmw_bro_id": "gmwBroId", + "tube_number": "tubeNumber", + } + meta_field_map = { + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(frd, field, value) + if updates: + frd.save(update_fields=list(updates.keys())) + + +# ── Delete functions ────────────────────────────────────────────────────────── + + +def delete_frd_measurement_configuration( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + config_id = sourcedocument_data.get("measurementConfigurationId") + config_qs = MeasurementConfiguration.objects.filter(frd=frd, data_owner=data_owner) + if config_id: + config_qs = config_qs.filter(measurement_configuration_id=config_id) + + deleted_count, _ = config_qs.delete() + logger.info( + f"Deleted {deleted_count} measurement configuration(s) for bro_id={bro_id}." + ) + + +def delete_frd_measurement( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + frd = FRD.objects.get(bro_id=bro_id, data_owner=data_owner) + except FRD.DoesNotExist: + logger.info(f"FRD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + measurement_qs = GeoElectricMeasurement.objects.filter( + frd=frd, data_owner=data_owner + ) + if date_to_correct: + measurement_qs = measurement_qs.filter(measurement_date=date_to_correct) + + deleted_count, _ = measurement_qs.delete() + logger.info( + f"Deleted {deleted_count} geo-electric measurement(s) for bro_id={bro_id}, date={date_to_correct}." + ) diff --git a/api/utils/gar_utils.py b/api/utils/gar_utils.py new file mode 100644 index 0000000..c1de1c2 --- /dev/null +++ b/api/utils/gar_utils.py @@ -0,0 +1,111 @@ +import logging + +from gar.models import ( + GAR, + Analysis, + AnalysisProcess, + FieldMeasurement, + LaboratoryResearch, +) + +logger = logging.getLogger(__name__) + + +def create_gar( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create a GAR with its field measurements and laboratory research hierarchy.""" + field_research = sourcedocument_data.get("fieldResearch", {}) + field_observation = field_research.get("fieldObservation", {}) + + gar, _ = GAR.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": metadata.get("deliveryAccountableParty"), + "quality_regime": metadata.get("qualityRegime"), + "quality_control_method": sourcedocument_data.get( + "qualityControlMethod", "onbekend" + ), + "gmw_bro_id": sourcedocument_data.get("gmwBroId"), + "tube_number": sourcedocument_data.get("tubeNumber"), + "sampling_datetime": field_research.get("samplingDateTime"), + "sampling_standard": field_research.get("samplingStandard"), + "primary_colour": field_research.get("primaryColour"), + "secondary_colour": field_research.get("secondaryColour"), + "colour_strength": field_research.get("colourStrength"), + "pump_type": field_research.get("pumpType"), + "abnormality_in_cooling": field_observation.get("abnormalityInCooling"), + "abnormality_in_device": field_observation.get("abnormalityInDevice"), + "polluted_by_engine": field_observation.get("pollutedByEngine"), + "filter_aerated": field_observation.get("filterAerated"), + "groundwater_level_dropped_too_much": field_observation.get( + "groundwaterLevelDroppedTooMuch" + ), + "abnormal_filter": field_observation.get("abnormalFilter"), + "sample_aerated": field_observation.get("sampleAerated"), + "hose_reused": field_observation.get("hoseReused"), + "temperature_difficult_to_measure": field_observation.get( + "temperatureDifficultToMeasure" + ), + }, + ) + + _create_field_measurements(gar, field_research, data_owner) + _create_laboratory_researches( + gar, sourcedocument_data.get("laboratoryAnalyses", []), data_owner + ) + logger.info(f"Successfully created/updated GAR bro_id={bro_id}.") + + +def _create_field_measurements(gar: GAR, field_research: dict, data_owner: str) -> None: + for measurement in field_research.get("fieldMeasurements", []): + FieldMeasurement.objects.update_or_create( + gar=gar, + parameter=int(measurement.get("parameter")), + data_owner=data_owner, + defaults={ + "unit": measurement.get("unit"), + "field_measurement_value": measurement.get("fieldMeasurementValue"), + "quality_control_status": measurement.get("qualityControlStatus"), + }, + ) + + +def _create_laboratory_researches( + gar: GAR, lab_analyses: list, data_owner: str +) -> None: + for lab_analysis in lab_analyses: + lab_research, _ = LaboratoryResearch.objects.update_or_create( + gar=gar, + laboratory_kvk_number=lab_analysis.get("responsibleLaboratoryKvk"), + data_owner=data_owner, + ) + for process in lab_analysis.get("analysisProcesses", []): + analysis_process, _ = AnalysisProcess.objects.update_or_create( + laboratory_research=lab_research, + analyses_date=process.get("date"), + data_owner=data_owner, + defaults={ + "analytical_technique": process.get("analyticalTechnique"), + "validation_method": process.get("valuationMethod"), + }, + ) + for analysis in process.get("analyses", []): + Analysis.objects.update_or_create( + analysis_process=analysis_process, + parameter=int(analysis.get("parameter")), + data_owner=data_owner, + defaults={ + "unit": analysis.get("unit"), + "value": str(analysis.get("analysisMeasurementValue")) + if analysis.get("analysisMeasurementValue") is not None + else None, + "limit_symbol": analysis.get("limitSymbol"), + "reporting_limit": str(analysis.get("reportingLimit")) + if analysis.get("reportingLimit") is not None + else None, + "status_quality_control": analysis.get("qualityControlStatus"), + }, + ) diff --git a/api/utils/gld_utils.py b/api/utils/gld_utils.py new file mode 100644 index 0000000..3ebace4 --- /dev/null +++ b/api/utils/gld_utils.py @@ -0,0 +1,172 @@ +import logging + +from gld.models import GLD, Observation + +logger = logging.getLogger(__name__) + + +def create_gld( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + GLD.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": metadata.get("deliveryAccountableParty"), + "linked_gmns": sourcedocument_data.get("linkedGmns", []), + "quality_regime": metadata.get("qualityRegime"), + "gmw_bro_id": sourcedocument_data.get("gmwBroId"), + "tube_number": sourcedocument_data.get("tubeNumber"), + }, + ) + return + + +def create_gld_addition( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + """Generic factory for creating GLD Observations.""" + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + Observation.objects.update_or_create( + gld=gld, + observation_id=sourcedocument_data.get("observationId"), + data_owner=data_owner, + defaults={ + "begin_position": sourcedocument_data.get("beginPosition"), + "end_position": sourcedocument_data.get("endPosition"), + "result_time": sourcedocument_data.get("resultTime"), + "validation_status": sourcedocument_data.get("validationStatus"), + "investigator_kvk": sourcedocument_data.get("investigatorKvk"), + "observation_type": sourcedocument_data.get("observationType"), + "process_reference": sourcedocument_data.get("processReference"), + "air_pressure_compensation_type": sourcedocument_data.get( + "airPressureCompensationType" + ), + "evaluation_procedure": sourcedocument_data.get("evaluationProcedure"), + "measurement_instrument_type": sourcedocument_data.get( + "measurementInstrumentType" + ), + }, + ) + + +# ── Update functions ────────────────────────────────────────────────────────── + + +def update_gld( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "linked_gmns": "linkedGmns", + "gmw_bro_id": "gmwBroId", + "tube_number": "tubeNumber", + } + meta_field_map = { + "delivery_accountable_party": "deliveryAccountableParty", + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(gld, field, value) + if updates: + gld.save(update_fields=list(updates.keys())) + + +def update_gld_observation( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) + if date_to_correct: + obs_qs = obs_qs.filter(begin_position=date_to_correct) + + observation = obs_qs.order_by("-begin_position").first() + if not observation: + logger.info( + f"Observation not found for bro_id={bro_id}, date={date_to_correct}." + ) + return + + source_field_map = { + "begin_position": "beginPosition", + "end_position": "endPosition", + "result_time": "resultTime", + "validation_status": "validationStatus", + "investigator_kvk": "investigatorKvk", + "observation_type": "observationType", + "process_reference": "processReference", + "air_pressure_compensation_type": "airPressureCompensationType", + "evaluation_procedure": "evaluationProcedure", + "measurement_instrument_type": "measurementInstrumentType", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + for field, value in updates.items(): + setattr(observation, field, value) + if updates: + observation.save(update_fields=list(updates.keys())) + + +# ── Delete functions ────────────────────────────────────────────────────────── + + +def delete_gld_observation( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gld = GLD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GLD.DoesNotExist: + logger.info(f"GLD not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + obs_qs = Observation.objects.filter(gld=gld, data_owner=data_owner) + if date_to_correct: + obs_qs = obs_qs.filter(begin_position=date_to_correct) + + deleted_count, _ = obs_qs.delete() + logger.info( + f"Deleted {deleted_count} observation(s) for bro_id={bro_id}, date={date_to_correct}." + ) diff --git a/api/utils/gmn_utils.py b/api/utils/gmn_utils.py new file mode 100644 index 0000000..8fb1319 --- /dev/null +++ b/api/utils/gmn_utils.py @@ -0,0 +1,244 @@ +import datetime +import logging + +from gmn.models import GMN, IntermediateEvent, Measuringpoint + +logger = logging.getLogger(__name__) + + +def find_linked_gmns(gmn_bro_ids: list[str] | str) -> list[GMN]: + if isinstance(gmn_bro_ids, str): + gmn_bro_ids = [gmn_bro_ids] + gmns = GMN.objects.filter(bro_id__in=gmn_bro_ids) + if len(gmns) != len(gmn_bro_ids): + found_bro_ids = {gmn.bro_id for gmn in gmns} + missing = set(gmn_bro_ids) - found_bro_ids + logger.info(f"Could not find GMNs with bro_id(s): {', '.join(missing)}") + return list(gmns) + + +def create_gmn( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + GMN.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "quality_regime": metadata.get("qualityRegime"), + "name": sourcedocument_data.get("name"), + "delivery_context": sourcedocument_data.get("deliveryContext"), + "monitoring_purpose": sourcedocument_data.get("monitoringPurpose"), + "groundwater_aspect": sourcedocument_data.get("groundwaterAspect"), + "start_date_monitoring": sourcedocument_data.get("startDateMonitoring"), + }, + ) + return + + +def create_gmn_measuringpoint( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + """Generic factory for creating GMN Measuring Points.""" + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + last_measuring_point = ( + Measuringpoint.objects.filter( + gmn=gmn, + measuringpoint_code=sourcedocument_data.get("measuringPointCode"), + data_owner=data_owner, + tube_start_date__lt=sourcedocument_data.get("eventDate"), + ) + .order_by("-tube_start_date") + .first() + ) + + Measuringpoint.objects.update_or_create( + gmn=gmn, + data_owner=data_owner, + measuringpoint_code=sourcedocument_data.get("measuringPointCode"), + gmw_bro_id=sourcedocument_data.get( + "broId" + ), # In GMN MeasuringPoint model, broId instead of gmwBroId is used. + tube_number=sourcedocument_data.get("tubeNumber"), + event_type=event_type, + defaults={ + "tube_start_date": sourcedocument_data.get("eventDate"), + "measuringpoint_start_date": last_measuring_point.measuringpoint_start_date + if last_measuring_point + else sourcedocument_data.get("eventDate"), + }, + ) + + if last_measuring_point and event_type == "GMN_MeasuringPointEndDate": + # Set all measuring points that started before this end date to have this end date + Measuringpoint.objects.filter( + gmn=gmn, + measuringpoint_code=sourcedocument_data.get("measuringPointCode"), + data_owner=data_owner, + tube_start_date__lt=datetime.datetime.strptime( + sourcedocument_data.get("eventDate", "1900-01-01"), "%Y-%m-%d" + ), + ).update( + measuringpoint_end_date=sourcedocument_data.get("eventDate"), + ) + + last_measuring_point.refresh_from_db() + # Update the last tube to have the tube_end_date as well + last_measuring_point.tube_end_date = sourcedocument_data.get("eventDate") + last_measuring_point.save(update_fields=["tube_end_date"]) + + elif last_measuring_point and event_type == "GMN_TubeReference": + last_measuring_point.tube_end_date = sourcedocument_data.get("eventDate") + last_measuring_point.save(update_fields=["tube_end_date"]) + + +# ── Update functions ────────────────────────────────────────────────────────── + + +def update_gmn( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "name": "name", + "delivery_context": "deliveryContext", + "monitoring_purpose": "monitoringPurpose", + "groundwater_aspect": "groundwaterAspect", + "start_date_monitoring": "startDateMonitoring", + } + meta_field_map = { + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + for field, value in updates.items(): + setattr(gmn, field, value) + if updates: + gmn.save(update_fields=list(updates.keys())) + + +def update_enddate_gmn( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + end_date_monitoring = sourcedocument_data.get("endDateMonitoring") + gmn.end_date_monitoring = end_date_monitoring + gmn.save(update_fields=["end_date_monitoring"]) + + +def remove_enddate_gmn( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + gmn.end_date_monitoring = None + gmn.save(update_fields=["end_date_monitoring"]) + + +def update_gmn_measuringpoint( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + mp_qs = Measuringpoint.objects.filter( + gmn=gmn, + measuringpoint_code=sourcedocument_data.get("measuringPointCode"), + data_owner=data_owner, + event_type=event_type, + ) + if date_to_correct: + mp_qs = mp_qs.filter(tube_start_date=date_to_correct) + + measuringpoint = mp_qs.order_by("-tube_start_date").first() + if not measuringpoint: + logger.info( + f"Measuringpoint not found for bro_id={bro_id}, " + f"code={sourcedocument_data.get('measuringPointCode')}, date={date_to_correct}." + ) + return + + source_field_map = { + "gmw_bro_id": "broId", + "tube_number": "tubeNumber", + "tube_start_date": "eventDate", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + for field, value in updates.items(): + setattr(measuringpoint, field, value) + if updates: + measuringpoint.save(update_fields=list(updates.keys())) + + +# ── Delete functions ────────────────────────────────────────────────────────── + + +def delete_gmn_intermediate_event( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmn = GMN.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMN.DoesNotExist: + logger.info(f"GMN not found for bro_id={bro_id}, owner={data_owner}") + return + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + event_qs = IntermediateEvent.objects.filter(gmn=gmn, data_owner=data_owner) + if date_to_correct: + event_qs = event_qs.filter(event_date=date_to_correct) + + deleted_count, _ = event_qs.delete() + logger.info( + f"Deleted {deleted_count} intermediate event(s) for bro_id={bro_id}, date={date_to_correct}." + ) diff --git a/api/utils/gmw_utils.py b/api/utils/gmw_utils.py new file mode 100644 index 0000000..ab2ad26 --- /dev/null +++ b/api/utils/gmw_utils.py @@ -0,0 +1,382 @@ +import logging + +from gmw.models import GMW, Event, MonitoringTube + +from .helpers import transformer + +logger = logging.getLogger(__name__) + + +def create_gmw( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + delivered_location = sourcedocument_data.get("deliveredLocation", "0 0").split(" ") + if delivered_location == "" or len(delivered_location) != 2: + logger.info( + f"Invalid deliveredLocation format for bro_id={bro_id}, owner={data_owner}. Expected 'x y', got: {sourcedocument_data.get('deliveredLocation')}" + ) + rd_x, rd_y = 0.0, 0.0 + else: + rd_x, rd_y = map(float, delivered_location) + + lon, lat = transformer.transform(rd_x, rd_y) + standardized_location = f"{lat} {lon}" + gmw = GMW.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": metadata.get("deliveryAccountableParty"), + "quality_regime": metadata.get("qualityRegime"), + "well_construction_date": sourcedocument_data.get("wellConstructionDate"), + "delivery_context": sourcedocument_data.get("deliveryContext"), + "construction_standard": sourcedocument_data.get("constructionStandard"), + "initial_function": sourcedocument_data.get("initialFunction"), + "ground_level_stable": sourcedocument_data.get("groundLevelStable"), + "well_stability": sourcedocument_data.get("wellStability"), + "nitg_code": sourcedocument_data.get("nitgCode"), + "well_code": sourcedocument_data.get("wellCode"), + "owner": sourcedocument_data.get("owner"), + "well_head_protector": sourcedocument_data.get("wellHeadProtector"), + "delivered_location": sourcedocument_data.get("deliveredLocation"), + "horizontal_positioning_method": sourcedocument_data.get( + "horizontalPositioningMethod" + ), + "local_vertical_reference_point": sourcedocument_data.get( + "localVerticalReferencePoint" + ), + "offset": sourcedocument_data.get("offset"), + "vertical_datum": sourcedocument_data.get("verticalDatum"), + "ground_level_position": sourcedocument_data.get("groundLevelPosition"), + "ground_level_positioning_method": sourcedocument_data.get( + "groundLevelPositioningMethod" + ), + "standardized_location": standardized_location, + "registration_status": "geregistreerd", + "removed": "nee", + }, + )[0] + for tube in sourcedocument_data.get("monitoringTubes", []): + position_tube_top = tube.get("tubeTopPosition") + plain_tube_length = tube.get("plainTubePartLength") + screen_top_position = ( + float(position_tube_top) - float(plain_tube_length) + if (plain_tube_length is not None and position_tube_top is not None) + else None + ) + screen_bottom_position = ( + float(screen_top_position) - float(tube.get("screenLength")) + if ( + screen_top_position is not None and tube.get("screenLength") is not None + ) + else None + ) + geo_ohm_cables = tube.get("geoOhmCables", []) + MonitoringTube.objects.update_or_create( + gmw=gmw, + tube_number=tube.get("tubeNumber"), + data_owner=gmw.data_owner, + defaults={ + "tube_type": tube.get("tubeType"), + "artesian_well_cap_present": tube.get("artesianWellCapPresent"), + "sediment_sump_present": tube.get("sedimentSumpPresent"), + "sediment_sump_length": tube.get("sedimentSumpLength"), + "number_of_geo_ohm_cables": tube.get("numberOfGeoOhmCables", 0), + "geo_ohm_cables": geo_ohm_cables if geo_ohm_cables else [], + "tube_top_diameter": tube.get("tubeTopDiameter"), + "variable_diameter": tube.get("variableDiameter"), + "tube_status": tube.get("tubeStatus"), + "tube_top_position": position_tube_top, + "tube_top_positioning_method": tube.get("tubeTopPositioningMethod"), + "tube_part_inserted": tube.get("tubePartInserted"), + "tube_in_use": tube.get("tubeInUse"), + "tube_packing_material": tube.get("tubePackingMaterial"), + "tube_material": tube.get("tubeMaterial"), + "glue": tube.get("glue"), + "screen_length": tube.get("screenLength"), + "screen_protection": tube.get("screenProtection"), + "sock_material": tube.get("sockMaterial"), + "screen_top_position": screen_top_position, + "screen_bottom_position": screen_bottom_position, + "plain_tube_part_length": plain_tube_length, + }, + ) + + logger.info(f"Sucessfully created {gmw} with {gmw.tubes.count()} monitoring tubes.") + return + + +def create_gmw_event( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + """Generic factory for creating GMW events.""" + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + Event.objects.update_or_create( + gmw=gmw, + event_name=event_type, + event_date=sourcedocument_data.get("eventDate"), + data_owner=data_owner, + defaults={ + "metadata": metadata, + "sourcedocument_data": sourcedocument_data, + }, + ) + + +def create_gmw_removal( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + """Generic factory for creating GMW events.""" + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + Event.objects.update_or_create( + gmw=gmw, + event_name="GMW_Removal", + data_owner=data_owner, + defaults={ + "event_date": sourcedocument_data.get("eventDate"), + "metadata": metadata, + "sourcedocument_data": sourcedocument_data, + }, + ) + + gmw.removed = "ja" + gmw.save(update_fields=["removed"]) + + +# ── Update functions ────────────────────────────────────────────────────────── + + +def update_gmw( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + source_field_map = { + "internal_id": "objectIdAccountableParty", + "well_construction_date": "wellConstructionDate", + "delivery_context": "deliveryContext", + "construction_standard": "constructionStandard", + "initial_function": "initialFunction", + "ground_level_stable": "groundLevelStable", + "well_stability": "wellStability", + "nitg_code": "nitgCode", + "well_code": "wellCode", + "owner": "owner", + "well_head_protector": "wellHeadProtector", + "delivered_location": "deliveredLocation", + "horizontal_positioning_method": "horizontalPositioningMethod", + "local_vertical_reference_point": "localVerticalReferencePoint", + "offset": "offset", + "vertical_datum": "verticalDatum", + "ground_level_position": "groundLevelPosition", + "ground_level_positioning_method": "groundLevelPositioningMethod", + } + meta_field_map = { + "delivery_accountable_party": "deliveryAccountableParty", + "quality_regime": "qualityRegime", + } + updates = { + field: sourcedocument_data[key] + for field, key in source_field_map.items() + if key in sourcedocument_data + } + updates.update( + { + field: metadata[key] + for field, key in meta_field_map.items() + if key in metadata + } + ) + if "deliveredLocation" in sourcedocument_data: + delivered_location = sourcedocument_data.get("deliveredLocation", "0 0").split( + " " + ) + if len(delivered_location) == 2: + rd_x, rd_y = map(float, delivered_location) + lon, lat = transformer.transform(rd_x, rd_y) + updates["standardized_location"] = f"{lat} {lon}" + + for field, value in updates.items(): + setattr(gmw, field, value) + if updates: + gmw.save(update_fields=list(updates.keys())) + logger.info(f"Successfully updated {gmw}.") + + monitoring_tubes_data = sourcedocument_data.get("monitoringTubes", []) + for tube in monitoring_tubes_data: + tube_number = tube.get("tubeNumber", 1) + + MonitoringTube.objects.update_or_create( + gmw=gmw, + tube_number=tube_number, + data_owner=data_owner, + defaults={ + "tube_type": tube.get("tubeType"), + "artesian_well_cap_present": tube.get("artesianWellCapPresent"), + "sediment_sump_present": tube.get("sedimentSumpPresent"), + "sediment_sump_length": tube.get("sedimentSumpLength"), + "number_of_geo_ohm_cables": tube.get("numberOfGeoOhmCables", 0), + "geo_ohm_cables": tube.get("geoOhmCables", []), + "tube_top_diameter": tube.get("tubeTopDiameter"), + "variable_diameter": tube.get("variableDiameter"), + "tube_status": tube.get("tubeStatus"), + "tube_top_position": tube.get("tubeTopPosition"), + "tube_top_positioning_method": tube.get("tubeTopPositioningMethod"), + "tube_part_inserted": tube.get("tubePartInserted"), + "tube_in_use": tube.get("tubeInUse"), + "tube_packing_material": tube.get("tubePackingMaterial"), + "tube_material": tube.get("tubeMaterial"), + "glue": tube.get("glue"), + "screen_length": tube.get("screenLength"), + "screen_protection": tube.get("screenProtection"), + "sock_material": tube.get("sockMaterial"), + "plain_tube_part_length": tube.get("plainTubePartLength"), + }, + ) + + +def update_gmw_event( + *, + bro_id: str, + event_type: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + request_type = "move" if date_to_correct else "replace" + + event_qs = Event.objects.filter( + gmw=gmw, event_name=event_type, data_owner=data_owner + ) + if request_type == "move": + event_qs = event_qs.filter(event_date=date_to_correct) + else: + event_qs = event_qs.filter(event_date=sourcedocument_data.get("eventDate")) + + if event_qs.count() != 1: + logger.warning( + f"Expected to find exactly 1 event for bro_id={bro_id}, event_type={event_type}, date={date_to_correct}, but found {event_qs.count()}. Skipping update." + ) + return + + event = event_qs.first() + if not event: + logger.info( + f"Event {event_type} not found for bro_id={bro_id}, date={date_to_correct}. Creating new event." + ) + + ## FUTURE: Make sure that the sourcedocument and metadata are formatted correctly + # Currently this creates, not updates the right event. + Event.objects.update( + gmw=gmw, + event_name=event_type, + event_date=sourcedocument_data.get("eventDate"), + metadata=metadata, + sourcedocument_data=sourcedocument_data, + data_owner=data_owner, + ) + return + + update_fields = ["metadata", "sourcedocument_data"] + event.metadata = metadata + event.sourcedocument_data = sourcedocument_data + new_event_date = sourcedocument_data.get("eventDate") + if new_event_date: + event.event_date = new_event_date + update_fields.append("event_date") + event.save(update_fields=update_fields) + + +def update_gmw_removal( + bro_id: str, + metadata: dict, + sourcedocument_data: dict, + data_owner: str, +) -> None: + try: + gmw = GMW.objects.get(bro_id=bro_id, data_owner=data_owner) + except GMW.DoesNotExist: + logger.info(f"GMW not found for bro_id={bro_id}, owner={data_owner}") + return + except GMW.MultipleObjectsReturned: + gmw = ( + GMW.objects.filter(bro_id=bro_id, data_owner=data_owner) + .order_by("created_at") + .first() + ) + + date_to_correct = sourcedocument_data.get("dateToBeCorrected") + event_qs = Event.objects.filter( + gmw=gmw, event_name="GMW_Removal", data_owner=data_owner + ) + if date_to_correct: + event_qs = event_qs.filter(event_date=date_to_correct) + + event = event_qs.order_by("-event_date").first() + if not event: + logger.info( + f"GMW_Removal event not found for bro_id={bro_id}, date={date_to_correct}." + ) + return + + update_fields = ["metadata", "sourcedocument_data"] + event.metadata = metadata + event.sourcedocument_data = sourcedocument_data + new_event_date = sourcedocument_data.get("eventDate") + if new_event_date: + event.event_date = new_event_date + update_fields.append("event_date") + event.save(update_fields=update_fields) diff --git a/api/utils/gpd_utils.py b/api/utils/gpd_utils.py new file mode 100644 index 0000000..6e7c8ea --- /dev/null +++ b/api/utils/gpd_utils.py @@ -0,0 +1,73 @@ +import logging + +from gpd.models import GPD, Report, VolumeSeries + +logger = logging.getLogger(__name__) + + +def create_gpd( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create a GPD record.""" + GPD.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": metadata.get("deliveryAccountableParty"), + "quality_regime": metadata.get("qualityRegime"), + }, + ) + logger.info(f"Successfully created/updated GPD bro_id={bro_id}.") + + +def create_gpd_report( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create a Report with its VolumeSeries records for an existing GPD.""" + try: + gpd = GPD.objects.get(bro_id=bro_id, data_owner=data_owner) + except GPD.DoesNotExist: + logger.info(f"GPD not found for bro_id={bro_id}, owner={data_owner}") + return + + volume_series_data = sourcedocument_data.get("volumeSeries", []) + + # Derive begin/end date of the report from the enclosed volume series + begin_dates = [ + vs.get("beginDate") for vs in volume_series_data if vs.get("beginDate") + ] + end_dates = [vs.get("endDate") for vs in volume_series_data if vs.get("endDate")] + report_begin = min(begin_dates) if begin_dates else None + report_end = max(end_dates) if end_dates else None + + report, _ = Report.objects.update_or_create( + gpd=gpd, + report_id=sourcedocument_data.get("reportId"), + data_owner=data_owner, + defaults={ + "method": sourcedocument_data.get("method", "onbekend"), + "begin_date": report_begin, + "end_date": report_end, + "groundwater_usage_facility_bro_id": sourcedocument_data.get( + "groundwaterUsageFacilityBroId", "" + ), + }, + ) + + for vs in volume_series_data: + VolumeSeries.objects.update_or_create( + report=report, + water_in_out=vs.get("waterInOut"), + temperature=vs.get("temperature", "onbekend"), + begin_date=vs.get("beginDate"), + end_date=vs.get("endDate"), + data_owner=data_owner, + defaults={ + "volume": vs.get("volume", 0.0), + }, + ) + + logger.info( + f"Successfully created/updated report {sourcedocument_data.get('reportId')} for GPD bro_id={bro_id}." + ) diff --git a/api/utils/guf_utils.py b/api/utils/guf_utils.py new file mode 100644 index 0000000..5e102bd --- /dev/null +++ b/api/utils/guf_utils.py @@ -0,0 +1,183 @@ +import datetime +import logging + +from guf.models import ( + GUF, + DesignInstallation, + DesignLoop, + DesignSurfaceInfiltration, + DesignWell, + EnergyCharacteristics, +) + +logger = logging.getLogger(__name__) + + +def _parse_flexible_date(date_str: str | None) -> datetime.date | None: + """Parse BRO dates with flexible granularity: YYYY-MM-DD, YYYY-MM, or YYYY.""" + if not date_str: + return None + try: + if len(date_str) == 4: # YYYY + return datetime.date(int(date_str), 1, 1) + elif len(date_str) == 7: # YYYY-MM + year, month = date_str.split("-") + return datetime.date(int(year), int(month), 1) + else: # YYYY-MM-DD + return datetime.date.fromisoformat(date_str) + except (ValueError, AttributeError): + return None + + +def create_guf( + bro_id: str, metadata: dict, sourcedocument_data: dict, data_owner: str +) -> None: + """Create a GUF with its design installations and all nested sub-records.""" + guf, _ = GUF.objects.update_or_create( + bro_id=bro_id, + data_owner=data_owner, + defaults={ + "internal_id": sourcedocument_data.get("objectIdAccountableParty"), + "delivery_accountable_party": metadata.get("deliveryAccountableParty"), + "quality_regime": metadata.get("qualityRegime"), + "delivery_context": sourcedocument_data.get("deliveryContext"), + "start_time": _parse_flexible_date(sourcedocument_data.get("startTime")), + "identification_licence": sourcedocument_data.get("identificationLicence"), + "legal_type": sourcedocument_data.get("legalType"), + "primary_usage_type": sourcedocument_data.get("primaryUsageType"), + "secondary_usage_types": sourcedocument_data.get("secondaryUsageTypes", []), + "human_consumption": sourcedocument_data.get("humanConsumption"), + "licensed_quantities": sourcedocument_data.get("licensedQuantities", []), + }, + ) + + for installation_data in sourcedocument_data.get("designInstallations", []): + _create_design_installation(guf, installation_data, data_owner) + + logger.info(f"Successfully created/updated GUF bro_id={bro_id}.") + + +def _create_design_installation( + guf: GUF, installation_data: dict, data_owner: str +) -> None: + installation, _ = DesignInstallation.objects.update_or_create( + guf=guf, + design_installation_id=installation_data.get("designInstallationId"), + data_owner=data_owner, + defaults={ + "installation_function": installation_data.get("installationFunction"), + "design_installation_pos": installation_data.get("designInstallationPos"), + "licensed_quantities": installation_data.get("licensedQuantities", []), + }, + ) + + energy_data = installation_data.get("energyCharacteristics") + if energy_data: + EnergyCharacteristics.objects.update_or_create( + installation=installation, + defaults={ + "energy_cold": str(energy_data.get("energyCold")) + if energy_data.get("energyCold") is not None + else None, + "energy_warm": str(energy_data.get("energyWarm")) + if energy_data.get("energyWarm") is not None + else None, + "maximum_infiltration_temperature_warm": str( + energy_data.get("maximumInfiltrationTemperatureWarm") + ) + if energy_data.get("maximumInfiltrationTemperatureWarm") is not None + else None, + "average_infiltration_temperature_cold": str( + energy_data.get("averageInfiltrationTemperatureCold") + ) + if energy_data.get("averageInfiltrationTemperatureCold") is not None + else None, + "average_infiltration_temperature_warm": str( + energy_data.get("averageInfiltrationTemperatureWarm") + ) + if energy_data.get("averageInfiltrationTemperatureWarm") is not None + else None, + "power_cold": str(energy_data.get("powerCold")) + if energy_data.get("powerCold") is not None + else None, + "power_warm": str(energy_data.get("powerWarm")) + if energy_data.get("powerWarm") is not None + else None, + "power": str(energy_data.get("power")) + if energy_data.get("power") is not None + else None, + "average_warm_water": str(energy_data.get("averageWarmWater")) + if energy_data.get("averageWarmWater") is not None + else None, + "average_cold_water": str(energy_data.get("averageColdWater")) + if energy_data.get("averageColdWater") is not None + else None, + "maximum_year_quantity_warm": str( + energy_data.get("maximumYearQuantityWarm") + ) + if energy_data.get("maximumYearQuantityWarm") is not None + else None, + "maximum_year_quantity_cold": str( + energy_data.get("maximumYearQuantityCold") + ) + if energy_data.get("maximumYearQuantityCold") is not None + else None, + "data_owner": installation.data_owner, + }, + ) + + for loop_data in installation_data.get("designLoops", []): + DesignLoop.objects.update_or_create( + installation=installation, + design_loop_id=loop_data.get("designLoopId"), + data_owner=data_owner, + defaults={ + "loop_type": loop_data.get("loopType"), + "start_date": _parse_flexible_date(loop_data.get("startDate")), + "end_date": _parse_flexible_date(loop_data.get("endDate")), + "geometry_type": loop_data.get("geometryType"), + "design_loop_pos": loop_data.get("designLoopPos"), + }, + ) + + for well_data in installation_data.get("designWells", []): + DesignWell.objects.update_or_create( + installation=installation, + design_well_id=well_data.get("designWellId"), + data_owner=data_owner, + defaults={ + "well_functions": well_data.get("wellFunctions", []), + "height": str(well_data.get("height")) + if well_data.get("height") is not None + else None, + "well_pos": well_data.get("wellPos"), + "geometry_publicly_available": well_data.get( + "geometryPubliclyAvailable" + ), + "maximum_well_depth": str(well_data.get("maximumWellDepth")) + if well_data.get("maximumWellDepth") is not None + else None, + "maximum_well_depth_publicly_available": well_data.get( + "maximumWellDepthPubliclyAvailable" + ), + "maximum_well_capacity": str(well_data.get("maximumWellCapacity")) + if well_data.get("maximumWellCapacity") is not None + else None, + "relative_temperature": well_data.get("relativeTemperature"), + "design_screen": well_data.get("designScreen") or {}, + }, + ) + + for infiltration_data in installation_data.get("designSurfaceInfiltrations", []): + DesignSurfaceInfiltration.objects.update_or_create( + installation=installation, + design_surface_infiltration_id=infiltration_data.get( + "designSurfaceInfiltrationId" + ), + data_owner=data_owner, + defaults={ + "design_surface_infiltration_pos": infiltration_data.get( + "designSurfaceInfiltrationPos" + ), + }, + ) diff --git a/api/utils/helpers.py b/api/utils/helpers.py new file mode 100644 index 0000000..ff6a6a0 --- /dev/null +++ b/api/utils/helpers.py @@ -0,0 +1,61 @@ +import logging + +from pyproj import Transformer + +logger = logging.getLogger(__name__) + +# Define transformer from RD New (EPSG:28992) to ETRS89 (EPSG:4258 = lat/lon) +transformer = Transformer.from_crs("EPSG:28992", "EPSG:4258", always_xy=True) + + +def empty_strings_to_none(d: dict) -> dict: + for key, value in d.items(): + if isinstance(value, str) and value.strip() == "": + d[key] = None + elif isinstance(value, dict): + d[key] = empty_strings_to_none(value) + elif isinstance(value, list): + d[key] = [ + empty_strings_to_none(v) + if isinstance(v, dict) + else (None if v == "" else v) + for v in value + ] + return d + + +def strip_whitespace(data): + if isinstance(data, dict): + return {k: strip_whitespace(v) for k, v in data.items()} + elif isinstance(data, list): + return [strip_whitespace(item) for item in data] + elif isinstance(data, str): + return data.strip() + return data + + +def drop_empty_strings(d: dict) -> dict: # noqa: C901 + cleaned = {} + for key, value in d.items(): + if isinstance(value, str): + if value.strip() == "": + continue # skip this field entirely + cleaned[key] = value + elif isinstance(value, dict): + nested = drop_empty_strings(value) + if nested: # only keep if not empty + cleaned[key] = nested + elif isinstance(value, list): + cleaned_list = [] + for v in value: + if isinstance(v, dict): + nested = drop_empty_strings(v) + if nested: + cleaned_list.append(nested) + elif not (isinstance(v, str) and v.strip() == ""): + cleaned_list.append(v) + if cleaned_list: + cleaned[key] = cleaned_list + else: + cleaned[key] = value + return cleaned From 55aaae700d4c51f0e1341c722f0ff3fe8ce75833 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Sat, 23 May 2026 21:54:05 +0200 Subject: [PATCH 04/11] correct GUF and GPD templates --- .../templates/delete_GPD_AddReport.html | 22 +++++++++---------- .../templates/registration_GPD_AddReport.html | 22 +++++++++---------- .../templates/replace_GPD_AddReport.html | 22 +++++++++---------- .../replace_GUF_StartRegistration.html | 13 ----------- 4 files changed, 33 insertions(+), 46 deletions(-) diff --git a/api/bro_upload/templates/delete_GPD_AddReport.html b/api/bro_upload/templates/delete_GPD_AddReport.html index 4b02ffa..38a35e2 100644 --- a/api/bro_upload/templates/delete_GPD_AddReport.html +++ b/api/bro_upload/templates/delete_GPD_AddReport.html @@ -17,10 +17,10 @@ {{ sourcedocs_data.method }} {% for volume in sourcedocs_data.volume_series %} - + {{ volume.beginDate }} {{ volume.endDate }} - + {{ volume.volumeNumber }} {{ volume.volumeTotalNumber }} {% if volume.temperature %} @@ -28,14 +28,14 @@ {% endif %} {% endfor %} - - - - - {{ sourcedocs_data.groundwaterUsageFacilityBroId }} - - - - + + + + + {{ sourcedocs_data.groundwaterUsageFacilityBroId }} + + + + diff --git a/api/bro_upload/templates/registration_GPD_AddReport.html b/api/bro_upload/templates/registration_GPD_AddReport.html index 697ded4..5d8978c 100644 --- a/api/bro_upload/templates/registration_GPD_AddReport.html +++ b/api/bro_upload/templates/registration_GPD_AddReport.html @@ -16,10 +16,10 @@ {{ sourcedocs_data.method }} {% for volume in sourcedocs_data.volume_series %} - + {{ volume.beginDate }} {{ volume.endDate }} - + {{ volume.waterInOut }} {{ volume.volume }} {% if volume.temperature %} @@ -27,15 +27,15 @@ {% endif %} {% endfor %} - - - - - {{ sourcedocs_data.groundwaterUsageFacilityBroId }} - - - - + + + + + {{ sourcedocs_data.groundwaterUsageFacilityBroId }} + + + + diff --git a/api/bro_upload/templates/replace_GPD_AddReport.html b/api/bro_upload/templates/replace_GPD_AddReport.html index 1827b11..244363b 100644 --- a/api/bro_upload/templates/replace_GPD_AddReport.html +++ b/api/bro_upload/templates/replace_GPD_AddReport.html @@ -17,10 +17,10 @@ {{ sourcedocs_data.method }} {% for volume in sourcedocs_data.volume_series %} - + {{ volume.beginDate }} {{ volume.endDate }} - + {{ volume.volumeNumber }} {{ volume.volumeTotalNumber }} {% if volume.temperature %} @@ -28,14 +28,14 @@ {% endif %} {% endfor %} - - - - - {{ sourcedocs_data.groundwaterUsageFacilityBroId }} - - - - + + + + + {{ sourcedocs_data.groundwaterUsageFacilityBroId }} + + + + diff --git a/api/bro_upload/templates/replace_GUF_StartRegistration.html b/api/bro_upload/templates/replace_GUF_StartRegistration.html index eae8006..1d89bcd 100644 --- a/api/bro_upload/templates/replace_GUF_StartRegistration.html +++ b/api/bro_upload/templates/replace_GUF_StartRegistration.html @@ -30,19 +30,6 @@ onbekend {% endif %} - {% if sourcedocs_data.endTime %} - - {% if sourcedocs_data.endTime|length == 10 %} - {{ sourcedocs_data.endTime }} - {% elif sourcedocs_data.endTime|length == 7 %} - {{ sourcedocs_data.endTime }} - {% elif sourcedocs_data.endTime|length == 4 %} - {{ sourcedocs_data.endTime }} - {% elif sourcedocs_data.endTime == 'onbekend' %} - onbekend - {% endif %} - - {% endif %} From 30e306e75b8acc8d636edd5c2036334706065de2 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Sat, 23 May 2026 21:54:47 +0200 Subject: [PATCH 05/11] extend testing, and add guf and gpd paths to test --- api/tests/fixtures.py | 26 +++++++++++++++++ api/tests/test_object_import.py | 18 ++++++------ api/tests/test_signals.py | 47 ++++++++++++++----------------- gmn/tests/test_gmn_serializers.py | 1 + pyproject.toml | 2 +- 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/api/tests/fixtures.py b/api/tests/fixtures.py index eb1cd01..4667677 100644 --- a/api/tests/fixtures.py +++ b/api/tests/fixtures.py @@ -10,6 +10,8 @@ from gld import models as gld_models from gmn import models as gmn_models from gmw import models as gmw_models +from gpd import models as gpd_models +from guf import models as guf_models TZ_INFO = pytz.timezone("Europe/Amsterdam") @@ -249,3 +251,27 @@ def observation(organisation, gld): observation_type="reguliereMeting", investigator_kvk="12345678", ) + + +@pytest.fixture +def guf(organisation): + return guf_models.GUF.objects.create( + data_owner=organisation, + bro_id="GUF123456789", + quality_regime="IMBRO/A", + delivery_context="waterwet", + identification_licence="LIC-001", + legal_type="melding", + primary_usage_type="bemaling", + human_consumption="nee", + ) + + +@pytest.fixture +def gpd(organisation): + return gpd_models.GPD.objects.create( + data_owner=organisation, + bro_id="GPD123456789", + quality_regime="IMBRO/A", + internal_id="test_gpd", + ) diff --git a/api/tests/test_object_import.py b/api/tests/test_object_import.py index b580c43..2d99b5b 100644 --- a/api/tests/test_object_import.py +++ b/api/tests/test_object_import.py @@ -297,41 +297,41 @@ def test_create_upload_task(api_client, user, organisation): def test_check_dates(): last_import_datetime = datetime.datetime(2021, 6, 15) assert ( - object_import.check_dates(last_import_datetime, "2021-01-01", None, None) + object_import.check_dates(last_import_datetime, "2021-01-01", None, None, None) is False ) # older assert ( - object_import.check_dates(last_import_datetime, "2021-12-31", None, None) + object_import.check_dates(last_import_datetime, "2021-12-31", None, None, None) is True ) # newer assert ( - object_import.check_dates(last_import_datetime, None, "2021-01-01", None) + object_import.check_dates(last_import_datetime, None, "2021-01-01", None, None) is False ) # older assert ( - object_import.check_dates(last_import_datetime, None, "2021-12-31", None) + object_import.check_dates(last_import_datetime, None, "2021-12-31", None, None) is True ) # newer assert ( - object_import.check_dates(last_import_datetime, None, None, "2021-01-01") + object_import.check_dates(last_import_datetime, None, None, "2021-01-01", None) is False ) # older assert ( - object_import.check_dates(last_import_datetime, None, None, "2021-12-31") + object_import.check_dates(last_import_datetime, None, None, "2021-12-31", None) is True ) # newer assert ( object_import.check_dates( - last_import_datetime, "2021-01-01", "2021-02-01", "2021-03-01" + last_import_datetime, "2021-01-01", "2021-02-01", "2021-03-01", None ) is False ) # all older assert ( object_import.check_dates( - last_import_datetime, "2022-01-01", "2010-01-01", None + last_import_datetime, "2022-01-01", "2010-01-01", None, None ) is True ) # one newer assert ( - object_import.check_dates(last_import_datetime, None, None, None) is True + object_import.check_dates(last_import_datetime, None, None, None, None) is True ) # no dates diff --git a/api/tests/test_signals.py b/api/tests/test_signals.py index 3842f89..6915cf6 100644 --- a/api/tests/test_signals.py +++ b/api/tests/test_signals.py @@ -8,6 +8,8 @@ gld, gmn, gmw, + gpd, + guf, intermediate_event, measurement_configuration, measuringpoint, @@ -15,9 +17,13 @@ organisation, tube, ) -from frd.models import FRD, GeoElectricMeasurement, MeasurementConfiguration +from frd.models import ( + FRD, + GeoElectricMeasurement, + MeasurementConfiguration, +) from gld.models import GLD, Observation # Example model to check side effects -from gmn.models import GMN, IntermediateEvent, Measuringpoint +from gmn.models import GMN, Measuringpoint from gmw.models import GMW, Event, MonitoringTube # Example model to check side effects from ..models import Organisation @@ -920,36 +926,25 @@ def test_post_save_uploadtask_triggers_update_objects_frd( @pytest.mark.django_db -def test_post_save_uploadtask_triggers_delete_objects_gld_closure( +def test_post_save_uploadtask_triggers_create_objects_gmn_closure( organisation: Organisation, # noqa: F811 - gld: GLD, # noqa: F811 - observation, # noqa: F811 + gmn: GMN, # noqa: F811 ): - """A delete UploadTask for GLD_Closure removes the matching Observation.""" - assert Observation.objects.filter(gld=gld).count() == 1 - UploadTask.objects.create( data_owner=organisation, - bro_domain="GLD", - registration_type="GLD_Closure", - request_type="delete", + bro_domain="GMN", + registration_type="GMN_Closure", + request_type="registration", status="COMPLETED", - bro_id=gld.bro_id, + bro_id=gmn.bro_id, metadata={}, - sourcedocument_data={"dateToBeCorrected": "2024-01-01"}, + sourcedocument_data={"endDateMonitoring": "2024-01-01"}, ) - assert Observation.objects.filter(gld=gld).count() == 0 - - -@pytest.mark.django_db -def test_post_save_uploadtask_triggers_delete_objects_gmn_closure( - organisation: Organisation, # noqa: F811 - gmn: GMN, # noqa: F811 - intermediate_event, # noqa: F811 -): - """A delete UploadTask for GMN_Closure removes the matching IntermediateEvent.""" - assert IntermediateEvent.objects.filter(gmn=gmn).count() == 1 + assert ( + GMN.objects.get(bro_id=gmn.bro_id).end_date_monitoring.isoformat() + == "2024-01-01" + ) UploadTask.objects.create( data_owner=organisation, @@ -959,10 +954,10 @@ def test_post_save_uploadtask_triggers_delete_objects_gmn_closure( status="COMPLETED", bro_id=gmn.bro_id, metadata={}, - sourcedocument_data={"dateToBeCorrected": "2024-01-01"}, + sourcedocument_data={"endDateMonitoring": "2024-01-01"}, ) - assert IntermediateEvent.objects.filter(gmn=gmn).count() == 0 + assert GMN.objects.get(bro_id=gmn.bro_id).end_date_monitoring is None @pytest.mark.django_db diff --git a/gmn/tests/test_gmn_serializers.py b/gmn/tests/test_gmn_serializers.py index 661666b..a568ef6 100644 --- a/gmn/tests/test_gmn_serializers.py +++ b/gmn/tests/test_gmn_serializers.py @@ -39,6 +39,7 @@ def test_gmn_deserialization(gmn, organisation): "monitoring_purpose": "test", "groundwater_aspect": "test", "start_date_monitoring": date(2000, 1, 1), + "end_date_monitoring": None, "object_registration_time": datetime(2000, 1, 1), "registration_status": "test", "color": "#000fff", diff --git a/pyproject.toml b/pyproject.toml index 244a92b..489df4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ test = ["pytest"] # pytest added by nens-meta py-modules = ["api", "gmn", "gmw", "gld", "gar", "frd"] [tool.pytest.ini_options] -testpaths = ["brostar_api", "api", "gmn", "gmw", "gld", "gar", "frd"] +testpaths = ["brostar_api", "api", "gmn", "gmw", "gld", "gar", "frd", "guf", "gpd"] log_level = "DEBUG" django_find_project = false DJANGO_SETTINGS_MODULE = "brostar_api.settings" From ac0fe1c507cbd2b32513d8e7529a07909b15217a Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Sat, 23 May 2026 21:55:06 +0200 Subject: [PATCH 06/11] improve models --- api/bro_upload/type_helpers.py | 2 +- api/bro_upload/upload_datamodels.py | 20 ++++++++---- .../0027_gmn_end_date_monitoring.py | 17 ++++++++++ gmn/models.py | 1 + ...ll_geometry_publicly_available_and_more.py | 32 +++++++++++++++++++ 5 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 gmn/migrations/0027_gmn_end_date_monitoring.py create mode 100644 guf/migrations/0005_alter_designwell_geometry_publicly_available_and_more.py diff --git a/api/bro_upload/type_helpers.py b/api/bro_upload/type_helpers.py index d76918f..29ac230 100644 --- a/api/bro_upload/type_helpers.py +++ b/api/bro_upload/type_helpers.py @@ -129,5 +129,5 @@ def literal_to_choices(literal_type): IndicationYesNoOptions = Literal["ja", "nee", "onbekend"] WaterInOutOptions = Literal["ingebracht", "onttrokken"] TemperatureOptions = Literal["koud", "warm", "onbekend", None] -PubliclyAvailableOptions = Literal["ja", "nee", "onbekend", None] +PubliclyAvailableOptions = Literal["ja", "nee", None] MethodOptions = Literal["berekening", "watermeter", "onbekend"] diff --git a/api/bro_upload/upload_datamodels.py b/api/bro_upload/upload_datamodels.py index 5e034a7..e6faf64 100644 --- a/api/bro_upload/upload_datamodels.py +++ b/api/bro_upload/upload_datamodels.py @@ -780,8 +780,9 @@ class DesignLoop(CamelModel): loop_type: DesignLoopTypeOptions | None = None # Added: soil loop type # Lifespan is formatted from these two fields - ISO-8601 date string - start_date: str - end_date: str + # Lifespan is not allowed to be present for NewLicense + start_date: str | None + end_date: str | None geometry_type: Literal["Point", "LineString"] = "Point" # Type of geometry @@ -1065,18 +1066,23 @@ def handle_empty_gml_id(cls, v): return v if v is not None and v != "" else f"_{uuid.uuid4()}" +class RealisedScreenChanges(CamelModel): + realised_screen_id: str + top_screen_depth: float # meters + + # Updated GUFHeight class class GUFHeight(CamelModel): """Source document data for GUF_Height""" realised_well_id: str - well_functions: list[WellFunctionOptions] - relative_temperature: RelativeTemperatureOptions | None = None + height: float # meters + well_depth: float | None = None start_validity: str = Field( ..., description="Can be YYYY-MM-DD (10 chars), YYYY-MM (7 chars), or YYYY (4 chars)", ) - realised_installation: RealisedInstallationHeightPart | None = None + realised_screens: list[RealisedScreenChanges] = [] # Updated RealisedScreen class @@ -1224,7 +1230,9 @@ class GUFClosure(CamelModel): class GPDStartRegistration(CamelModel): object_id_accountable_party: str - publicly_available: PubliclyAvailableOptions = None + publicly_available: PubliclyAvailableOptions = ( + None # Only registered for drinkwater + ) class VolumeSeries(CamelModel): diff --git a/gmn/migrations/0027_gmn_end_date_monitoring.py b/gmn/migrations/0027_gmn_end_date_monitoring.py new file mode 100644 index 0000000..febbf99 --- /dev/null +++ b/gmn/migrations/0027_gmn_end_date_monitoring.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.7 on 2026-05-23 19:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gmn", "0026_measuringpoint_monitoring_tube"), + ] + + operations = [ + migrations.AddField( + model_name="gmn", + name="end_date_monitoring", + field=models.DateField(null=True), + ), + ] diff --git a/gmn/models.py b/gmn/models.py index 702048b..ddf7b16 100644 --- a/gmn/models.py +++ b/gmn/models.py @@ -33,6 +33,7 @@ class GMN(models.Model): monitoring_purpose = models.CharField(max_length=100, null=True) groundwater_aspect = models.CharField(max_length=100, null=True) start_date_monitoring = models.DateField(null=True) + end_date_monitoring = models.DateField(null=True) object_registration_time = models.DateTimeField(null=True) registration_status = models.CharField(max_length=50, null=True) color = models.CharField(max_length=7, null=True, blank=True, default=None) diff --git a/guf/migrations/0005_alter_designwell_geometry_publicly_available_and_more.py b/guf/migrations/0005_alter_designwell_geometry_publicly_available_and_more.py new file mode 100644 index 0000000..95bef8a --- /dev/null +++ b/guf/migrations/0005_alter_designwell_geometry_publicly_available_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 5.2.7 on 2026-05-23 19:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("guf", "0004_alter_designinstallation_installation_function_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="designwell", + name="geometry_publicly_available", + field=models.CharField( + blank=True, + choices=[("ja", "Ja"), ("nee", "Nee")], + max_length=100, + null=True, + ), + ), + migrations.AlterField( + model_name="designwell", + name="maximum_well_depth_publicly_available", + field=models.CharField( + blank=True, + choices=[("ja", "Ja"), ("nee", "Nee")], + max_length=100, + null=True, + ), + ), + ] From c749485c0dbda1fe52e83402b038eeb84efe2b84 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Sat, 23 May 2026 21:57:12 +0200 Subject: [PATCH 07/11] update changelog --- CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 41e08f0..815f9b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,10 @@ ## 1.71 (unreleased) -- Nothing changed yet. +- Enhancement [UploadTask]: On succesful task also handle updates and deletes +- Rework [API]: Split utils into multiple files, as it became too large to properly handle +- Bugfix [GPD]: Fix a minor bug in the AddReport template + ## 1.70 (2026-05-12) From c7325efe0d49894eb6f152bb79d67ba7b90157fc Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Mon, 25 May 2026 12:55:13 +0200 Subject: [PATCH 08/11] Add VolumeSeries API pages, improve import --- CHANGES.md | 3 +++ api/bro_import/object_import.py | 18 +++++++++++------- gpd/filters.py | 18 +++++++++++++++++- gpd/serializers.py | 10 +++++++++- gpd/urls.py | 8 ++++++++ gpd/views.py | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 815f9b0..819cecc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,9 +3,12 @@ ## 1.71 (unreleased) +- Feat [GPD]: Add VolumesSeries API pages - Enhancement [UploadTask]: On succesful task also handle updates and deletes +- Enhancement [GPD]: Add filtering on Report and VolumeSeries API pages - Rework [API]: Split utils into multiple files, as it became too large to properly handle - Bugfix [GPD]: Fix a minor bug in the AddReport template +- Bugfix [GPD]: Correct method import to rertrieve text attribute diff --git a/api/bro_import/object_import.py b/api/bro_import/object_import.py index f350d8d..ea006d0 100644 --- a/api/bro_import/object_import.py +++ b/api/bro_import/object_import.py @@ -120,6 +120,12 @@ def __init__(self, bro_id: str, data_owner: Organisation) -> None: self.s.mount("http://", adapter) self.s.mount("https://", adapter) + def _extract_text_from_xml_element(self, element: dict | str | None) -> str | None: + """Extract text from XML elements that may have attributes""" + if isinstance(element, dict): + return element.get("#text") + return element + def should_import(self) -> bool: """Check PDOK API to see if the last_correction_date or the last_addition_date is more recent than the last_import_date""" last_import_task = ( @@ -1474,12 +1480,6 @@ def _parse_flexible_date(self, date_str: str | None) -> datetime.date | None: except (ValueError, AttributeError): return None - def _extract_text_from_xml_element(self, element: dict | str | None) -> str | None: - """Extract text from XML elements that may have attributes""" - if isinstance(element, dict): - return element.get("#text") - return element - def _save_guf_data(self, guf_ppo: dict[str, Any]): # noqa C901 - This function is complex but breaking it down further would reduce readability due to the nested structure of the data. # Extract basic metadata bro_id = guf_ppo.get("brocom:broId") @@ -1868,13 +1868,17 @@ def _save_data_to_database(self, json_data: dict[str, Any]) -> None: # Get GUF reference guf_ref = self._extract_guf_reference(report_obj) + ## Need to extract the text from method, because it can be an XML element with attributes (e.g. {"#text": "value", "@codeSpace": "someURI"}) + method = report_obj.get("gpdcommon:method", "onbekend") + method_value = self._extract_text_from_xml_element(method) or "onbekend" + # Create or update report report_model, created = Report.objects.update_or_create( gpd=gpd_obj, report_id=report_obj.get("gpdcommon:reportId"), data_owner=self.data_owner, defaults={ - "method": report_obj.get("gpdcommon:method", "onbekend"), + "method": method_value, "begin_date": datetime.datetime.fromisoformat(report_begin).date() if report_begin else None, diff --git a/gpd/filters.py b/gpd/filters.py index 005fd9e..e797989 100644 --- a/gpd/filters.py +++ b/gpd/filters.py @@ -4,7 +4,7 @@ from api.mixins import DateTimeFilterMixin -from .models import GPD +from .models import GPD, Report, VolumeSeries class GpdFilter(DateTimeFilterMixin, FilterSet): @@ -18,3 +18,19 @@ class Meta: "filter_class": CharFilter, }, } + + +class ReportFilter(DateTimeFilterMixin, FilterSet): + report_id__icontains = filters.CharFilter( + field_name="report_id", lookup_expr="icontains" + ) + + class Meta: + model = Report + fields = "__all__" + + +class VolumeSeriesFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = VolumeSeries + fields = "__all__" diff --git a/gpd/serializers.py b/gpd/serializers.py index 0c09ecb..3fb6d05 100644 --- a/gpd/serializers.py +++ b/gpd/serializers.py @@ -1,7 +1,7 @@ from rest_framework import serializers from api.mixins import RequiredFieldsMixin, UrlFieldMixin -from gpd.models import GPD, Report +from gpd.models import GPD, Report, VolumeSeries class GPDSerializer(UrlFieldMixin, RequiredFieldsMixin, serializers.ModelSerializer): @@ -25,3 +25,11 @@ class ReportSerializer(UrlFieldMixin, RequiredFieldsMixin, serializers.ModelSeri class Meta: model = Report fields = "__all__" + + +class VolumeSeriesSerializer( + UrlFieldMixin, RequiredFieldsMixin, serializers.ModelSerializer +): + class Meta: + model = VolumeSeries + fields = "__all__" diff --git a/gpd/urls.py b/gpd/urls.py index 318e5b5..ab705c6 100644 --- a/gpd/urls.py +++ b/gpd/urls.py @@ -12,4 +12,12 @@ path( "reports//", views.ReportDetailView.as_view(), name="report-detail" ), + path( + "volume-series/", views.VolumeSeriesListView.as_view(), name="volumeseries-list" + ), + path( + "volume-series//", + views.VolumeSeriesDetailView.as_view(), + name="volumeseries-detail", + ), ] diff --git a/gpd/views.py b/gpd/views.py index 7523229..0e3459f 100644 --- a/gpd/views.py +++ b/gpd/views.py @@ -66,6 +66,7 @@ class ReportListView(mixins.UserOrganizationMixin, generics.ListAPIView): serializer_class = serializers.ReportSerializer queryset = gpd_models.Report.objects.all().order_by("-created") + filterset_class = filters.ReportFilter filter_backends = [DjangoFilterBackend] ordering = ["id"] @@ -84,3 +85,35 @@ class ReportDetailView(mixins.UserOrganizationMixin, generics.RetrieveAPIView): queryset = gpd_models.Report.objects.all() serializer_class = serializers.ReportSerializer lookup_field = "uuid" + + +class VolumeSeriesListView(mixins.UserOrganizationMixin, generics.ListAPIView): + """ + API view to retrieve a list of VolumeSeries for the user's organization. + + Returns: + list: List of VolumeSeries ordered by creation date (descending). + """ + + serializer_class = serializers.VolumeSeriesSerializer + queryset = gpd_models.VolumeSeries.objects.all().order_by("-created") + + filterset_class = filters.VolumeSeriesFilter + filter_backends = [DjangoFilterBackend] + ordering = ["id"] + + +class VolumeSeriesDetailView(mixins.UserOrganizationMixin, generics.RetrieveAPIView): + """ + API view to retrieve the details of a single VolumeSeries by UUID. + + Args: + uuid (str): UUID of the VolumeSeries. + + Returns: + VolumeSeries: Detailed information about the specified VolumeSeries. + """ + + queryset = gpd_models.VolumeSeries.objects.all() + serializer_class = serializers.VolumeSeriesSerializer + lookup_field = "uuid" From e1a650b662bf4db59e066b55cecc987ca937b514 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Mon, 25 May 2026 16:49:21 +0200 Subject: [PATCH 09/11] improve guf import, urls and filtering --- api/bro_import/object_import.py | 193 ++++++++++++++++++++++++++++---- guf/filters.py | 76 ++++++++++++- guf/urls.py | 26 +++-- guf/views.py | 6 + 4 files changed, 268 insertions(+), 33 deletions(-) diff --git a/api/bro_import/object_import.py b/api/bro_import/object_import.py index ea006d0..c1d431e 100644 --- a/api/bro_import/object_import.py +++ b/api/bro_import/object_import.py @@ -1,4 +1,5 @@ import datetime +import importlib import logging import time import xml.etree.ElementTree as ET @@ -120,12 +121,33 @@ def __init__(self, bro_id: str, data_owner: Organisation) -> None: self.s.mount("http://", adapter) self.s.mount("https://", adapter) + def _to_float(value: Any, default: float = 0.0) -> float: + if value is None: + return default + if isinstance(value, dict): + value = value.get("#text") + try: + return float(value) + except (TypeError, ValueError): + return default + def _extract_text_from_xml_element(self, element: dict | str | None) -> str | None: """Extract text from XML elements that may have attributes""" if isinstance(element, dict): return element.get("#text") return element + def _extract_flexible_date_value(self, value: Any) -> str | None: + if isinstance(value, dict): + return ( + value.get("brocom:date") + or value.get("brocom:yearMonth") + or value.get("brocom:year") + ) + if isinstance(value, str): + return value + return None + def should_import(self) -> bool: """Check PDOK API to see if the last_correction_date or the last_addition_date is more recent than the last_import_date""" last_import_task = ( @@ -840,7 +862,7 @@ def _save_data_to_database(self, json_data: dict[str, Any]) -> None: return gar_data = dispatch_document_data.get("GAR_O") - monitoring_point_data = gar_data.get("monitoringPoint", None).get( + monitoring_point_data = gar_data.get("monitoringPoint").get( "garcommon:GroundwaterMonitoringTube", None ) field_research_data = gar_data.get("fieldResearch", None) @@ -1465,20 +1487,39 @@ def _save_data_to_database(self, json_data: dict[str, Any]) -> None: # Process licence and nested installations/wells self._save_installations_and_wells(guf_ppo, guf_obj) - def _parse_flexible_date(self, date_str: str | None) -> datetime.date | None: - """Parse BRO dates with flexible granularity (YYYY, YYYY-MM, YYYY-MM-DD)""" - if not date_str: - return None + def _resolve_guf_upload_model( + self, source_document: dict[str, Any] + ) -> tuple[type | None, str | None, dict[str, Any]]: + if not isinstance(source_document, dict) or not source_document: + return None, None, {} + + parent_key = next( + ( + key + for key in source_document.keys() + if self._strip_namespace(key).startswith("GUF_") + ), + None, + ) + if not parent_key: + return None, None, {} + + parent_type = self._strip_namespace( + parent_key + ) # e.g. GUF_AddRealisedInstallation + model_name = parent_type.replace("_", "") # e.g. GUFAddRealisedInstallation + parent_payload = source_document.get(parent_key, {}) or {} + try: - if len(date_str) == 4: # YYYY - return datetime.date(int(date_str), 1, 1) - elif len(date_str) == 7: # YYYY-MM - year, month = date_str.split("-") - return datetime.date(int(year), int(month), 1) - else: # YYYY-MM-DD - return datetime.datetime.fromisoformat(date_str).date() - except (ValueError, AttributeError): - return None + upload_models = importlib.import_module("api.bro_upload.upload_datamodels") + model_cls = getattr(upload_models, model_name, None) + except Exception: + model_cls = None + + if isinstance(parent_payload, list): + parent_payload = parent_payload[0] if parent_payload else {} + + return model_cls, model_name, parent_payload def _save_guf_data(self, guf_ppo: dict[str, Any]): # noqa C901 - This function is complex but breaking it down further would reduce readability due to the nested structure of the data. # Extract basic metadata @@ -1494,13 +1535,7 @@ def _save_guf_data(self, guf_ppo: dict[str, Any]): # noqa C901 - This function # Extract lifespan with flexible date handling lifespan = guf_ppo.get("lifespan", {}) start_time_data = lifespan.get("gufcommon:startTime", {}) - start_time_str = None - if start_time_data.get("brocom:date"): - start_time_str = start_time_data.get("brocom:date") - elif start_time_data.get("brocom:yearMonth"): - start_time_str = start_time_data.get("brocom:yearMonth") - elif start_time_data.get("brocom:year"): - start_time_str = start_time_data.get("brocom:year") + start_time_str = self._extract_flexible_date_value(start_time_data) # Extract licence data for GUF fields. # The XML may contain multiple elements, which xmltodict parses as a list. @@ -1580,6 +1615,120 @@ def _save_guf_data(self, guf_ppo: dict[str, Any]): # noqa C901 - This function return guf_obj + def _build_add_realised_installation_payload( + self, parent_payload: dict[str, Any] + ) -> dict[str, Any]: + geometry = parent_payload.get("gufcommon:geometry", {}) + point = geometry.get("gml:Point", {}) + realised_loop_pos = point.get("gml:pos") + + installation_function = self._extract_text_from_xml_element( + parent_payload.get("gufcommon:installationFunction") + ) + realised_installation_id = parent_payload.get( + "gufcommon:realisedInstallationId" + ) + + start_validity = ( + self._extract_flexible_date_value( + parent_payload.get("gufcommon:validityPeriod", {}).get( + "gufcommon:startValidity" + ) + ) + or self._extract_flexible_date_value( + parent_payload.get("gufcommon:startTime") + ) + or "1900-01-01" + ) + + realised_wells_payload = [] + raw_wells = self._as_list(parent_payload.get("gufcommon:realisedWell", [])) + for raw_well in raw_wells: + well = raw_well.get("gufcommon:RealisedWell", raw_well) + if not isinstance(well, dict): + continue + + well_geometry = well.get("gufcommon:geometry", {}) + well_point = well_geometry.get("gml:Point", {}) + well_pos = well_point.get("gml:pos") + + raw_well_functions = self._as_list(well.get("gufcommon:wellFunction", [])) + well_functions = [ + self._extract_text_from_xml_element(wf) + for wf in raw_well_functions + if self._extract_text_from_xml_element(wf) + ] + if not well_functions: + well_functions = ["onttrekking"] + + raw_screens = self._as_list(well.get("gufcommon:realisedScreen", [])) + realised_screens = [] + for raw_screen in raw_screens: + screen = raw_screen.get("gufcommon:RealisedScreen", raw_screen) + if not isinstance(screen, dict): + continue + + realised_screens.append( + { + "realised_screen_id": screen.get("gufcommon:realisedScreenId"), + "screen_type": self._extract_text_from_xml_element( + screen.get("gufcommon:screenType") + ) + or "onbekend", + "top_screen_depth": self._to_float( + screen.get("gufcommon:topScreenDepth") + ), + "length": self._to_float(screen.get("gufcommon:length")), + } + ) + + realised_wells_payload.append( + { + "realised_well_id": well.get("gufcommon:realisedWellId"), + "well_functions": well_functions, + "height": self._to_float(well.get("gufcommon:height")), + "well_depth": self._to_float(well.get("gufcommon:wellDepth")), + "wellPos": well_pos or "", + "relative_temperature": self._extract_text_from_xml_element( + well.get("gufcommon:relativeTemperature") + ), + "realised_screens": realised_screens, + } + ) + + return { + "realised_installation_id": realised_installation_id, + "installation_function": installation_function, + "realised_loop_pos": realised_loop_pos or "", + "start_validity": start_validity, + "realised_wells": realised_wells_payload, + } + + def _build_guf_sourcedocument_data( + self, event_data: dict[str, Any] + ) -> dict[str, Any]: + source_document = event_data.get("sourceDocument", {}) or {} + model_cls, model_name, parent_payload = self._resolve_guf_upload_model( + source_document + ) + + if not model_cls: + return source_document + + if model_name == "GUFAddRealisedInstallation": + payload = self._build_add_realised_installation_payload(parent_payload) + else: + payload = parent_payload + + try: + validated = model_cls.model_validate(payload) + return validated.model_dump(by_alias=True, exclude_none=True) + except Exception as exc: + logger.info( + f"GUF sourcedocument model validation failed for {model_name}: {exc}" + ) + return source_document + def _save_guf_events(self, guf_ppo: dict[str, Any], guf_obj: GUF) -> None: # Extract events from objectHistory object_history = guf_ppo.get("objectHistory", {}) @@ -1603,7 +1752,7 @@ def _save_guf_events(self, guf_ppo: dict[str, Any], guf_obj: GUF) -> None: } # Sourcedocument data (entire event structure) - sourcedocument_data = event_data.get("sourceDocument", {}) + sourcedocument_data = self._build_guf_sourcedocument_data(event_data) GUFEvent.objects.update_or_create( guf=guf_obj, diff --git a/guf/filters.py b/guf/filters.py index 432f380..76ea325 100644 --- a/guf/filters.py +++ b/guf/filters.py @@ -4,7 +4,15 @@ from api.mixins import DateTimeFilterMixin -from .models import GUF +from .models import ( + GUF, + DesignInstallation, + DesignLoop, + DesignSurfaceInfiltration, + DesignWell, + EnergyCharacteristics, + GUFEvent, +) class GufFilter(DateTimeFilterMixin, FilterSet): @@ -18,3 +26,69 @@ class Meta: "filter_class": CharFilter, }, } + + +class DesignInstallationFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = DesignInstallation + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } + + +class DesignLoopFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = DesignLoop + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } + + +class DesignWellFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = DesignWell + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } + + +class DesignSurfaceInfiltrationFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = DesignSurfaceInfiltration + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } + + +class GUFEventFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = GUFEvent + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } + + +class EnergyCharacteristicsFilter(DateTimeFilterMixin, FilterSet): + class Meta: + model = EnergyCharacteristics + fields = "__all__" + filter_overrides = { + JSONField: { + "filter_class": CharFilter, + }, + } diff --git a/guf/urls.py b/guf/urls.py index 2322d9a..7ab31cf 100644 --- a/guf/urls.py +++ b/guf/urls.py @@ -13,38 +13,44 @@ path( "installations/", views.DesignInstallationListView.as_view(), - name="installation-list", + name="designinstallation-list", ), path( "installations//", views.DesignInstallationDetailView.as_view(), - name="installation-detail", + name="designinstallation-detail", ), # Design Loop endpoints - path("loops/", views.DesignLoopListView.as_view(), name="loop-list"), + path("loops/", views.DesignLoopListView.as_view(), name="designloop-list"), path( - "loops//", views.DesignLoopDetailView.as_view(), name="loop-detail" + "loops//", + views.DesignLoopDetailView.as_view(), + name="designloop-detail", ), # Design Well endpoints - path("wells/", views.DesignWellListView.as_view(), name="well-list"), + path("wells/", views.DesignWellListView.as_view(), name="designwell-list"), path( - "wells//", views.DesignWellDetailView.as_view(), name="well-detail" + "wells//", + views.DesignWellDetailView.as_view(), + name="designwell-detail", ), # Design Surface Infiltration endpoints path( "surfaceinfiltrations/", views.DesignSurfaceInfiltrationListView.as_view(), - name="surfaceinfiltration-list", + name="designsurfaceinfiltration-list", ), path( "surfaceinfiltrations//", views.DesignSurfaceInfiltrationDetailView.as_view(), - name="surfaceinfiltration-detail", + name="designsurfaceinfiltration-detail", ), # GUF Event endpoints - path("events/", views.GUFEventListView.as_view(), name="event-list"), + path("events/", views.GUFEventListView.as_view(), name="gufevent-list"), path( - "events//", views.GUFEventDetailView.as_view(), name="event-detail" + "events//", + views.GUFEventDetailView.as_view(), + name="gufevent-detail", ), # Energy Characteristics endpoints path( diff --git a/guf/views.py b/guf/views.py index 02b376c..23600e7 100644 --- a/guf/views.py +++ b/guf/views.py @@ -67,6 +67,7 @@ class DesignInstallationListView(mixins.UserOrganizationMixin, generics.ListAPIV serializer_class = serializers.DesignInstallationSerializer queryset = guf_models.DesignInstallation.objects.all().order_by("-created") + filterset_class = filters.DesignInstallationFilter filter_backends = [DjangoFilterBackend] ordering = ["id"] @@ -99,6 +100,7 @@ class DesignLoopListView(mixins.UserOrganizationMixin, generics.ListAPIView): queryset = guf_models.DesignLoop.objects.all().order_by("-created") filter_backends = [DjangoFilterBackend] + filterset_class = filters.DesignLoopFilter ordering = ["id"] @@ -122,6 +124,7 @@ class DesignWellListView(mixins.UserOrganizationMixin, generics.ListAPIView): queryset = guf_models.DesignWell.objects.all().order_by("-created") filter_backends = [DjangoFilterBackend] + filterset_class = filters.DesignWellFilter ordering = ["id"] @@ -147,6 +150,7 @@ class DesignSurfaceInfiltrationListView( queryset = guf_models.DesignSurfaceInfiltration.objects.all().order_by("-created") filter_backends = [DjangoFilterBackend] + filterset_class = filters.DesignSurfaceInfiltrationFilter ordering = ["id"] @@ -172,6 +176,7 @@ class GUFEventListView(mixins.UserOrganizationMixin, generics.ListAPIView): queryset = guf_models.GUFEvent.objects.all().order_by("-created") filter_backends = [DjangoFilterBackend] + filterset_class = filters.GUFEventFilter ordering = ["id"] @@ -195,6 +200,7 @@ class EnergyCharacteristicsListView(mixins.UserOrganizationMixin, generics.ListA queryset = guf_models.EnergyCharacteristics.objects.all().order_by("-created") filter_backends = [DjangoFilterBackend] + filterset_class = filters.EnergyCharacteristicsFilter ordering = ["id"] From f936eb4cddc082ef5deba2ece803015492bcbc9c Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Wed, 27 May 2026 15:21:12 +0200 Subject: [PATCH 10/11] improve whitespace correction --- api/tests/test_views.py | 56 +++++++++++++++++++++++++++++++++++++++++ api/views.py | 46 ++++++++++++--------------------- 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/api/tests/test_views.py b/api/tests/test_views.py index 5489b01..8cdbfca 100644 --- a/api/tests/test_views.py +++ b/api/tests/test_views.py @@ -219,6 +219,62 @@ def test_uploadtask_view_post_valid_data(api_client, user, userprofile, organisa assert response_data["data_owner"] == str(organisation.uuid) +@pytest.mark.django_db +def test_uploadtask_view_post_whitespace_correction( + api_client, user, userprofile, organisation +): + """Test posting on the uploadtask enpoint + Note: userprofile needs to be used as fixture for this test + """ + api_client.force_authenticate(user=user) + url = "/api/uploadtasks/" + + data = { + "bro_domain": "GMN", + "project_number": "1", + "registration_type": "GMN_StartRegistration", + "request_type": "registration", + "metadata": { + "requestReference": "test", + "deliveryAccountableParty": "12345678", + "qualityRegime": "IMBRO", + }, + "sourcedocument_data": { + "objectIdAccountableParty": "test", + "name": "test", + "deliveryContext": " kaderrichtlijnWater", + "monitoringPurpose": "strategischBeheerKwaliteitRegionaal", + "groundwaterAspect": "kwantiteit ", + "startDateMonitoring": "2024-01-01", + "measuringPoints": [ + { + "measuringPointCode": "GMW000000038946 ", + "broId": "GMW000000038946", + "tubeNumber": "1", + } + ], + }, + "data_owner": organisation.uuid, + } + + response = api_client.post(url, data, format="json") + + assert response.status_code == status.HTTP_201_CREATED + + # Additional check to assert the returned data + response_data = response.json() + + # Check if whitespace is stripped in the returned data + assert ( + response_data["sourcedocument_data"]["deliveryContext"] == "kaderrichtlijnWater" + ) + assert response_data["sourcedocument_data"]["groundwaterAspect"] == "kwantiteit" + assert ( + response_data["sourcedocument_data"]["measuringPoints"][0]["measuringPointCode"] + == "GMW000000038946" + ) + + @pytest.mark.django_db def test_uploadtask_view_post_valid_data2(api_client, user, userprofile, organisation): """Test posting on the uploadtask enpoint diff --git a/api/views.py b/api/views.py index c909fa0..c4a91ea 100644 --- a/api/views.py +++ b/api/views.py @@ -348,40 +348,26 @@ def create(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpRespons return Response({"detail": errors}, status=status.HTTP_400_BAD_REQUEST) # Validate the sourcedocument_data input + raw_sourcedocument_data = serializer.validated_data["sourcedocument_data"] + sourcedocument_data = strip_whitespace( + drop_empty_strings(raw_sourcedocument_data) + ) + validation_class = registration_type_datamodel_mapping.get( serializer.validated_data["registration_type"] ) - logger.info(f"Validation_class: {validation_class}") if validation_class: - try: - # For GLD addition, some uuids are backfilled into the sourcedocs data - if ( - serializer.validated_data["registration_type"] == "GLD_Addition" - or "GUF" in serializer.validated_data["registration_type"] - ): - validated_sourcedocument_data = validation_class( - **serializer.validated_data["sourcedocument_data"] - ) - - # Update sourcedocument_data with validated data, including any modifications (like the UUID generation) - serializer.validated_data["sourcedocument_data"] = ( - validated_sourcedocument_data.model_dump(by_alias=True) - ) - # Else, just a pydantic validation is required - else: - sourcedocument_data = strip_whitespace( - drop_empty_strings( - serializer.validated_data["sourcedocument_data"] - ) - ) - validation_class(**sourcedocument_data) - serializer.validated_data["sourcedocument_data"] = ( - sourcedocument_data - ) - - except ValidationError as e: - errors = utils.simplify_validation_errors(e.errors()) - return Response({"detail": errors}, status=status.HTTP_400_BAD_REQUEST) + if ( + serializer.validated_data["registration_type"] == "GLD_Addition" + or "GUF" in serializer.validated_data["registration_type"] + ): + validated_sourcedocument_data = validation_class(**sourcedocument_data) + serializer.validated_data["sourcedocument_data"] = ( + validated_sourcedocument_data.model_dump(by_alias=True) + ) + else: + validation_class(**sourcedocument_data) + serializer.validated_data["sourcedocument_data"] = sourcedocument_data # Accessing the authenticated user's organization user_profile = models.UserProfile.objects.get(user=request.user) From 1316af0930a57a855425013ae4c7e19790f9cc25 Mon Sep 17 00:00:00 2001 From: StevenHosper Date: Wed, 27 May 2026 15:25:59 +0200 Subject: [PATCH 11/11] Improved GAR-Bulk handling --- CHANGES.md | 3 + api/bro_upload/config.py | 1292 +++++++++++++++++++---------- api/bro_upload/gar_bulk_upload.py | 352 +++++--- api/tests/test_gar_bulk_upload.py | 154 +++- 4 files changed, 1236 insertions(+), 565 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 819cecc..1458535 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,10 +5,13 @@ - Feat [GPD]: Add VolumesSeries API pages - Enhancement [UploadTask]: On succesful task also handle updates and deletes +- Enhancement [UploadTask]: Better handling of leading or trailing whitespace - Enhancement [GPD]: Add filtering on Report and VolumeSeries API pages +- Enhancement [GAR-Bulk]: Swapped matching style to use polars for faster handling - Rework [API]: Split utils into multiple files, as it became too large to properly handle - Bugfix [GPD]: Fix a minor bug in the AddReport template - Bugfix [GPD]: Correct method import to rertrieve text attribute +- Bugfix [GAR-Bulk]: Improved handling and coupling of lab and field values diff --git a/api/bro_upload/config.py b/api/bro_upload/config.py index 84bfd2c..a58b5e8 100644 --- a/api/bro_upload/config.py +++ b/api/bro_upload/config.py @@ -1,57 +1,70 @@ -FIELD_PARAMETER_OPTIONS = { - "pH": { +FIELD_PARAMETER_OPTIONS = [ + { + "code": "pH", "parameter_id": 1398, "unit": "1", }, - "Zuurstof (mg/l)": { + { + "code": "Zuurstof (mg/l)", "parameter_id": 1701, "unit": "mg/l", }, - "Geleidbaarheid (mS/m)": { + { + "code": "Geleidbaarheid (mS/m)", "parameter_id": 3548, "unit": "mS/m", }, - "Temperatuur (°C)": { + { + "code": "Temperatuur (°C)", "parameter_id": 1522, "unit": "Cel", }, - "Troebelheid (NTU)": { + { + "code": "Troebelheid (NTU)", "parameter_id": 2031, "unit": "[NTU]", }, - "Alkaliniteit (HCO3 - mg/l)": { + { + "code": "Alkaliniteit (HCO3 - mg/l)", "parameter_id": 374, "unit": "mg/l", }, - "Chloride (mg/l)": { + { + "code": "Chloride (mg/l)", "parameter_id": 508, "unit": "mg/l", }, -} +] -DAWACO_GAR_FIELD_DATA_MAPPING = { - "T": { +DAWACO_GAR_FIELD_DATA_MAPPING = [ + { + "code": "T", "parameter_id": 1522, "unit": "Cel", }, - "pH": { + { + "code": "pH", "parameter_id": 1398, "unit": "1", }, - "GELDHD": { + { + "code": "GELDHD", "parameter_id": 3548, "unit": "mS/m", }, - "HCO3": { + { + "code": "HCO3", "parameter_id": 374, "unit": "mg/l", }, - "O2": { + { + "code": "O2", "parameter_id": 1701, "unit": "mg/l", }, -} +] + # This list was automatically created by using the 3rd tab in a lab research excel file. # Save the aquacode, validationmethod and analyticalTechnique columns as csv. @@ -60,2480 +73,2879 @@ # is used to fill in the parameter_ids and units. # Some parameters from the lab Excel were not found in the BRO list. These have been removed. -LAB_PARAMETER_OPTIONS = { - "FRD-903": { +LAB_PARAMETER_OPTIONS = [ + { + "code": "FRD-903", "parameter_id": 5741, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "134DClFy3C1y": { + { + "code": "134DClFy3C1y", "parameter_id": 2817, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "134DClFyurum": { + { + "code": "134DClFyurum", "parameter_id": 2818, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "14iC3yFyurum": { + { + "code": "14iC3yFyurum", "parameter_id": 4944, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "11DClC2a": { + { + "code": "11DClC2a", "parameter_id": 8, "unit": "ug/l", "validationMethod": "I15680.03", "analyticalTechnique": "GC-MS", }, - "t1011DHOx101": { + { + "code": "t1011DHOx101", "parameter_id": 5111, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "etnetDol": { + { + "code": "etnetDol", "parameter_id": 2294, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "17bestDol": { + { + "code": "17bestDol", "parameter_id": 2314, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "H-PFC10asfzr": { + { + "code": "H-PFC10asfzr", "parameter_id": 5830, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "H-PFC12asfzr": { + { + "code": "H-PFC12asfzr", "parameter_id": 5831, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "H-PFC6asfzr": { + { + "code": "H-PFC6asfzr", "parameter_id": 5996, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "2PFC6yC2a1sf": { + { + "code": "2PFC6yC2a1sf", "parameter_id": 5517, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "234TClFol": { + { + "code": "234TClFol", "parameter_id": 83, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "235TClFol": { + { + "code": "235TClFol", "parameter_id": 86, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "236TClFol": { + { + "code": "236TClFol", "parameter_id": 89, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "245TClFol": { + { + "code": "245TClFol", "parameter_id": 101, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "245T": { + { + "code": "245T", "parameter_id": 102, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "246TClFol": { + { + "code": "246TClFol", "parameter_id": 106, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "s2425DClAn": { + { + "code": "s2425DClAn", "parameter_id": 108, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "GC-MS", }, - "24D": { + { + "code": "24D", "parameter_id": 116, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "24DDD": { + { + "code": "24DDD", "parameter_id": 111, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "24DDT": { + { + "code": "24DDT", "parameter_id": 113, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "24DClFol": { + { + "code": "24DClFol", "parameter_id": 115, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "24DB": { + { + "code": "24DB", "parameter_id": 117, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "24DC1yFol": { + { + "code": "24DC1yFol", "parameter_id": 124, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "GC-MS", }, - "24DNO2Fol": { + { + "code": "24DNO2Fol", "parameter_id": 669, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "26DClBenAd": { + { + "code": "26DClBenAd", "parameter_id": 135, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "26xyldne": { + { + "code": "26xyldne", "parameter_id": 2944, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "GC-MS", }, - "2Aoactfnn": { + { + "code": "2Aoactfnn", "parameter_id": 2950, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "2HOxatzne": { + { + "code": "2HOxatzne", "parameter_id": 2999, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DNOC": { + { + "code": "DNOC", "parameter_id": 668, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "2NO2Fol": { + { + "code": "2NO2Fol", "parameter_id": 170, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "345TClFol": { + { + "code": "345TClFol", "parameter_id": 177, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "44DDE": { + { + "code": "44DDE", "parameter_id": 216, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "42monoPAP": { + { + "code": "42monoPAP", "parameter_id": 6453, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "clxlnl": { + { + "code": "clxlnl", "parameter_id": 3486, "unit": "ug/l", "validationMethod": "D12673.99", "analyticalTechnique": "GC-MS", }, - "4ClFol": { + { + "code": "4ClFol", "parameter_id": 229, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "4CPA": { + { + "code": "4CPA", "parameter_id": 2412, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "4NO2Fol": { + { + "code": "4NO2Fol", "parameter_id": 235, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "bisPFC10yPO4": { + { + "code": "bisPFC10yPO4", "parameter_id": 5998, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "abmtne": { + { + "code": "abmtne", "parameter_id": 2304, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "acsfmeK": { + { + "code": "acsfmeK", "parameter_id": 3131, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "actmpd": { + { + "code": "actmpd", "parameter_id": 3135, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "acnfn": { + { + "code": "acnfn", "parameter_id": 3127, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "alDcSO": { + { + "code": "alDcSO", "parameter_id": 267, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "aedsfn": { + { + "code": "aedsfn", "parameter_id": 269, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Al": { + { + "code": "Al", "parameter_id": 284, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "amttdn": { + { + "code": "amttdn", "parameter_id": 5242, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "amdTzinzr": { + { + "code": "amdTzinzr", "parameter_id": 3165, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Aoprld": { + { + "code": "Aoprld", "parameter_id": 5561, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "amsbm": { + { + "code": "amsbm", "parameter_id": 5137, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "amtl": { + { + "code": "amtl", "parameter_id": 287, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "LC-MS-MS", }, - "NH4": { + { + "code": "NH4", "parameter_id": 289, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "AMPA": { + { + "code": "AMPA", "parameter_id": 292, "unit": "ug/l", "validationMethod": "I16308.17", "analyticalTechnique": "LC-MS-MS", }, - "antcnn": { + { + "code": "antcnn", "parameter_id": 300, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Sb": { + { + "code": "Sb", "parameter_id": 301, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "As": { + { + "code": "As", "parameter_id": 310, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "aslm": { + { + "code": "aslm", "parameter_id": 323, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "atnll": { + { + "code": "atnll", "parameter_id": 3186, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "atzne": { + { + "code": "atzne", "parameter_id": 325, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "azsfrn": { + { + "code": "azsfrn", "parameter_id": 5028, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C2yazfs": { + { + "code": "C2yazfs", "parameter_id": 327, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "C1yazfs": { + { + "code": "C1yazfs", "parameter_id": 328, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "aztmcne": { + { + "code": "aztmcne", "parameter_id": 3198, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "azoxsbn": { + { + "code": "azoxsbn", "parameter_id": 3196, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Ba": { + { + "code": "Ba", "parameter_id": 333, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "benlxl": { + { + "code": "benlxl", "parameter_id": 336, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "benDocb": { + { + "code": "benDocb", "parameter_id": 337, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "benfn": { + { + "code": "benfn", "parameter_id": 3213, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "bentzn": { + { + "code": "bentzn", "parameter_id": 340, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "bentavlcbiC3": { + { + "code": "bentavlcbiC3", "parameter_id": 3218, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Be": { + { + "code": "Be", "parameter_id": 362, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "bedsfn": { + { + "code": "bedsfn", "parameter_id": 365, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "bezafbt": { + { + "code": "bezafbt", "parameter_id": 2463, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "bfnx": { + { + "code": "bfnx", "parameter_id": 375, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "biftn": { + { + "code": "biftn", "parameter_id": 376, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "bispll": { + { + "code": "bispll", "parameter_id": 3266, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "bittnl": { + { + "code": "bittnl", "parameter_id": 3268, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "bixfn": { + { + "code": "bixfn", "parameter_id": 5077, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "B": { + { + "code": "B", "parameter_id": 386, "unit": "ug/l", "validationMethod": "I11885.98", "analyticalTechnique": "ICP-MS", }, - "boscld": { + { + "code": "boscld", "parameter_id": 3272, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "bromcl": { + { + "code": "bromcl", "parameter_id": 390, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Br": { + { + "code": "Br", "parameter_id": 391, "unit": "mg/l", "validationMethod": "I10304-1.09", "analyticalTechnique": "IC ", }, - "BrOxnl": { + { + "code": "BrOxnl", "parameter_id": 394, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "buprmt": { + { + "code": "buprmt", "parameter_id": 403, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Cd": { + { + "code": "Cd", "parameter_id": 441, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "caffine": { + { + "code": "caffine", "parameter_id": 442, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Ca": { + { + "code": "Ca", "parameter_id": 447, "unit": "mg/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "carbmzpne": { + { + "code": "carbmzpne", "parameter_id": 3380, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "carbrl": { + { + "code": "carbrl", "parameter_id": 458, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "carbdzm": { + { + "code": "carbdzm", "parameter_id": 460, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "carbtAd": { + { + "code": "carbtAd", "parameter_id": 3384, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "carftznC2y": { + { + "code": "carftznC2y", "parameter_id": 3386, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Clafncl": { + { + "code": "Clafncl", "parameter_id": 2497, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "chloratnlpl": { + { + "code": "chloratnlpl", "parameter_id": 5175, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Clpfm": { + { + "code": "Clpfm", "parameter_id": 492, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "C2yClprfs": { + { + "code": "C2yClprfs", "parameter_id": 494, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "ClT4ccne": { + { + "code": "ClT4ccne", "parameter_id": 3481, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Cltlnl": { + { + "code": "Cltlnl", "parameter_id": 497, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Cltlrn": { + { + "code": "Cltlrn", "parameter_id": 503, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Clidzn": { + { + "code": "Clidzn", "parameter_id": 507, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Cl": { + { + "code": "Cl", "parameter_id": 508, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "Cr": { + { + "code": "Cr", "parameter_id": 517, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "cipfxcne": { + { + "code": "cipfxcne", "parameter_id": 3436, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "clartmcne": { + { + "code": "clartmcne", "parameter_id": 3448, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "clodnfppgl": { + { + "code": "clodnfppgl", "parameter_id": 3466, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "clofbnzr": { + { + "code": "clofbnzr", "parameter_id": 2464, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "clomzn": { + { + "code": "clomzn", "parameter_id": 3468, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "clopdl": { + { + "code": "clopdl", "parameter_id": 6136, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "cloprld": { + { + "code": "cloprld", "parameter_id": 526, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "clotandne": { + { + "code": "clotandne", "parameter_id": 3472, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "clotmzl": { + { + "code": "clotmzl", "parameter_id": 3473, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "crotmtn": { + { + "code": "crotmtn", "parameter_id": 6451, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "CNazne": { + { + "code": "CNazne", "parameter_id": 534, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cyazfAd": { + { + "code": "cyazfAd", "parameter_id": 3528, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cycffAd": { + { + "code": "cycffAd", "parameter_id": 2478, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "cycxdm": { + { + "code": "cycxdm", "parameter_id": 2462, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cyffAd": { + { + "code": "cyffAd", "parameter_id": 5562, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cyftn": { + { + "code": "cyftn", "parameter_id": 2232, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "cypcnzl": { + { + "code": "cypcnzl", "parameter_id": 3537, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cypdnl": { + { + "code": "cypdnl", "parameter_id": 3538, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "cyrmzne": { + { + "code": "cyrmzne", "parameter_id": 3542, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "damnzde": { + { + "code": "damnzde", "parameter_id": 3551, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DEET": { + { + "code": "DEET", "parameter_id": 2308, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "dmtn": { + { + "code": "dmtn", "parameter_id": 563, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "desC2yatzne": { + { + "code": "desC2yatzne", "parameter_id": 571, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "desC2ytC4yaz": { + { + "code": "desC2ytC4yaz", "parameter_id": 3614, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "desiC3yatzne": { + { + "code": "desiC3yatzne", "parameter_id": 573, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "desmdfm": { + { + "code": "desmdfm", "parameter_id": 3616, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "desFyClidzn": { + { + "code": "desFyClidzn", "parameter_id": 3615, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "valum": { + { + "code": "valum", "parameter_id": 4913, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Daznn": { + { + "code": "Daznn", "parameter_id": 592, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dcba": { + { + "code": "Dcba", "parameter_id": 602, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dcbnl": { + { + "code": "Dcbnl", "parameter_id": 603, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "24DP": { + { + "code": "24DP", "parameter_id": 140, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DClvs": { + { + "code": "DClvs", "parameter_id": 624, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dclofnc": { + { + "code": "Dclofnc", "parameter_id": 2466, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "dieldn": { + { + "code": "dieldn", "parameter_id": 633, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Dfncnzl": { + { + "code": "Dfncnzl", "parameter_id": 3627, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dfbzrn": { + { + "code": "Dfbzrn", "parameter_id": 648, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dffncn": { + { + "code": "Dffncn", "parameter_id": 3624, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dikglc": { + { + "code": "Dikglc", "parameter_id": 3644, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "dikglNa": { + { + "code": "dikglNa", "parameter_id": 5563, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DmtCl": { + { + "code": "DmtCl", "parameter_id": 3652, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DmtAd": { + { + "code": "DmtAd", "parameter_id": 3651, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dmtat": { + { + "code": "Dmtat", "parameter_id": 653, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dmtmf": { + { + "code": "Dmtmf", "parameter_id": 3654, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DMST": { + { + "code": "DMST", "parameter_id": 3650, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dnsb": { + { + "code": "Dnsb", "parameter_id": 671, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Dntb": { + { + "code": "Dntb", "parameter_id": 672, "unit": "ug/l", "validationMethod": "D38407-35.10", "analyticalTechnique": "LC-MS-MS", }, - "Dtann": { + { + "code": "Dtann", "parameter_id": 3692, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "sDtocbmt": { + { + "code": "sDtocbmt", "parameter_id": 681, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "GC-MS", }, - "Durn": { + { + "code": "Durn", "parameter_id": 683, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Corg": { + { + "code": "Corg", "parameter_id": 1318, "unit": "mg/l", "validationMethod": "N1484.97", "analyticalTechnique": "IR", }, - "dodmf": { + { + "code": "dodmf", "parameter_id": 3668, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "dodne": { + { + "code": "dodne", "parameter_id": 2455, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "doxccne": { + { + "code": "doxccne", "parameter_id": 3672, "unit": "ng/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "emmtn": { + { + "code": "emmtn", "parameter_id": 3705, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "enrfxcne": { + { + "code": "enrfxcne", "parameter_id": 3713, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "epxcnzl": { + { + "code": "epxcnzl", "parameter_id": 3718, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "ertmcne": { + { + "code": "ertmcne", "parameter_id": 2480, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "esfvlrt": { + { + "code": "esfvlrt", "parameter_id": 3723, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "esTol": { + { + "code": "esTol", "parameter_id": 2295, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "oestn": { + { + "code": "oestn", "parameter_id": 2313, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "etfmst": { + { + "code": "etfmst", "parameter_id": 721, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "etpfs": { + { + "code": "etpfs", "parameter_id": 722, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C2oxsfrn": { + { + "code": "C2oxsfrn", "parameter_id": 3346, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "etxzl": { + { + "code": "etxzl", "parameter_id": 3739, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "eTDazl": { + { + "code": "eTDazl", "parameter_id": 740, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "fenmdn": { + { + "code": "fenmdn", "parameter_id": 2362, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fenamfs": { + { + "code": "fenamfs", "parameter_id": 745, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fenzn": { + { + "code": "fenzn", "parameter_id": 2481, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "fenhxAd": { + { + "code": "fenhxAd", "parameter_id": 3764, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "feNO2ton": { + { + "code": "feNO2ton", "parameter_id": 750, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "fenfbt": { + { + "code": "fenfbt", "parameter_id": 2482, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "fenOxcb": { + { + "code": "fenOxcb", "parameter_id": 3768, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fenppmf": { + { + "code": "fenppmf", "parameter_id": 3771, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "fenprAe": { + { + "code": "fenprAe", "parameter_id": 5564, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fenrn": { + { + "code": "fenrn", "parameter_id": 2451, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fipnl": { + { + "code": "fipnl", "parameter_id": 3786, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "floncmd": { + { + "code": "floncmd", "parameter_id": 3793, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "flurslm": { + { + "code": "flurslm", "parameter_id": 3829, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fluaznm": { + { + "code": "fluaznm", "parameter_id": 773, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "flubDad": { + { + "code": "flubDad", "parameter_id": 5174, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fludoxnl": { + { + "code": "fludoxnl", "parameter_id": 3808, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "flumoxzn": { + { + "code": "flumoxzn", "parameter_id": 3812, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fluopclde": { + { + "code": "fluopclde", "parameter_id": 3816, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fluoprm": { + { + "code": "fluoprm", "parameter_id": 5245, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "F": { + { + "code": "F", "parameter_id": 1853, "unit": "mg/l", "validationMethod": "N6578.85", "analyticalTechnique": "POTM", }, - "fluoxsbn": { + { + "code": "fluoxsbn", "parameter_id": 3818, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "flurOxpr": { + { + "code": "flurOxpr", "parameter_id": 2363, "unit": "ug/l", "validationMethod": "D38407-35.10", "analyticalTechnique": "LC-MS-MS", }, - "flutlnl": { + { + "code": "flutlnl", "parameter_id": 2305, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fluxprxd": { + { + "code": "fluxprxd", "parameter_id": 5078, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "forasfrn": { + { + "code": "forasfrn", "parameter_id": 3836, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fostazt": { + { + "code": "fostazt", "parameter_id": 3845, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "fursmde": { + { + "code": "fursmde", "parameter_id": 3854, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "gabptne": { + { + "code": "gabptne", "parameter_id": 3857, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "cHCH": { + { + "code": "cHCH", "parameter_id": 816, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "GELDHD": { + { + "code": "GELDHD", "parameter_id": 3548, "unit": "mS/m", "validationMethod": "I7888.94", "analyticalTechnique": "COND", }, - "gemfbzl": { + { + "code": "gemfbzl", "parameter_id": 2469, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "glufsnt": { + { + "code": "glufsnt", "parameter_id": 3877, "unit": "ug/l", "validationMethod": "I16308.17", "analyticalTechnique": "LC-MS-MS", }, - "glyfst": { + { + "code": "glyfst", "parameter_id": 840, "unit": "ug/l", "validationMethod": "I16308.17", "analyticalTechnique": "LC-MS-MS", }, - "Hxznn": { + { + "code": "Hxznn", "parameter_id": 871, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "hextazx": { + { + "code": "hextazx", "parameter_id": 3935, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "HCltazde": { + { + "code": "HCltazde", "parameter_id": 3928, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "ibpfn": { + { + "code": "ibpfn", "parameter_id": 2470, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "iffAd": { + { + "code": "iffAd", "parameter_id": 3984, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Fe": { + { + "code": "Fe", "parameter_id": 879, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "imzll": { + { + "code": "imzll", "parameter_id": 2300, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "imzmx": { + { + "code": "imzmx", "parameter_id": 5047, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "imdcpd": { + { + "code": "imdcpd", "parameter_id": 2306, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "indmtcne": { + { + "code": "indmtcne", "parameter_id": 2471, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "IsfrnC1y": { + { + "code": "IsfrnC1y", "parameter_id": 4027, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "johxl": { + { + "code": "johxl", "parameter_id": 4040, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "jopmdl": { + { + "code": "jopmdl", "parameter_id": 4043, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "jopmde": { + { + "code": "jopmde", "parameter_id": 4042, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "jotlmnzr": { + { + "code": "jotlmnzr", "parameter_id": 4045, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "joxgnzr": { + { + "code": "joxgnzr", "parameter_id": 4046, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "ipDon": { + { + "code": "ipDon", "parameter_id": 890, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "iClidzn": { + { + "code": "iClidzn", "parameter_id": 3981, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "iptrn": { + { + "code": "iptrn", "parameter_id": 913, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "isprzm": { + { + "code": "isprzm", "parameter_id": 5079, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "isxbn": { + { + "code": "isxbn", "parameter_id": 4030, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "iOaftl": { + { + "code": "iOaftl", "parameter_id": 4010, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "jompl": { + { + "code": "jompl", "parameter_id": 4041, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "K": { + { + "code": "K", "parameter_id": 920, "unit": "mg/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "ketpfn": { + { + "code": "ketpfn", "parameter_id": 2472, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Co": { + { + "code": "Co", "parameter_id": 527, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Cu": { + { + "code": "Cu", "parameter_id": 971, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "kresOxmC1y": { + { + "code": "kresOxmC1y", "parameter_id": 4063, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Hg": { + { + "code": "Hg", "parameter_id": 1097, "unit": "ug/l", "validationMethod": "I12846.12", "analyticalTechnique": "AAS-KD", }, - "lcyhltn": { + { + "code": "lcyhltn", "parameter_id": 2233, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "lencl": { + { + "code": "lencl", "parameter_id": 1898, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "lidcine": { + { + "code": "lidcine", "parameter_id": 4093, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "linrn": { + { + "code": "linrn", "parameter_id": 1114, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Li": { + { + "code": "Li", "parameter_id": 1115, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Pb": { + { + "code": "Pb", "parameter_id": 1116, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "lufnrn": { + { + "code": "lufnrn", "parameter_id": 4104, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Mg": { + { + "code": "Mg", "parameter_id": 1125, "unit": "mg/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "malton": { + { + "code": "malton", "parameter_id": 1127, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mandppAd": { + { + "code": "mandppAd", "parameter_id": 4117, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Mn": { + { + "code": "Mn", "parameter_id": 1128, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "MCPA": { + { + "code": "MCPA", "parameter_id": 225, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "MCPP": { + { + "code": "MCPP", "parameter_id": 73, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mefpDC2y": { + { + "code": "mefpDC2y", "parameter_id": 4143, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "mepqCl": { + { + "code": "mepqCl", "parameter_id": 5248, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metocb": { + { + "code": "metocb", "parameter_id": 2307, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "messfrnC1y": { + { + "code": "messfrnC1y", "parameter_id": 4152, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "meston": { + { + "code": "meston", "parameter_id": 4153, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mesnl": { + { + "code": "mesnl", "parameter_id": 4151, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "mfmzn": { + { + "code": "mfmzn", "parameter_id": 5254, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mlxl": { + { + "code": "mlxl", "parameter_id": 1149, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mAh": { + { + "code": "mAh", "parameter_id": 4110, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mmtn": { + { + "code": "mmtn", "parameter_id": 1151, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mzCl": { + { + "code": "mzCl", "parameter_id": 1152, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metcnzl": { + { + "code": "metcnzl", "parameter_id": 4156, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metfmne": { + { + "code": "metfmne", "parameter_id": 4157, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "metbtazrn": { + { + "code": "metbtazrn", "parameter_id": 1148, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metdton": { + { + "code": "metdton", "parameter_id": 1155, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C1oxfnzde": { + { + "code": "C1oxfnzde", "parameter_id": 3318, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C1ydesFyClid": { + { + "code": "C1ydesFyClid", "parameter_id": 3331, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metbmrn": { + { + "code": "metbmrn", "parameter_id": 1184, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metlCl": { + { + "code": "metlCl", "parameter_id": 1185, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metpll": { + { + "code": "metpll", "parameter_id": 2485, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "metxrn": { + { + "code": "metxrn", "parameter_id": 1187, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metfnn": { + { + "code": "metfnn", "parameter_id": 4158, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "metbzn": { + { + "code": "metbzn", "parameter_id": 1188, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C1ymsfrn": { + { + "code": "C1ymsfrn", "parameter_id": 2453, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "mevfs": { + { + "code": "mevfs", "parameter_id": 1191, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Mo": { + { + "code": "Mo", "parameter_id": 1243, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Mlnrn": { + { + "code": "Mlnrn", "parameter_id": 1253, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "monrn": { + { + "code": "monrn", "parameter_id": 1254, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "DC1ysAd": { + { + "code": "DC1ysAd", "parameter_id": 5565, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "nappAd": { + { + "code": "nappAd", "parameter_id": 4230, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "napxn": { + { + "code": "napxn", "parameter_id": 2474, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Na": { + { + "code": "Na", "parameter_id": 1262, "unit": "mg/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "EtFOSAA": { + { + "code": "EtFOSAA", "parameter_id": 5744, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "nicsfrn": { + { + "code": "nicsfrn", "parameter_id": 4254, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Ni": { + { + "code": "Ni", "parameter_id": 1267, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "NO3": { + { + "code": "NO3", "parameter_id": 1270, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "NO2": { + { + "code": "NO2", "parameter_id": 1273, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "MeFOSA": { + { + "code": "MeFOSA", "parameter_id": 6001, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "N-MeFOSAA": { + { + "code": "N-MeFOSAA", "parameter_id": 5937, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "norfxcne": { + { + "code": "norfxcne", "parameter_id": 4278, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "ofxcne": { + { + "code": "ofxcne", "parameter_id": 4310, "unit": "ng/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "PO4": { + { + "code": "PO4", "parameter_id": 1334, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "Oaml": { + { + "code": "Oaml", "parameter_id": 1343, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Oasfrn": { + { + "code": "Oasfrn", "parameter_id": 4301, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "oxzpm": { + { + "code": "oxzpm", "parameter_id": 4337, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "C1yOxdmtn": { + { + "code": "C1yOxdmtn", "parameter_id": 568, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "OxT4ccnHCl": { + { + "code": "OxT4ccnHCl", "parameter_id": 5124, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "pacbtzl": { + { + "code": "pacbtzl", "parameter_id": 4344, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "parctml": { + { + "code": "parctml", "parameter_id": 4347, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "paroonC1y": { + { + "code": "paroonC1y", "parameter_id": 4350, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pencnzl": { + { + "code": "pencnzl", "parameter_id": 2302, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "penccrn": { + { + "code": "penccrn", "parameter_id": 2301, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pendmtln": { + { + "code": "pendmtln", "parameter_id": 1971, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "PeClFol": { + { + "code": "PeClFol", "parameter_id": 1390, "unit": "ug/l", "validationMethod": "N12673.99", "analyticalTechnique": "GC-MS", }, - "L_PFBS": { + { + "code": "L_PFBS", "parameter_id": 3895, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFBA": { + { + "code": "PFBA", "parameter_id": 4437, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "L_PFDS": { + { + "code": "L_PFDS", "parameter_id": 3898, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFDA": { + { + "code": "PFDA", "parameter_id": 4438, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFDPA": { + { + "code": "PFDPA", "parameter_id": 6455, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFDoA": { + { + "code": "PFDoA", "parameter_id": 4439, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "42FTOH": { + { + "code": "42FTOH", "parameter_id": 6452, "unit": "ug/l", "validationMethod": "Inhouse", "analyticalTechnique": "GC-MS", }, - "L_PFHpS": { + { + "code": "L_PFHpS", "parameter_id": 3931, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFHpA": { + { + "code": "PFHpA", "parameter_id": 4440, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "L_PFHxS": { + { + "code": "L_PFHxS", "parameter_id": 3932, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFHxA": { + { + "code": "PFHxA", "parameter_id": 4441, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFC16azr": { + { + "code": "PFC16azr", "parameter_id": 5735, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFNA": { + { + "code": "PFNA", "parameter_id": 4442, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFOSA": { + { + "code": "PFOSA", "parameter_id": 4446, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFOS": { + { + "code": "PFOS", "parameter_id": 4445, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFOA": { + { + "code": "PFOA", "parameter_id": 4443, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFOA_NH4": { + { + "code": "PFOA_NH4", "parameter_id": 5970, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFC18azr": { + { + "code": "PFC18azr", "parameter_id": 5736, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFOPA": { + { + "code": "PFOPA", "parameter_id": 6454, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFC5asfzr": { + { + "code": "PFC5asfzr", "parameter_id": 5935, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFPA": { + { + "code": "PFPA", "parameter_id": 4448, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFPeDA": { + { + "code": "PFPeDA", "parameter_id": 6456, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFTeDA": { + { + "code": "PFTeDA", "parameter_id": 4450, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFTDA": { + { + "code": "PFTDA", "parameter_id": 4449, "unit": "ug/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "PFUdA": { + { + "code": "PFUdA", "parameter_id": 4451, "unit": "ng/l", "validationMethod": "I21675.19", "analyticalTechnique": "LC-MS-MS", }, - "permtn": { + { + "code": "permtn", "parameter_id": 1394, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "pH": { + { + "code": "pH", "parameter_id": 1398, "unit": "1", "validationMethod": "I10523.12", "analyticalTechnique": "POTM", }, - "fenmdfm": { + { + "code": "fenmdfm", "parameter_id": 3765, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pinadn": { + { + "code": "pinadn", "parameter_id": 4459, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "piprnbO": { + { + "code": "piprnbO", "parameter_id": 1405, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pirmcb": { + { + "code": "pirmcb", "parameter_id": 1406, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "C1yprmfs": { + { + "code": "C1yprmfs", "parameter_id": 1408, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "primdn": { + { + "code": "primdn", "parameter_id": 2489, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "proClaz": { + { + "code": "proClaz", "parameter_id": 1416, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "procmdn": { + { + "code": "procmdn", "parameter_id": 1417, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "propmcb": { + { + "code": "propmcb", "parameter_id": 1424, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "propcnzl": { + { + "code": "propcnzl", "parameter_id": 1429, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "propxr": { + { + "code": "propxr", "parameter_id": 1432, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "propnll": { + { + "code": "propnll", "parameter_id": 2491, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "propfnzn": { + { + "code": "propfnzn", "parameter_id": 4494, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "propAd": { + { + "code": "propAd", "parameter_id": 1438, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "prosfcb": { + { + "code": "prosfcb", "parameter_id": 2361, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "prosfrn": { + { + "code": "prosfrn", "parameter_id": 4507, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "protocnzl": { + { + "code": "protocnzl", "parameter_id": 4510, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "protocnzdto": { + { + "code": "protocnzdto", "parameter_id": 5252, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pymtzne": { + { + "code": "pymtzne", "parameter_id": 4521, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrcsbn": { + { + "code": "pyrcsbn", "parameter_id": 4524, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrffnC2y": { + { + "code": "pyrffnC2y", "parameter_id": 4529, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrdt": { + { + "code": "pyrdt", "parameter_id": 4527, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrdll": { + { + "code": "pyrdll", "parameter_id": 5249, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrmtnl": { + { + "code": "pyrmtnl", "parameter_id": 2303, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrpxfn": { + { + "code": "pyrpxfn", "parameter_id": 4533, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "pyrslm": { + { + "code": "pyrslm", "parameter_id": 4535, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "quinmrc": { + { + "code": "quinmrc", "parameter_id": 4543, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "quincmn": { + { + "code": "quincmn", "parameter_id": 4540, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "quizlfC2y": { + { + "code": "quizlfC2y", "parameter_id": 4546, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "rimsfrn": { + { + "code": "rimsfrn", "parameter_id": 4570, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "roxtmcne": { + { + "code": "roxtmcne", "parameter_id": 2492, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Se": { + { + "code": "Se", "parameter_id": 1476, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "siltofm": { + { + "code": "siltofm", "parameter_id": 4654, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "simzne": { + { + "code": "simzne", "parameter_id": 1480, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "endsfn": { + { + "code": "endsfn", "parameter_id": 2161, "unit": "ug/l", "validationMethod": "I6468.97", "analyticalTechnique": "AUTOMATISCH", }, - "sotll": { + { + "code": "sotll", "parameter_id": 4675, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "spirttmt": { + { + "code": "spirttmt", "parameter_id": 5056, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Sr": { + { + "code": "Sr", "parameter_id": 1501, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "sulcton": { + { + "code": "sulcton", "parameter_id": 4735, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "SO4": { + { + "code": "SO4", "parameter_id": 1508, "unit": "mg/l", "validationMethod": "I15923-1.13", "analyticalTechnique": "DA-S", }, - "sulfdazne": { + { + "code": "sulfdazne", "parameter_id": 4737, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "sulfdmdne": { + { + "code": "sulfdmdne", "parameter_id": 2494, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "sulfmtoazl": { + { + "code": "sulfmtoazl", "parameter_id": 2475, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "sulfprdne": { + { + "code": "sulfprdne", "parameter_id": 4742, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "sulfsfrn": { + { + "code": "sulfsfrn", "parameter_id": 5058, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "tebcnzl": { + { + "code": "tebcnzl", "parameter_id": 2298, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "tefbzrn": { + { + "code": "tefbzrn", "parameter_id": 2454, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "teftn": { + { + "code": "teftn", "parameter_id": 4825, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Te": { + { + "code": "Te", "parameter_id": 1520, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "tembtone": { + { + "code": "tembtone", "parameter_id": 5112, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "T": { + { + "code": "T", "parameter_id": 1522, "unit": "Cel", "validationMethod": "I10523.12", "analyticalTechnique": "POTM", }, - "terbtn": { + { + "code": "terbtn", "parameter_id": 1524, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "terC4yazne": { + { + "code": "terC4yazne", "parameter_id": 1525, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "T4mtn": { + { + "code": "T4mtn", "parameter_id": 1548, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Tl": { + { + "code": "Tl", "parameter_id": 1553, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "tabdzl": { + { + "code": "tabdzl", "parameter_id": 4770, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "thiacpd": { + { + "code": "thiacpd", "parameter_id": 4863, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "thiamtxm": { + { + "code": "thiamtxm", "parameter_id": 4864, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "thifsfrnC1y": { + { + "code": "thifsfrnC1y", "parameter_id": 4867, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "toDcb": { + { + "code": "toDcb", "parameter_id": 2447, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "tofnC1y": { + { + "code": "tofnC1y", "parameter_id": 4882, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Sn": { + { + "code": "Sn", "parameter_id": 1564, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Ti": { + { + "code": "Ti", "parameter_id": 1565, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "tolcfsC1y": { + { + "code": "tolcfsC1y", "parameter_id": 1567, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "tolfande": { + { + "code": "tolfande", "parameter_id": 2235, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "topmzn": { + { + "code": "topmzn", "parameter_id": 4884, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Ptot": { + { + "code": "Ptot", "parameter_id": 4188, "unit": "mg/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "tramdl": { + { + "code": "tramdl", "parameter_id": 4891, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "Tadmfn": { + { + "code": "Tadmfn", "parameter_id": 1586, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Tadmnl": { + { + "code": "Tadmnl", "parameter_id": 2297, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Talt": { + { + "code": "Talt", "parameter_id": 4773, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "TbnrC1y": { + { + "code": "TbnrC1y", "parameter_id": 4785, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "TC4yPO4": { + { + "code": "TC4yPO4", "parameter_id": 1591, "unit": "ug/l", "validationMethod": "D38407-37.13", "analyticalTechnique": "GC-MS-MS", }, - "Tcpr": { + { + "code": "Tcpr", "parameter_id": 2450, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Tfxsbn": { + { + "code": "Tfxsbn", "parameter_id": 4846, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "TfsfrnC1y": { + { + "code": "TfsfrnC1y", "parameter_id": 4844, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "Tmtpm": { + { + "code": "Tmtpm", "parameter_id": 2496, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "TnxpcC2y": { + { + "code": "TnxpcC2y", "parameter_id": 4878, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "tritsfrn": { + { + "code": "tritsfrn", "parameter_id": 4897, "unit": "ug/l", "validationMethod": "D38407-36.14", "analyticalTechnique": "LC-MS-MS", }, - "tylsne": { + { + "code": "tylsne", "parameter_id": 4902, "unit": "ug/l", "validationMethod": "I21676.22", "analyticalTechnique": "LC-MS-MS", }, - "U": { + { + "code": "U", "parameter_id": 1637, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "V": { + { + "code": "V", "parameter_id": 1642, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "HCO3": { + { + "code": "HCO3", "parameter_id": 374, "unit": "mg/l", "validationMethod": "I9963-1.96", "analyticalTechnique": "POTM_TITM", }, - "W": { + { + "code": "W", "parameter_id": 1685, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Ag": { + { + "code": "Ag", "parameter_id": 1692, "unit": "ug/l", "validationMethod": "I17294-2.04", "analyticalTechnique": "ICP-MS", }, - "Zn": { - "parameter_id": 1693, - "unit": "ug/l", - "validationMethod": "I17294-2.04", - "analyticalTechnique": "ICP-MS", - }, - "Zr": { - "parameter_id": 1695, - "unit": "ug/l", - "validationMethod": "I17294-2.04", - "analyticalTechnique": "ICP-MS", - }, -} +] + # This parameter is also derived from https://docs.geostandaarden.nl/bro/def-im-gar-20230607/#detail_class_Model_Parameterlijst # It serves as translation between cas-number and parameter ID diff --git a/api/bro_upload/gar_bulk_upload.py b/api/bro_upload/gar_bulk_upload.py index d718fba..b18442b 100644 --- a/api/bro_upload/gar_bulk_upload.py +++ b/api/bro_upload/gar_bulk_upload.py @@ -1,6 +1,7 @@ import logging import os import re +from collections import defaultdict from typing import TypeVar import pandas as pd @@ -23,6 +24,120 @@ T = TypeVar("T", bound="api_models.UploadFile") +_REPORTING_LIMIT_COLUMN_RE = re.compile(r"^\s*Rapportagegrens\s+(.+?)\s+\(.*\)\s*$") +_ANALYSIS_DATE_COLUMN_RE = re.compile(r"^\s*Analysedatum\s+(.+?)\s+\(.*\)\s*$") +_MEASUREMENT_VALUE_COLUMN_RE = re.compile(r"^\s*(.+?)\s+\(.*\)\s*$") +_OLD_FORMAT_TRIGGER_HEADERS = {"Zuurstof (mg/l)"} + + +def _normalize_header(value: str) -> str: + return re.sub(r"\s+", "", str(value)).lower() + + +def _is_missing_measurement_value(value: object) -> bool: + if pd.isna(value): + return True + if isinstance(value, str) and value.strip().lower() in {"", "niet bepaald"}: + return True + return False + + +def _resolve_field_measurement_columns( + row: pd.Series, +) -> tuple[dict[str, str], dict[str, str]]: + """Map parameter names to source columns, preferring *_field over unsuffixed columns. + + *_lab columns are ignored for field measurements. + Suffixes are stripped before regex matching to correctly identify parameters. + """ + columns_by_base_name: dict[str, str] = {} + normalized_columns: dict[str, str] = {} + + for column in row.index.tolist(): + column_str = str(column) + if column_str.endswith("_lab"): + continue + + base_name = column_str[:-6] if column_str.endswith("_field") else column_str + is_field_column = column_str.endswith("_field") + + if base_name not in columns_by_base_name or is_field_column: + columns_by_base_name[base_name] = column_str + + normalized_base_name = _normalize_header(base_name) + if normalized_base_name not in normalized_columns or is_field_column: + normalized_columns[normalized_base_name] = column_str + + return columns_by_base_name, normalized_columns + + +def _resolve_lab_analysis_columns( + row: pd.Series, +) -> tuple[dict[str, str], dict[str, str], dict[str, str]]: + """Map lab parameter names to source columns, preferring *_lab over unsuffixed. + + *_field columns are ignored for lab analyses to avoid field/lab interference. + Suffixes are stripped before regex matching to correctly identify parameters. + """ + value_columns_by_parameter: dict[str, str] = {} + reporting_limit_columns_by_parameter: dict[str, str] = {} + analysis_date_columns_by_parameter: dict[str, str] = {} + + for column in row.index: + column_str = str(column) + is_lab_column = column_str.endswith("_lab") + + if column_str.endswith("_field"): + continue + + # Strip suffix before applying regex to correctly identify parameter names + base_column = column_str[:-4] if is_lab_column else column_str + + reporting_limit_match = _REPORTING_LIMIT_COLUMN_RE.match(base_column) + if reporting_limit_match: + parameter = reporting_limit_match.group(1) + if parameter not in reporting_limit_columns_by_parameter or is_lab_column: + reporting_limit_columns_by_parameter[parameter] = column_str + continue + + analysis_date_match = _ANALYSIS_DATE_COLUMN_RE.match(base_column) + if analysis_date_match: + parameter = analysis_date_match.group(1) + if parameter not in analysis_date_columns_by_parameter or is_lab_column: + analysis_date_columns_by_parameter[parameter] = column_str + continue + + measurement_match = _MEASUREMENT_VALUE_COLUMN_RE.match(base_column) + if measurement_match: + parameter = measurement_match.group(1) + if parameter not in value_columns_by_parameter or is_lab_column: + value_columns_by_parameter[parameter] = column_str + + return ( + value_columns_by_parameter, + reporting_limit_columns_by_parameter, + analysis_date_columns_by_parameter, + ) + + +def _build_lab_parameters_by_process() -> dict[ + tuple[str, str], list[tuple[str, int, str]] +]: + grouped_parameters: dict[tuple[str, str], list[tuple[str, int, str]]] = defaultdict( + list + ) + for parameter in config.LAB_PARAMETER_OPTIONS: + process_key = (parameter["analyticalTechnique"], parameter["validationMethod"]) + grouped_parameters[process_key].append( + (parameter["code"], parameter["parameter_id"], parameter["unit"]) + ) + + return dict(grouped_parameters) + + +LAB_PARAMETERS_BY_PROCESS = _build_lab_parameters_by_process() + + class GARBulkUploader: """Handles the upload process for bulk GAR data. @@ -83,18 +198,21 @@ def _process_fieldwork_and_lab_dfs( ) -> pd.DataFrame: try: # Rename headers - required_fields = ["GMW BRO ID", "Datum bemonsterd", "Filternummer"] + required_fields_field = ["GMW BRO ID", "Datum bemonsterd", "Filternummer"] + required_fields_lab = ["GMW BRO ID", "Datum bemonsterd", "Filternummer"] has_lab = True - if all(field in fieldwork_df.columns for field in required_fields) and all( - field in lab_df.columns for field in required_fields - ): + if all( + field in fieldwork_df.columns for field in required_fields_field + ) and all(field in lab_df.columns for field in required_fields_lab): merged_df = merge_fieldwork_and_lab_dfs(fieldwork_df, lab_df) - elif all(field in fieldwork_df.columns for field in required_fields): + elif all(field in fieldwork_df.columns for field in required_fields_field): merged_df = fieldwork_df has_lab = False else: merged_df = lab_df + logger.info(f"has lab: {has_lab}") + fieldwork_df_rename_dict = { "GMW BRO ID": "bro_id", "Datum bemonsterd": "date", @@ -111,7 +229,9 @@ def _process_fieldwork_and_lab_dfs( "Monsternummer lab", ] trimmed_df = remove_df_columns(merged_df, field_columns_exclude) - + logger.info( + f"Trimmed the dataframe to the following columns: {trimmed_df.columns.tolist()}" + ) # Pandas DF: create new column meetronde, which should only have the year of datum bemonsterd, as a string trimmed_df["Meetronde"] = trimmed_df["date"].dt.year.astype(str) @@ -139,6 +259,8 @@ def process(self) -> None: logger.info("Initialized the dataframes.") # Step 2: transform the pandas files to a useable format + logger.info(fieldwork_df.columns.tolist()) + logger.info(lab_df.columns.tolist()) trimmed_df, has_lab = self._process_fieldwork_and_lab_dfs(fieldwork_df, lab_df) if trimmed_df is None: return @@ -224,10 +346,12 @@ def merge_fieldwork_and_lab_dfs( This filters out the location/date combinations that are only present in 1 file.""" return pd.merge( - fieldwork_df, - lab_df, - on=["GMW BRO ID", "Datum bemonsterd", "Filternummer"], + left=fieldwork_df, + right=lab_df, + left_on=["GMW BRO ID", "Datum bemonsterd", "Filternummer"], + right_on=["GMW BRO ID", "Datum bemonsterd", "Filternummer"], how="inner", + suffixes=("_field", "_lab"), ) @@ -374,41 +498,53 @@ def create_gar_field_measurements(row: pd.Series) -> list[FieldMeasurement]: curdir = os.path.dirname(os.path.abspath(__file__)) logger.info(f"column names: {row.index.tolist()}") - if "Zuurstof (mg/L)" in row: - logger.info("Using old GAR parameter format.") - # Old format from Provincie Noord-Brabant - for parameter, details in config.FIELD_PARAMETER_OPTIONS.items(): - logger.info(f"Checking column: {parameter}") - if parameter in row and pd.notna(row[parameter]): - parameter_dict = { - "parameter": details["parameter_id"], - "unit": details["unit"], - "fieldMeasurementValue": row[parameter], - "qualityControlStatus": "onbeslist", - } - field_measurement = FieldMeasurement(**parameter_dict) - field_measurement_list.append(field_measurement) + columns_by_base_name, _ = _resolve_field_measurement_columns(row) + logger.info(f"columns by base name: {columns_by_base_name}") + has_old_format_trigger = any( + trigger in columns_by_base_name for trigger in _OLD_FORMAT_TRIGGER_HEADERS + ) + if has_old_format_trigger: + logger.info("Using old format parameter config from hardcoded dict.") + # Old format from Provincie Noord-Brabant + df = pl.DataFrame(config.FIELD_PARAMETER_OPTIONS) else: - logger.info("Using new GAR parameter format.") + logger.info("Using new format parameter config from CSV file.") df = pl.read_csv(os.path.join(curdir, "20260107_GARVarList.csv"), separator=";") - for column in row.index.tolist(): - logger.info(f"Checking column: {column}") - if column in df["aquocode"].to_list() and row[column] != "niet bepaald": - parameter_details = df.filter(pl.col("aquocode") == column).to_dicts()[ - 0 - ] - parameter = column - parameter_dict = { - "parameter": parameter_details["ID"], - "unit": parameter_details["eenheid"], - "fieldMeasurementValue": row[parameter], - "qualityControlStatus": "onbeslist", - } + df = df.rename({"aquocode": "code", "ID": "parameter_id", "eenheid": "unit"}) - field_measurement = FieldMeasurement(**parameter_dict) - field_measurement_list.append(field_measurement) + for param in columns_by_base_name.keys(): + logger.info(f"Checking parameter '{param}' against config options.") + + parameter = df.filter(pl.col("code") == param).select("parameter_id", "unit") + if parameter.height == 0: + logger.warning( + f"Parameter '{param}' not found in config, skipping this parameter." + ) + continue + + parameter = parameter.row(0, named=True) + source_column = columns_by_base_name.get(param) + value = row[source_column] + if _is_missing_measurement_value(value): + logger.warning( + f"Skipping missing value for parameter '{parameter['parameter_id']}' in column '{source_column}'" + ) + continue + + parameter_dict = { + "parameter": parameter["parameter_id"], + "unit": parameter["unit"], + "fieldMeasurementValue": value, + "qualityControlStatus": "onbeslist", + } + logger.info( + f"Created field measurement dict for parameter '{parameter['parameter_id']}': {parameter_dict}" + ) + + field_measurement = FieldMeasurement(**parameter_dict) + field_measurement_list.append(field_measurement) return field_measurement_list @@ -425,94 +561,68 @@ def create_gar_lab_analysis( return [LaboratoryAnalysis(**lab_analysis)] -def create_analysis_process(row: pd.Series) -> list[AnalysisProcess]: +def create_analysis_process(row: pd.Series) -> list[AnalysisProcess]: # noqa C901 analysis_processes = [] - for parameter, details in config.LAB_PARAMETER_OPTIONS.items(): - value_column_pattern = rf"^\s*{parameter}\s+\(.*\)\s*$" - value_column = next( - (col for col in row.index if re.search(value_column_pattern, col)), None - ) - - value = row[value_column] if value_column else None - - if value_column and isinstance(value, int | float) and pd.notna(value): - reporting_limit_column_pattern = ( - rf"^\s*Rapportagegrens\s+{parameter}\s+\(.*\)\s*$" - ) - reporting_limit_column = next( - ( - col - for col in row.index - if re.search(reporting_limit_column_pattern, col) - ), - None, - ) - - date_column_pattern = rf"^\s*Analysedatum\s+{parameter}\s+\(.*\)\s*$" - date_column = next( - (col for col in row.index if re.search(date_column_pattern, col)), None + ( + value_columns_by_parameter, + reporting_limit_columns_by_parameter, + analysis_date_columns_by_parameter, + ) = _resolve_lab_analysis_columns(row) + + for (technique, method), parameters in LAB_PARAMETERS_BY_PROCESS.items(): + analyses = [] + process_date = None + + for parameter, parameter_id, unit in parameters: + value_column = value_columns_by_parameter.get(parameter) + if not value_column: + continue + + date_column = analysis_date_columns_by_parameter.get(parameter) + if not date_column: + continue + + date_value = row.get(date_column) + if pd.isna(date_value) or date_value == "": + continue + + value = row[value_column] + reporting_limit_column = reporting_limit_columns_by_parameter.get(parameter) + reporting_limit = ( + row[reporting_limit_column] if reporting_limit_column else None ) - analysis_dict = { - "parameter": details["parameter_id"], - "unit": details["unit"], - "analysisMeasurementValue": value, - "reportingLimit": row[reporting_limit_column] - if reporting_limit_column - else None, - "qualityControlStatus": "onbeslist", - } - analysis = Analysis(**analysis_dict) - - analysis_process_dict = { - "date": row[date_column] if date_column else None, - "analyticalTechnique": details["analyticalTechnique"], - "valuationMethod": details["validationMethod"], - "analyses": [analysis], - } - - analysis_process = AnalysisProcess(**analysis_process_dict) - analysis_processes.append(analysis_process) - - elif value in ["<", "GT"]: - value = "LT" if value == "<" else value - reporting_limit_column_pattern = ( - rf"^\s*Rapportagegrens\s+{parameter}\s+\(.*\)\s*$" - ) - reporting_limit_column = next( - ( - col - for col in row.index - if re.search(reporting_limit_column_pattern, col) - ), - None, - ) - - date_column_pattern = rf"^\s*Analysedatum\s+{parameter}\s+\(.*\)\s*$" - date_column = next( - (col for col in row.index if re.search(date_column_pattern, col)), None - ) - - analysis_dict = { - "parameter": details["parameter_id"], - "unit": details["unit"], - "reportingLimit": row[reporting_limit_column] - if reporting_limit_column - else None, - "limitSymbol": value, - "qualityControlStatus": "onbeslist", - } - analysis = Analysis(**analysis_dict) + if pd.api.types.is_number(value) and pd.notna(value): + analysis_dict = { + "parameter": parameter_id, + "unit": unit, + "analysisMeasurementValue": value, + "reportingLimit": reporting_limit, + "qualityControlStatus": "onbeslist", + } + analyses.append(Analysis(**analysis_dict)) + if process_date is None: + process_date = date_value + elif value in ["<", "GT"]: + analysis_dict = { + "parameter": parameter_id, + "unit": unit, + "reportingLimit": reporting_limit, + "limitSymbol": "LT" if value == "<" else value, + "qualityControlStatus": "onbeslist", + } + analyses.append(Analysis(**analysis_dict)) + if process_date is None: + process_date = date_value + if analyses and process_date is not None: analysis_process_dict = { - "date": row[date_column] if date_column else None, - "analyticalTechnique": details["analyticalTechnique"], - "valuationMethod": details["validationMethod"], - "analyses": [analysis], + "date": process_date, + "analyticalTechnique": technique, + "valuationMethod": method, + "analyses": analyses, } - - analysis_process = AnalysisProcess(**analysis_process_dict) - analysis_processes.append(analysis_process) + analysis_processes.append(AnalysisProcess(**analysis_process_dict)) return analysis_processes diff --git a/api/tests/test_gar_bulk_upload.py b/api/tests/test_gar_bulk_upload.py index 8b6c0bc..d41b2d2 100644 --- a/api/tests/test_gar_bulk_upload.py +++ b/api/tests/test_gar_bulk_upload.py @@ -383,13 +383,12 @@ def mock_csv_data(self): def test_old_format_single_parameter(self): """Test old format with a single field parameter""" row = pd.Series( - {"Zuurstof (mg/L)": 8.5, "pH": 7.2, "other_column": "some_value"} + {"Zuurstof (mg/l)": 8.5, "pH": 7.2, "other_column": "some_value"} ) result = create_gar_field_measurements(row) - # Zuurstof is written wrong - assert len(result) == 1 + assert len(result) == 2 assert all(isinstance(fm, FieldMeasurement) for fm in result) # Check pH measurement @@ -550,12 +549,57 @@ def test_new_format_csv_path(self, mock_read_csv, mock_csv_data): def test_format_detection_old_format(self): """Test that old format is correctly detected""" - row = pd.Series({"Zuurstof (mg/L)": 8.5, "pH": 7.2}) + row = pd.Series({"Zuurstof (mg/L)_field": 8.5, "pH_field": 7.2}) # Should use old format logic result = create_gar_field_measurements(row) assert len(result) == 1 # Only pH + def test_format_detection_old_format_lowercase_unit(self): + """Test that old format is detected for lowercase mg/l headers.""" + row = pd.Series({"Zuurstof (mg/l)_field": 8.5, "pH_field": 7.2}) + + result = create_gar_field_measurements(row) + + assert len(result) == 2 + parameter_ids = {measurement.parameter for measurement in result} + assert parameter_ids == {1398, 1701} + + def test_old_format_skips_niet_bepaald(self): + """Test old format skips textual missing values.""" + row = pd.Series( + { + "Zuurstof (mg/l)_field": 8.5, + "pH_field": 7.2, + "Alkaliniteit (HCO3 - mg/l)_field": "niet bepaald", + } + ) + + result = create_gar_field_measurements(row) + + assert len(result) == 2 + parameter_ids = {measurement.parameter for measurement in result} + assert 374 not in parameter_ids + + def test_old_format_prefers_field_suffix_when_lab_suffix_exists(self): + """When both files are merged, *_field columns should drive field measurements.""" + row = pd.Series( + { + "Zuurstof (mg/l)_field": 8.5, + "pH_field": 7.2, + "pH_lab": 99.9, + } + ) + + result = create_gar_field_measurements(row) + + assert len(result) == 2 + measurements_by_parameter = { + measurement.parameter: measurement for measurement in result + } + assert measurements_by_parameter[1398].field_measurement_value == 7.2 + assert measurements_by_parameter[1701].field_measurement_value == 8.5 + @patch("polars.read_csv") def test_format_detection_new_format(self, mock_read_csv, mock_csv_data): """Test that new format is correctly detected""" @@ -567,6 +611,26 @@ def test_format_detection_new_format(self, mock_read_csv, mock_csv_data): result = create_gar_field_measurements(row) assert len(result) == 1 + @patch("polars.read_csv") + def test_new_format_uses_field_suffix_and_ignores_lab_suffix( + self, mock_read_csv, mock_csv_data + ): + """Aquocode matching should map *_field columns and ignore *_lab columns.""" + mock_read_csv.return_value = mock_csv_data + + row = pd.Series( + { + "1112T4ClC2a_field": "5.2", + "1112T4ClC2a_lab": "9.9", + } + ) + + result = create_gar_field_measurements(row) + + assert len(result) == 1 + assert result[0].parameter == 3 + assert result[0].field_measurement_value == 5.2 + def test_empty_series(self): """Test with completely empty series""" row = pd.Series(dtype=object) @@ -685,3 +749,85 @@ def test_create_analysis_process(): analysis_process = analysis_process[0] assert analysis_process.analyses[0].analysis_measurement_value is None assert analysis_process.analyses[0].limit_symbol == result + + +def test_create_analysis_process_groups_parameters_per_method_and_technique(): + row = pd.Series( + { + "Cl (mg/l)": 0.1, + "Rapportagegrens Cl (mg/l)": 1, + "Analysedatum Cl (mg/l)": "2018-10-25", + "NO3 (mg/l)": 0.2, + "Rapportagegrens NO3 (mg/l)": 2, + "Analysedatum NO3 (mg/l)": "2018-10-25", + } + ) + + analysis_processes = create_analysis_process(row) + da_s_processes = [ + process + for process in analysis_processes + if process.analytical_technique == "DA-S" + and process.valuation_method == "I15923-1.13" + ] + + assert len(da_s_processes) == 1 + assert len(da_s_processes[0].analyses) == 2 + assert {analysis.parameter for analysis in da_s_processes[0].analyses} == { + 508, + 1270, + } + + +def test_create_analysis_process_skips_analysis_without_analysis_date(): + row = pd.Series( + { + "Cl (mg/l)": 0.1, + "Rapportagegrens Cl (mg/l)": 1, + } + ) + + analysis_processes = create_analysis_process(row) + + assert analysis_processes == [] + + +def test_create_analysis_process_prefers_lab_suffix_and_ignores_field_suffix(): + row = pd.Series( + { + "Cl (mg/l)_field": 99.9, + "Rapportagegrens Cl (mg/l)_field": 99, + "Analysedatum Cl (mg/l)_field": "2018-10-24", + "Cl (mg/l)_lab": 0.1, + "Rapportagegrens Cl (mg/l)_lab": 1, + "Analysedatum Cl (mg/l)_lab": "2018-10-25", + } + ) + + analysis_processes = create_analysis_process(row) + da_s_processes = [ + process + for process in analysis_processes + if process.analytical_technique == "DA-S" + and process.valuation_method == "I15923-1.13" + ] + + assert len(da_s_processes) == 1 + assert len(da_s_processes[0].analyses) == 1 + assert da_s_processes[0].date == "2018-10-25" + assert da_s_processes[0].analyses[0].analysis_measurement_value == 0.1 + assert da_s_processes[0].analyses[0].reporting_limit == 1 + + +def test_create_analysis_process_ignores_field_only_lab_measurement_columns(): + row = pd.Series( + { + "Cl (mg/l)_field": 0.1, + "Rapportagegrens Cl (mg/l)_field": 1, + "Analysedatum Cl (mg/l)_field": "2018-10-25", + } + ) + + analysis_processes = create_analysis_process(row) + + assert analysis_processes == []