From f6966d07454acbcdeb3382d98fa8142608100910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Wed, 23 Jul 2025 11:11:02 +0100 Subject: [PATCH 1/5] update tests --- tests/table_validator_tests/test_file_io.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/table_validator_tests/test_file_io.py b/tests/table_validator_tests/test_file_io.py index 5d523b1..f7263e0 100644 --- a/tests/table_validator_tests/test_file_io.py +++ b/tests/table_validator_tests/test_file_io.py @@ -75,3 +75,35 @@ def test_data_mapping(): assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" assert not sorted(my_ref_fileio_object_no_config.fields) == sorted(my_query_fileio_object.fields), "Column names should not match when config is not used" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_different_colnames(): + ## only colnames differ, values all same + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_colname.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_colname_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_colname.csv", sample_id_key="col_right_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_different_valmaps(): + ## only values mapped differently, colnames all same + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_valmap.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_valmap_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_valmap.csv", sample_id_key="col_left_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_subset_difference(): + ## colname different for col 2, valmap different for col 3 + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_one_valmap_one_col.csv", sample_id_key="col_right_1", config=TEST_DATA_DIR / "data_map_one_valmap_one_col_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_one_valmap_one_col.csv", sample_id_key="col_right_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" From 9a3cb7b5c50117aec330c5bd835e563d50ab7d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Wed, 23 Jul 2025 11:12:08 +0100 Subject: [PATCH 2/5] update code to allow flexile config definitions --- validation_toolkit/TableValidator/file_io.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index 06216a3..ab5bc03 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -305,17 +305,20 @@ def map_data(self, config): with open(config, "rt") as config_handle: config_dict = yaml.safe_load(config_handle) - col_mappings = {k: v["col_name"] for k, v in config_dict.items()} - val_mappings = {k: v["value_map"] for k, v in config_dict.items()} + col_mappings = {k: v.get("col_name", None) for k, v in config_dict.items()} + val_mappings = {k: v.get("value_map", {}) for k, v in config_dict.items()} mapped = {} for k, v in self.data_dict.items(): mapped_vals = {} for val_k, val_v in v.items(): - if val_mappings[val_k]: - mapped_vals[col_mappings[val_k]] = val_mappings[val_k][val_v] + alt_valname = col_mappings.get(val_k, "") + mapped_valname = alt_valname if alt_valname else val_k + + if val_mappings.get(val_k, ""): + mapped_vals[mapped_valname] = val_mappings[val_k][val_v] else: - mapped_vals[col_mappings[val_k]] = val_v + mapped_vals[mapped_valname] = val_v mapped[k] = mapped_vals - + print(f"{mapped = }") return mapped \ No newline at end of file From 5b1d114e96382dfcb50433b9f91763545293cad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Wed, 23 Jul 2025 11:12:52 +0100 Subject: [PATCH 3/5] add inputs for new tests --- .../test_data/data_map_colname_config.yaml | 6 ++++++ .../data_map_one_valmap_one_col_config.yaml | 7 +++++++ .../test_data/data_map_test_left_colname.csv | 6 ++++++ .../data_map_test_left_one_valmap_one_col.csv | 6 ++++++ .../test_data/data_map_test_left_valmap.csv | 6 ++++++ .../test_data/data_map_test_right_colname.csv | 6 ++++++ .../data_map_test_right_one_valmap_one_col.csv | 6 ++++++ .../test_data/data_map_valmap_config.yaml | 12 ++++++++++++ 8 files changed, 55 insertions(+) create mode 100644 tests/table_validator_tests/test_data/data_map_colname_config.yaml create mode 100644 tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml create mode 100644 tests/table_validator_tests/test_data/data_map_test_left_colname.csv create mode 100644 tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv create mode 100644 tests/table_validator_tests/test_data/data_map_test_left_valmap.csv create mode 100644 tests/table_validator_tests/test_data/data_map_test_right_colname.csv create mode 100644 tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv create mode 100644 tests/table_validator_tests/test_data/data_map_valmap_config.yaml diff --git a/tests/table_validator_tests/test_data/data_map_colname_config.yaml b/tests/table_validator_tests/test_data/data_map_colname_config.yaml new file mode 100644 index 0000000..9dd7916 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_colname_config.yaml @@ -0,0 +1,6 @@ +col_left_1: + col_name: col_right_1 +col_left_2: + col_name: col_right_2 +col_left_3: + col_name: col_right_3 diff --git a/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml b/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml new file mode 100644 index 0000000..102d75d --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml @@ -0,0 +1,7 @@ +col_left_2: + col_name: col_right_2 +col_right_3: + value_map: + resistant: 1 + sensitive: 0 + unknown: 2 diff --git a/tests/table_validator_tests/test_data/data_map_test_left_colname.csv b/tests/table_validator_tests/test_data/data_map_test_left_colname.csv new file mode 100644 index 0000000..bbf95a5 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_colname.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv b/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv new file mode 100644 index 0000000..9525121 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv @@ -0,0 +1,6 @@ +col_right_1,col_left_2,col_right_3 +sample1,1,resistant +sample2,0,resistant +sample3,0,unknown +sample4,0,sensitive +sample5,1,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv b/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv new file mode 100644 index 0000000..bbf95a5 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_right_colname.csv b/tests/table_validator_tests/test_data/data_map_test_right_colname.csv new file mode 100644 index 0000000..64ac8c4 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_colname.csv @@ -0,0 +1,6 @@ +col_right_1,col_right_2,col_right_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv b/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv new file mode 100644 index 0000000..220329d --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv @@ -0,0 +1,6 @@ +col_right_1,col_right_2,col_right_3 +sample1,1,1 +sample2,0,1 +sample3,0,2 +sample4,0,0 +sample5,1,0 \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_valmap_config.yaml b/tests/table_validator_tests/test_data/data_map_valmap_config.yaml new file mode 100644 index 0000000..a4d66d1 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_valmap_config.yaml @@ -0,0 +1,12 @@ +col_left_1: + value_map: null +col_left_2: + value_map: + true: 1 + false: 0 + unknown: null +col_left_3: + value_map: + resistant: 1 + sensitive: 0 + unknown: 2 From 62f51d3c68f2b5084a730af9c9690a9cfa945971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Wed, 23 Jul 2025 11:13:30 +0100 Subject: [PATCH 4/5] add inputs for new tests --- .../test_data/data_map_test_right_valmap.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/table_validator_tests/test_data/data_map_test_right_valmap.csv diff --git a/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv b/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv new file mode 100644 index 0000000..8237e01 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,1,1 +sample2,0,1 +sample3,0,2 +sample4,0,0 +sample5,1,0 \ No newline at end of file From 8cd929752b27d6ae6f67f50034b4d362bf8be48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Mon, 28 Jul 2025 21:29:15 +0100 Subject: [PATCH 5/5] remove unnecessary print statement --- validation_toolkit/TableValidator/file_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index ab5bc03..b093375 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -320,5 +320,4 @@ def map_data(self, config): else: mapped_vals[mapped_valname] = val_v mapped[k] = mapped_vals - print(f"{mapped = }") return mapped \ No newline at end of file