diff --git a/tests/table_validator_tests/test_data/data_map_colname_config.yaml b/tests/table_validator_tests/test_data/data_map_colname_config.yaml new file mode 100644 index 0000000..9dd7916 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_colname_config.yaml @@ -0,0 +1,6 @@ +col_left_1: + col_name: col_right_1 +col_left_2: + col_name: col_right_2 +col_left_3: + col_name: col_right_3 diff --git a/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml b/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml new file mode 100644 index 0000000..102d75d --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml @@ -0,0 +1,7 @@ +col_left_2: + col_name: col_right_2 +col_right_3: + value_map: + resistant: 1 + sensitive: 0 + unknown: 2 diff --git a/tests/table_validator_tests/test_data/data_map_test_left_colname.csv b/tests/table_validator_tests/test_data/data_map_test_left_colname.csv new file mode 100644 index 0000000..bbf95a5 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_colname.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv b/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv new file mode 100644 index 0000000..9525121 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv @@ -0,0 +1,6 @@ +col_right_1,col_left_2,col_right_3 +sample1,1,resistant +sample2,0,resistant +sample3,0,unknown +sample4,0,sensitive +sample5,1,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv b/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv new file mode 100644 index 0000000..bbf95a5 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_left_valmap.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_right_colname.csv b/tests/table_validator_tests/test_data/data_map_test_right_colname.csv new file mode 100644 index 0000000..64ac8c4 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_colname.csv @@ -0,0 +1,6 @@ +col_right_1,col_right_2,col_right_3 +sample1,True,resistant +sample2,False,resistant +sample3,False,unknown +sample4,False,sensitive +sample5,True,sensitive \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv b/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv new file mode 100644 index 0000000..220329d --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv @@ -0,0 +1,6 @@ +col_right_1,col_right_2,col_right_3 +sample1,1,1 +sample2,0,1 +sample3,0,2 +sample4,0,0 +sample5,1,0 \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv b/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv new file mode 100644 index 0000000..8237e01 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_test_right_valmap.csv @@ -0,0 +1,6 @@ +col_left_1,col_left_2,col_left_3 +sample1,1,1 +sample2,0,1 +sample3,0,2 +sample4,0,0 +sample5,1,0 \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/data_map_valmap_config.yaml b/tests/table_validator_tests/test_data/data_map_valmap_config.yaml new file mode 100644 index 0000000..a4d66d1 --- /dev/null +++ b/tests/table_validator_tests/test_data/data_map_valmap_config.yaml @@ -0,0 +1,12 @@ +col_left_1: + value_map: null +col_left_2: + value_map: + true: 1 + false: 0 + unknown: null +col_left_3: + value_map: + resistant: 1 + sensitive: 0 + unknown: 2 diff --git a/tests/table_validator_tests/test_file_io.py b/tests/table_validator_tests/test_file_io.py index 5d523b1..f7263e0 100644 --- a/tests/table_validator_tests/test_file_io.py +++ b/tests/table_validator_tests/test_file_io.py @@ -75,3 +75,35 @@ def test_data_mapping(): assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" assert not sorted(my_ref_fileio_object_no_config.fields) == sorted(my_query_fileio_object.fields), "Column names should not match when config is not used" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_different_colnames(): + ## only colnames differ, values all same + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_colname.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_colname_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_colname.csv", sample_id_key="col_right_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_different_valmaps(): + ## only values mapped differently, colnames all same + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_valmap.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_valmap_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_valmap.csv", sample_id_key="col_left_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_data_mapping_only_subset_difference(): + ## colname different for col 2, valmap different for col 3 + my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_one_valmap_one_col.csv", sample_id_key="col_right_1", config=TEST_DATA_DIR / "data_map_one_valmap_one_col_config.yaml") + my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_one_valmap_one_col.csv", sample_id_key="col_right_1") + + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" + assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" + for k, v in my_ref_fileio_object.data_dict.items(): + assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index 06216a3..b093375 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -305,17 +305,19 @@ def map_data(self, config): with open(config, "rt") as config_handle: config_dict = yaml.safe_load(config_handle) - col_mappings = {k: v["col_name"] for k, v in config_dict.items()} - val_mappings = {k: v["value_map"] for k, v in config_dict.items()} + col_mappings = {k: v.get("col_name", None) for k, v in config_dict.items()} + val_mappings = {k: v.get("value_map", {}) for k, v in config_dict.items()} mapped = {} for k, v in self.data_dict.items(): mapped_vals = {} for val_k, val_v in v.items(): - if val_mappings[val_k]: - mapped_vals[col_mappings[val_k]] = val_mappings[val_k][val_v] + alt_valname = col_mappings.get(val_k, "") + mapped_valname = alt_valname if alt_valname else val_k + + if val_mappings.get(val_k, ""): + mapped_vals[mapped_valname] = val_mappings[val_k][val_v] else: - mapped_vals[col_mappings[val_k]] = val_v + mapped_vals[mapped_valname] = val_v mapped[k] = mapped_vals - return mapped \ No newline at end of file