Allow flexible schema config definitions

tests/table_validator_tests/test_data/data_map_colname_config.yaml

@@ -0,0 +1,6 @@
+col_left_1:
+  col_name: col_right_1
+col_left_2:
+  col_name: col_right_2
+col_left_3:
+  col_name: col_right_3

tests/table_validator_tests/test_data/data_map_one_valmap_one_col_config.yaml

@@ -0,0 +1,7 @@
+col_left_2:
+  col_name: col_right_2
+col_right_3:
+  value_map:
+    resistant: 1
+    sensitive: 0
+    unknown: 2

tests/table_validator_tests/test_data/data_map_test_left_colname.csv

@@ -0,0 +1,6 @@
+col_left_1,col_left_2,col_left_3
+sample1,True,resistant
+sample2,False,resistant
+sample3,False,unknown
+sample4,False,sensitive
+sample5,True,sensitive

tests/table_validator_tests/test_data/data_map_test_left_one_valmap_one_col.csv

@@ -0,0 +1,6 @@
+col_right_1,col_left_2,col_right_3
+sample1,1,resistant
+sample2,0,resistant
+sample3,0,unknown
+sample4,0,sensitive
+sample5,1,sensitive

tests/table_validator_tests/test_data/data_map_test_left_valmap.csv

@@ -0,0 +1,6 @@
+col_left_1,col_left_2,col_left_3
+sample1,True,resistant
+sample2,False,resistant
+sample3,False,unknown
+sample4,False,sensitive
+sample5,True,sensitive

tests/table_validator_tests/test_data/data_map_test_right_colname.csv

@@ -0,0 +1,6 @@
+col_right_1,col_right_2,col_right_3
+sample1,True,resistant
+sample2,False,resistant
+sample3,False,unknown
+sample4,False,sensitive
+sample5,True,sensitive

tests/table_validator_tests/test_data/data_map_test_right_one_valmap_one_col.csv

@@ -0,0 +1,6 @@
+col_right_1,col_right_2,col_right_3
+sample1,1,1
+sample2,0,1
+sample3,0,2
+sample4,0,0
+sample5,1,0

tests/table_validator_tests/test_data/data_map_test_right_valmap.csv

@@ -0,0 +1,6 @@
+col_left_1,col_left_2,col_left_3
+sample1,1,1
+sample2,0,1
+sample3,0,2
+sample4,0,0
+sample5,1,0

tests/table_validator_tests/test_data/data_map_valmap_config.yaml

@@ -0,0 +1,12 @@
+col_left_1:
+  value_map: null
+col_left_2:
+  value_map:
+    true: 1
+    false: 0
+    unknown: null
+col_left_3:
+  value_map:
+    resistant: 1
+    sensitive: 0
+    unknown: 2

tests/table_validator_tests/test_file_io.py

@@ -75,3 +75,35 @@ def test_data_mapping():
     assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids"
     assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong"
     assert not sorted(my_ref_fileio_object_no_config.fields) == sorted(my_query_fileio_object.fields), "Column names should not match when config is not used"
+    for k, v in my_ref_fileio_object.data_dict.items():
+        assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}"
+
+def test_data_mapping_only_different_colnames():
+    ## only colnames differ, values all same
+    my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_colname.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_colname_config.yaml")
+    my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_colname.csv", sample_id_key="col_right_1")
+
+    assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids"
+    assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong"
+    for k, v in my_ref_fileio_object.data_dict.items():
+        assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}"
+
+def test_data_mapping_only_different_valmaps():
+    ## only values mapped differently, colnames all same
+    my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_valmap.csv", sample_id_key="col_left_1", config=TEST_DATA_DIR / "data_map_valmap_config.yaml")
+    my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_valmap.csv", sample_id_key="col_left_1")
+
+    assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids"
+    assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong"
+    for k, v in my_ref_fileio_object.data_dict.items():
+        assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}"
+
+def test_data_mapping_only_subset_difference():
+    ## colname different for col 2, valmap different for col 3
+    my_ref_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_left_one_valmap_one_col.csv", sample_id_key="col_right_1", config=TEST_DATA_DIR / "data_map_one_valmap_one_col_config.yaml")
+    my_query_fileio_object = File_IO(input_file=TEST_DATA_DIR / "data_map_test_right_one_valmap_one_col.csv", sample_id_key="col_right_1")
+
+    assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids"
+    assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong"
+    for k, v in my_ref_fileio_object.data_dict.items():
+        assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}"

validation_toolkit/TableValidator/file_io.py

@@ -305,17 +305,19 @@ def map_data(self, config):
         with open(config, "rt") as config_handle:
             config_dict = yaml.safe_load(config_handle)
 
-        col_mappings = {k: v["col_name"] for k, v in config_dict.items()}
-        val_mappings = {k: v["value_map"] for k, v in config_dict.items()}
+        col_mappings = {k: v.get("col_name", None) for k, v in config_dict.items()}
+        val_mappings = {k: v.get("value_map", {}) for k, v in config_dict.items()}
 
         mapped = {}
         for k, v in self.data_dict.items():
             mapped_vals = {}
             for val_k, val_v in v.items():
-                if val_mappings[val_k]:
-                    mapped_vals[col_mappings[val_k]] = val_mappings[val_k][val_v]
+                alt_valname = col_mappings.get(val_k, "")
+                mapped_valname = alt_valname if alt_valname else val_k
+
+                if val_mappings.get(val_k, ""):
+                    mapped_vals[mapped_valname] = val_mappings[val_k][val_v]
                 else:
-                    mapped_vals[col_mappings[val_k]] = val_v
+                    mapped_vals[mapped_valname] = val_v
             mapped[k] = mapped_vals
-
         return mapped