Handle composite IDs, refactor tests

README.md

@@ -138,3 +138,5 @@ Metadata lines are formatted as comments starting with two hashes; these are wri
 - `-c`, `--custom_config`: Custom config file containing col_name and value mappings in .yaml format.
 - `-o`, `--output_filename`: Path to output file to be written. [REQUIRED: TRUE]
 - `-f`, `--write_format`: File format to write comparison output to. [VALID: 'csv', 'json']
+
+> `ref_sampleid_key` and `query_sampleid_key` can be either a string, eg 'sample_id' or a comma-separated list of strings, eg 'sample_id,replicate_id' - referring to two columns, here sample_id and replicate_id, which when combined will produce a unique identifier. This is useful when columns like sample_id may contain duplicate entries but joining them with another column containing relevant identifiers produces unique IDs, so the 'sample_id' column in this example will contain duplicate entries if the example data contains replicates, but joining sample_id with replicate_id will produce a unique identifier for each row of data.

tests/table_validator_tests/test_data/composite_ids_test_config.yaml

@@ -0,0 +1,7 @@
+uid:
+  col_name: uuid
+data_col_2:
+  col_name: data_col_alt_2
+  value_map:
+    true: 1
+    false: 0

tests/table_validator_tests/test_data/composite_ids_test_left.csv

@@ -0,0 +1,10 @@
+sample_id,uid,data_col_1,data_col_2
+sample_1,1,true,true
+sample_1,2,true,false
+sample_1,3,false,true
+sample_2,1,true,true
+sample_2,2,true,false
+sample_2,3,false,true
+sample_3,1,true,true
+sample_3,2,true,false
+sample_3,3,false,true

tests/table_validator_tests/test_data/composite_ids_test_right.csv

@@ -0,0 +1,10 @@
+sample_id,uuid,data_col_1,data_col_alt_2
+sample_1,1,true,0
+sample_1,2,true,0
+sample_1,3,false,1
+sample_2,1,true,1
+sample_2,2,true,0
+sample_2,3,false,0
+sample_3,1,true,1
+sample_3,2,true,0
+sample_3,3,true,1

tests/table_validator_tests/test_file_compare.py

@@ -1,4 +1,6 @@
-import pytest, warnings
+import pytest
+import json
+import filecmp
 import difflib
 from importlib_resources import files
 
@@ -90,20 +92,7 @@ def test_compare_values_config(test_init_config):
     assert test_object_w_config.value_mismatches["sample1"] == {"sample_id": "sample1", "col_right_1": "ValueMatch", "col_right_2": "ValueMatch", "col_right_3": "ValueMatch"}
 
 def _files_are_identical(file1, file2):
-    with open(file1, "r") as f1:
-        with open(file2, "r") as f2:
-            diff = difflib.unified_diff(
-                f1.readlines(),
-                f2.readlines(),
-                fromfile="f1",
-                tofile="f2",
-            )
-    diff = [i for i in diff]
-
-    if diff:
-        return False, diff
-    else:
-        return True, []
+    return filecmp.cmp(f1=file1, f2=file2, shallow=False)
 
 def test_output_files_json(test_init, tmp_path):
     test_object = test_init
@@ -113,9 +102,13 @@ def test_output_files_json(test_init, tmp_path):
                 "prog": "TableValidator_pytest",
                 "prog_version": "0.0.1dev",
                 }
-    test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.json", out_format="json")
 
-    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match.json", file2=f"{tmp_path}/test_out_all_match.json")[0], "Files should be identical!"
+    test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.json", out_format="json")
+    with open(f"{tmp_path}/test_out_all_match.json", "r") as created_json:
+        out_data = json.load(created_json)
+    created_json.close()
+    expected_keys = ["metadata", "schema_check", "value_check"]
+    assert sorted(list(out_data.keys())) == expected_keys, f"Did not find all expected keys in output JSON. Expected: f{expected_keys}, found: {sorted(list(out_data.keys()))}"
 
 def test_output_files_csv(test_init, tmp_path):
     test_object = test_init
@@ -127,8 +120,10 @@ def test_output_files_csv(test_init, tmp_path):
                 }
     test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.csv", out_format="csv")
 
-    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_schema_check.csv", file2=f"{tmp_path}/test_out_all_match_schema_check.csv")[0], "Files should be identical!"
-    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_value_check.csv", file2=f"{tmp_path}/test_out_all_match_value_check.csv")[0], "Files should be identical!"
+    ## using fixture files here as it is useful to check this in a static way
+    ## fixtures can also be used as a reference for expected FileCompare output when using CSV as output format
+    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_schema_check.csv", file2=f"{tmp_path}/test_out_all_match_schema_check.csv"), "Files should be identical!"
+    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_value_check.csv", file2=f"{tmp_path}/test_out_all_match_value_check.csv"), "Files should be identical!"
 
 def test_output_files_schema_mismatch():
 
@@ -155,4 +150,17 @@ def test_output_files_value_mismatch(tmp_path):
                 }
     test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_value_mismatch.json", out_format="json")
 
-    assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_value_mismatch.json", file2=f"{tmp_path}/test_out_value_mismatch.json")[0], "Files should be identical!"
+    with open(f"{tmp_path}/test_out_value_mismatch.json", "r") as created_json:
+        out_data = json.load(created_json)
+    created_json.close()
+    assert out_data["value_check"]["sample_4"]["field2"] == "ValueMismatch. Expected 42 Found fourtwo"
+    assert out_data["metadata"] == metadata
+
+def test_composite_ids_compare_with_config():
+    comp_object = FileCompare(left_file= TEST_DATA_DIR / "composite_ids_test_left.csv", right_file=TEST_DATA_DIR / "composite_ids_test_right.csv", l_sample_id_key="sample_id,uid", r_sample_id_key="sample_id,uuid", config=TEST_DATA_DIR / "composite_ids_test_config.yaml")
+    with pytest.warns(ValueMismatchWarning):
+        comp_object.compare_values()
+
+    assert comp_object.value_mismatches["sample_1.1"]["data_col_alt_2"] == "ValueMismatch. Expected 1 Found 0", "Failed to find expected value mismatch!"
+    assert comp_object.value_mismatches["sample_2.3"]["data_col_alt_2"] == "ValueMismatch. Expected 1 Found 0", "Failed to find expected value mismatch!"
+    assert comp_object.value_mismatches["sample_3.3"]["data_col_1"] == "ValueMismatch. Expected False Found True", "Failed to find expected value mismatch!"

tests/table_validator_tests/test_file_io.py

@@ -66,6 +66,7 @@ def test_infer_types():
     assert my_fileio_object._infer_type(var="fAlse") == False, "Bool inference fails"
     assert my_fileio_object._infer_type(var="3.14") == 3.14, "Float inference fails"
     assert my_fileio_object._infer_type(var="simple_string") == "simple_string", "Inference incorrectly applied to string"
+    assert not my_fileio_object._infer_type(var=""), "empty string should evaluate to NoneType"
 
 def test_data_mapping():
     my_ref_fileio_object = File_IO(input_file= TEST_DATA_DIR / "data_map_test_left.csv", sample_id_key="col_left_1", config= TEST_DATA_DIR / "sample_config.yaml")
@@ -107,3 +108,8 @@ def test_data_mapping_only_subset_difference():
     assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong"
     for k, v in my_ref_fileio_object.data_dict.items():
         assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}"
+
+def test_composite_ids():
+    my_file_io_object = File_IO(input_file= TEST_DATA_DIR / "composite_ids_test_left.csv", sample_id_key="sample_id,uid")
+
+    assert "sample_1.1" in my_file_io_object.data_dict.keys(), "Composite IDs should have been constructed as 'sample_id_val' + '.' + 'uid_val' "

validation_toolkit/TableValidator/file_io.py

@@ -157,6 +157,12 @@ def _read_table(self, csv_or_tsv_file: str, sample_id_key: str):
         if not self.input_file_type in ["csv", "tsv"]:
             raise UnexpectedFileFormatError("Found unexpected file format. Supported formats are: [csv, tsv]")
 
+        if "," in sample_id_key:
+            sample_id_key_list = sample_id_key.split(",")
+            print(f"Using composite identifiers: {sample_id_key_list}")
+        else:
+            sample_id_key_list = [sample_id_key]
+
         ## use the file type to read file in with the correct delimiter
         with open(csv_or_tsv_file, "rt") as in_csv_or_tsv_handle:
             if self.input_file_type == "csv":
@@ -167,9 +173,12 @@ def _read_table(self, csv_or_tsv_file: str, sample_id_key: str):
             ## populate the data dictionary
             data = {}
             for row in file_reader:
-                sample_id = row.get(sample_id_key, "")
-                if not sample_id:
-                    raise KeyError(f"Specified sample ID key: {sample_id_key} not found in table.")
+                sample_id_components = [row.get(id_key, "") for id_key in sample_id_key_list]
+                for i, sample_id_found in enumerate(sample_id_components):
+                    if sample_id_found == "":
+                        sample_id_not_found = sample_id_key_list[i]
+                        raise KeyError(f"Specified sample ID key: {sample_id_not_found} not found in table.")
+                sample_id = ".".join(sample_id_components)
                 sample_data = {k: v for k, v in row.items()}
                 data[sample_id] = sample_data
         in_csv_or_tsv_handle.close()
@@ -232,14 +241,17 @@ def _infer_type(self, var: str):
             str | int | float | bool: The var input, converted to int/float/bool as applicable
         """
         bool_map = {"true": True, "false": False}
+        ## catch case where var is empty
+        if not var:
+            return None
         ## first check if the string is numeric
         ## if yes, return the int
         if var.isnumeric():
             return int(var)
         ## next check if string might contain a bool expression
         ## if so, return the correct bool
         elif var.lower() in ["true", "false"]:
-                return bool_map[var.lower()]
+            return bool_map[var.lower()]
         else:
             try:
                 ## next try casting it to float