diff --git a/README.md b/README.md index 0da7121..c59ccc3 100644 --- a/README.md +++ b/README.md @@ -138,3 +138,5 @@ Metadata lines are formatted as comments starting with two hashes; these are wri - `-c`, `--custom_config`: Custom config file containing col_name and value mappings in .yaml format. - `-o`, `--output_filename`: Path to output file to be written. [REQUIRED: TRUE] - `-f`, `--write_format`: File format to write comparison output to. [VALID: 'csv', 'json'] + +> `ref_sampleid_key` and `query_sampleid_key` can be either a string, eg 'sample_id' or a comma-separated list of strings, eg 'sample_id,replicate_id' - referring to two columns, here sample_id and replicate_id, which when combined will produce a unique identifier. This is useful when columns like sample_id may contain duplicate entries but joining them with another column containing relevant identifiers produces unique IDs, so the 'sample_id' column in this example will contain duplicate entries if the example data contains replicates, but joining sample_id with replicate_id will produce a unique identifier for each row of data. diff --git a/tests/table_validator_tests/test_data/composite_ids_test_config.yaml b/tests/table_validator_tests/test_data/composite_ids_test_config.yaml new file mode 100644 index 0000000..c873b8f --- /dev/null +++ b/tests/table_validator_tests/test_data/composite_ids_test_config.yaml @@ -0,0 +1,7 @@ +uid: + col_name: uuid +data_col_2: + col_name: data_col_alt_2 + value_map: + true: 1 + false: 0 \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/composite_ids_test_left.csv b/tests/table_validator_tests/test_data/composite_ids_test_left.csv new file mode 100644 index 0000000..975edd4 --- /dev/null +++ b/tests/table_validator_tests/test_data/composite_ids_test_left.csv @@ -0,0 +1,10 @@ +sample_id,uid,data_col_1,data_col_2 +sample_1,1,true,true +sample_1,2,true,false +sample_1,3,false,true +sample_2,1,true,true +sample_2,2,true,false +sample_2,3,false,true +sample_3,1,true,true +sample_3,2,true,false +sample_3,3,false,true \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/composite_ids_test_right.csv b/tests/table_validator_tests/test_data/composite_ids_test_right.csv new file mode 100644 index 0000000..fa73d61 --- /dev/null +++ b/tests/table_validator_tests/test_data/composite_ids_test_right.csv @@ -0,0 +1,10 @@ +sample_id,uuid,data_col_1,data_col_alt_2 +sample_1,1,true,0 +sample_1,2,true,0 +sample_1,3,false,1 +sample_2,1,true,1 +sample_2,2,true,0 +sample_2,3,false,0 +sample_3,1,true,1 +sample_3,2,true,0 +sample_3,3,true,1 \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/pytest_out_all_match.json b/tests/table_validator_tests/test_data/pytest_out_all_match.json deleted file mode 100644 index 87a7252..0000000 --- a/tests/table_validator_tests/test_data/pytest_out_all_match.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "metadata": { - "prog": "TableValidator_pytest", - "prog_version": "0.0.1dev" - }, - "schema_check": { - "sample_1": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_2": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_3": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_4": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_5": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - } - }, - "value_check": { - "sample_1": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_2": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_3": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_4": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_5": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - } - } -} \ No newline at end of file diff --git a/tests/table_validator_tests/test_data/pytest_out_value_mismatch.json b/tests/table_validator_tests/test_data/pytest_out_value_mismatch.json deleted file mode 100644 index 2da9970..0000000 --- a/tests/table_validator_tests/test_data/pytest_out_value_mismatch.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "metadata": { - "prog": "TableValidator_pytest", - "prog_version": "0.0.1dev" - }, - "schema_check": { - "sample_1": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_2": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_3": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_4": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - }, - "sample_5": { - "sample_id": "SchemaMatch", - "field1": "SchemaMatch", - "field2": "SchemaMatch", - "field3": "SchemaMatch" - } - }, - "value_check": { - "sample_1": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_2": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_3": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - }, - "sample_4": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMismatch. Expected 42 Found fourtwo", - "field3": "ValueMatch" - }, - "sample_5": { - "sample_id": "ValueMatch", - "field1": "ValueMatch", - "field2": "ValueMatch", - "field3": "ValueMatch" - } - } -} \ No newline at end of file diff --git a/tests/table_validator_tests/test_file_compare.py b/tests/table_validator_tests/test_file_compare.py index f76db80..d4ada50 100644 --- a/tests/table_validator_tests/test_file_compare.py +++ b/tests/table_validator_tests/test_file_compare.py @@ -1,4 +1,6 @@ -import pytest, warnings +import pytest +import json +import filecmp import difflib from importlib_resources import files @@ -90,20 +92,7 @@ def test_compare_values_config(test_init_config): assert test_object_w_config.value_mismatches["sample1"] == {"sample_id": "sample1", "col_right_1": "ValueMatch", "col_right_2": "ValueMatch", "col_right_3": "ValueMatch"} def _files_are_identical(file1, file2): - with open(file1, "r") as f1: - with open(file2, "r") as f2: - diff = difflib.unified_diff( - f1.readlines(), - f2.readlines(), - fromfile="f1", - tofile="f2", - ) - diff = [i for i in diff] - - if diff: - return False, diff - else: - return True, [] + return filecmp.cmp(f1=file1, f2=file2, shallow=False) def test_output_files_json(test_init, tmp_path): test_object = test_init @@ -113,9 +102,13 @@ def test_output_files_json(test_init, tmp_path): "prog": "TableValidator_pytest", "prog_version": "0.0.1dev", } - test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.json", out_format="json") - assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match.json", file2=f"{tmp_path}/test_out_all_match.json")[0], "Files should be identical!" + test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.json", out_format="json") + with open(f"{tmp_path}/test_out_all_match.json", "r") as created_json: + out_data = json.load(created_json) + created_json.close() + expected_keys = ["metadata", "schema_check", "value_check"] + assert sorted(list(out_data.keys())) == expected_keys, f"Did not find all expected keys in output JSON. Expected: f{expected_keys}, found: {sorted(list(out_data.keys()))}" def test_output_files_csv(test_init, tmp_path): test_object = test_init @@ -127,8 +120,10 @@ def test_output_files_csv(test_init, tmp_path): } test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_all_match.csv", out_format="csv") - assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_schema_check.csv", file2=f"{tmp_path}/test_out_all_match_schema_check.csv")[0], "Files should be identical!" - assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_value_check.csv", file2=f"{tmp_path}/test_out_all_match_value_check.csv")[0], "Files should be identical!" + ## using fixture files here as it is useful to check this in a static way + ## fixtures can also be used as a reference for expected FileCompare output when using CSV as output format + assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_schema_check.csv", file2=f"{tmp_path}/test_out_all_match_schema_check.csv"), "Files should be identical!" + assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_value_check.csv", file2=f"{tmp_path}/test_out_all_match_value_check.csv"), "Files should be identical!" def test_output_files_schema_mismatch(): @@ -155,4 +150,17 @@ def test_output_files_value_mismatch(tmp_path): } test_object.write_results(metadata=metadata, output=f"{tmp_path}/test_out_value_mismatch.json", out_format="json") - assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_value_mismatch.json", file2=f"{tmp_path}/test_out_value_mismatch.json")[0], "Files should be identical!" + with open(f"{tmp_path}/test_out_value_mismatch.json", "r") as created_json: + out_data = json.load(created_json) + created_json.close() + assert out_data["value_check"]["sample_4"]["field2"] == "ValueMismatch. Expected 42 Found fourtwo" + assert out_data["metadata"] == metadata + +def test_composite_ids_compare_with_config(): + comp_object = FileCompare(left_file= TEST_DATA_DIR / "composite_ids_test_left.csv", right_file=TEST_DATA_DIR / "composite_ids_test_right.csv", l_sample_id_key="sample_id,uid", r_sample_id_key="sample_id,uuid", config=TEST_DATA_DIR / "composite_ids_test_config.yaml") + with pytest.warns(ValueMismatchWarning): + comp_object.compare_values() + + assert comp_object.value_mismatches["sample_1.1"]["data_col_alt_2"] == "ValueMismatch. Expected 1 Found 0", "Failed to find expected value mismatch!" + assert comp_object.value_mismatches["sample_2.3"]["data_col_alt_2"] == "ValueMismatch. Expected 1 Found 0", "Failed to find expected value mismatch!" + assert comp_object.value_mismatches["sample_3.3"]["data_col_1"] == "ValueMismatch. Expected False Found True", "Failed to find expected value mismatch!" diff --git a/tests/table_validator_tests/test_file_io.py b/tests/table_validator_tests/test_file_io.py index f7263e0..a6848ea 100644 --- a/tests/table_validator_tests/test_file_io.py +++ b/tests/table_validator_tests/test_file_io.py @@ -66,6 +66,7 @@ def test_infer_types(): assert my_fileio_object._infer_type(var="fAlse") == False, "Bool inference fails" assert my_fileio_object._infer_type(var="3.14") == 3.14, "Float inference fails" assert my_fileio_object._infer_type(var="simple_string") == "simple_string", "Inference incorrectly applied to string" + assert not my_fileio_object._infer_type(var=""), "empty string should evaluate to NoneType" def test_data_mapping(): my_ref_fileio_object = File_IO(input_file= TEST_DATA_DIR / "data_map_test_left.csv", sample_id_key="col_left_1", config= TEST_DATA_DIR / "sample_config.yaml") @@ -107,3 +108,8 @@ def test_data_mapping_only_subset_difference(): assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" for k, v in my_ref_fileio_object.data_dict.items(): assert v == my_query_fileio_object.data_dict[k], f"Value discrepancies found for sample {k}.\nLeft = {v}\nRight = {my_query_fileio_object.data_dict[k]}" + +def test_composite_ids(): + my_file_io_object = File_IO(input_file= TEST_DATA_DIR / "composite_ids_test_left.csv", sample_id_key="sample_id,uid") + + assert "sample_1.1" in my_file_io_object.data_dict.keys(), "Composite IDs should have been constructed as 'sample_id_val' + '.' + 'uid_val' " diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index b093375..d54bb70 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -157,6 +157,12 @@ def _read_table(self, csv_or_tsv_file: str, sample_id_key: str): if not self.input_file_type in ["csv", "tsv"]: raise UnexpectedFileFormatError("Found unexpected file format. Supported formats are: [csv, tsv]") + if "," in sample_id_key: + sample_id_key_list = sample_id_key.split(",") + print(f"Using composite identifiers: {sample_id_key_list}") + else: + sample_id_key_list = [sample_id_key] + ## use the file type to read file in with the correct delimiter with open(csv_or_tsv_file, "rt") as in_csv_or_tsv_handle: if self.input_file_type == "csv": @@ -167,9 +173,12 @@ def _read_table(self, csv_or_tsv_file: str, sample_id_key: str): ## populate the data dictionary data = {} for row in file_reader: - sample_id = row.get(sample_id_key, "") - if not sample_id: - raise KeyError(f"Specified sample ID key: {sample_id_key} not found in table.") + sample_id_components = [row.get(id_key, "") for id_key in sample_id_key_list] + for i, sample_id_found in enumerate(sample_id_components): + if sample_id_found == "": + sample_id_not_found = sample_id_key_list[i] + raise KeyError(f"Specified sample ID key: {sample_id_not_found} not found in table.") + sample_id = ".".join(sample_id_components) sample_data = {k: v for k, v in row.items()} data[sample_id] = sample_data in_csv_or_tsv_handle.close() @@ -232,6 +241,9 @@ def _infer_type(self, var: str): str | int | float | bool: The var input, converted to int/float/bool as applicable """ bool_map = {"true": True, "false": False} + ## catch case where var is empty + if not var: + return None ## first check if the string is numeric ## if yes, return the int if var.isnumeric(): @@ -239,7 +251,7 @@ def _infer_type(self, var: str): ## next check if string might contain a bool expression ## if so, return the correct bool elif var.lower() in ["true", "false"]: - return bool_map[var.lower()] + return bool_map[var.lower()] else: try: ## next try casting it to float