From 32116250694c6e26e1619142421f195fc72a271b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:08:55 +0100 Subject: [PATCH 1/9] cleanup, use data_reader_dict arg in fn map_data --- validation_toolkit/TableValidator/file_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index 9e7ded2..4747f23 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -55,7 +55,6 @@ def __init__(self, input_file: str, sample_id_key: str, config: str = None, data else: self.data_reader = read_functions[self.input_file_type](self.full_path_input_file) - ##TODO: Add this method if config: self.data_reader = self.map_data(data_reader_dict=self.data_reader, config=config) @@ -310,7 +309,7 @@ def map_data(self, data_reader_dict, config): val_mappings = {k: v["value_map"] for k, v in config_dict.items()} mapped = {} - for k, v in self.data_reader.items(): + for k, v in data_reader_dict.items(): mapped_vals = {} for val_k, val_v in v.items(): if val_mappings[val_k]: From ce4afd196fc318dda8983848faab2cbfad5c3cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:24:42 +0100 Subject: [PATCH 2/9] move FileIO invocation and config handling to init, call schema methods inside compare method --- .../TableValidator/file_compare.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/validation_toolkit/TableValidator/file_compare.py b/validation_toolkit/TableValidator/file_compare.py index 40976a3..ffe86f7 100644 --- a/validation_toolkit/TableValidator/file_compare.py +++ b/validation_toolkit/TableValidator/file_compare.py @@ -1,10 +1,11 @@ import warnings import json from collections import defaultdict +from validation_toolkit.TableValidator.file_io import File_IO from validation_toolkit.TableValidator.custom_exceptions import KeyMismatchWarning, SchemaMismatchWarning, FieldMismatchWarning, ValueMismatchWarning, MissingSampleWarning class FileCompare(): - def __init__(self, left_data: dict, right_data: dict = None, strict: bool = False): + def __init__(self, left_file: str, right_file: str, l_sample_id_key: str, r_sample_id_key: str = None, config: str = None, strict: bool = False): """Initialise a file comparison object with reference and query datasets Args: @@ -12,8 +13,26 @@ def __init__(self, left_data: dict, right_data: dict = None, strict: bool = Fals right_data (dict, optional): Data formatted by FileIO from the query/subject table strict (bool, optional): Whether to elevate warnings to exceptions and terminate the program. Defaults to False. """ - self.ref_table = left_data - self.query_table = right_data + + self.ref_file = left_file + self.query_file = right_file + + ## load reference data + ## if config provided then this will be used to map column names + self.ref_table = File_IO(input_file=left_file, sample_id_key=l_sample_id_key, config=config).data_reader + + ## load query data + ## if unique key provided for sample_id column, use that + ## else assume same sampl_id_key as reference data + if r_sample_id_key: + self.query_table = File_IO(input_file=right_file, sample_id_key=r_sample_id_key).data_reader + else: + print("No sample_id_key provided for query table, assuming it is the same as sample_id_key for reference table.") + try: + self.query_table = File_IO(input_file=right_file, sample_id_key=l_sample_id_key).data_reader + except KeyError as ke: + raise ValueError(f"\n==\nDid not find any column {l_sample_id_key} in query table. Please check that you have provided a config file and a valid query sample_id_key on the command line.\n==\n") + self.schema_errors = defaultdict(dict) self.value_mismatches = defaultdict(dict) @@ -21,7 +40,7 @@ def __init__(self, left_data: dict, right_data: dict = None, strict: bool = Fals if strict: warnings.filterwarnings("error") - def create_schema(self, required_fields: list = None): + def _create_schema(self, required_fields: list = None): """Using the reference data, infer a "schema": expected field names and expected data types for each field Args: @@ -62,7 +81,7 @@ def create_schema(self, required_fields: list = None): for k in self.sample_schema.keys(): self.sample_schema[k].update({"required": True}) - def confirm_schema(self): + def _confirm_schema(self): """Function to check data types in the query/subject table to ensure it matches the schema inferred from the reference table """ @@ -106,6 +125,13 @@ def compare_values(self, field_names: list = None, col_maps: dict = None): field_names (list, optional): List of field names to compare values for. Defaults to None and all fields are checked. col_maps (dict, optional): Mapping of column names between reference and query datasets if the schemas are not identical. Defaults to None. """ + + ## create schema from ref_table + ## check query_table schema against it + print("Setting up and validating schema...") + self._create_schema() + self._confirm_schema() + if col_maps: raise NotImplementedError("Operation mode that uses this argument is not yet implemented.") From 682a36b306187c553bf2f96f49168da3be3a2f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:25:27 +0100 Subject: [PATCH 3/9] update FileCompare invocation to match class definition --- .../test_file_compare.py | 80 ++++++++----------- 1 file changed, 32 insertions(+), 48 deletions(-) diff --git a/tests/table_validator_tests/test_file_compare.py b/tests/table_validator_tests/test_file_compare.py index c1c2ace..f76db80 100644 --- a/tests/table_validator_tests/test_file_compare.py +++ b/tests/table_validator_tests/test_file_compare.py @@ -12,75 +12,65 @@ @pytest.fixture def test_init(): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_json.json", sample_id_key="sample_id") - test_object = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) + test_object = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "ref_toy_json.json", l_sample_id_key="sample_id") assert test_object, "A test file_compare object should be built" return test_object @pytest.fixture def test_init_config(): - my_ref_fileio_object = File_IO(input_file= TEST_DATA_DIR / "data_map_test_left.csv", sample_id_key="col_left_1", config= TEST_DATA_DIR / "sample_config.yaml") - my_ref_fileio_object_no_config = File_IO(input_file= TEST_DATA_DIR / "data_map_test_left.csv", sample_id_key="col_left_1") - my_query_fileio_object = File_IO(input_file= TEST_DATA_DIR / "data_map_test_right.csv", sample_id_key="col_right_1") - test_object_no_config = FileCompare(left_data=my_ref_fileio_object_no_config.data_reader, right_data=my_query_fileio_object.data_reader) - test_object_w_config = FileCompare(left_data=my_ref_fileio_object.data_reader, right_data=my_query_fileio_object.data_reader) + test_object_no_config = FileCompare(left_file=TEST_DATA_DIR / "data_map_test_left.csv", right_file=TEST_DATA_DIR / "data_map_test_right.csv", l_sample_id_key="col_left_1", r_sample_id_key="col_right_1") + test_object_w_config = FileCompare(left_file=TEST_DATA_DIR / "data_map_test_left.csv", right_file=TEST_DATA_DIR / "data_map_test_right.csv", l_sample_id_key="col_left_1", r_sample_id_key="col_right_1", config=TEST_DATA_DIR / "sample_config.yaml") return test_object_no_config, test_object_w_config def test_create_schema(test_init): test_object = test_init - test_object.create_schema() + test_object._create_schema() assert len(test_object.sample_schema.keys()) == 4, "There should be 4 fields in the schema" assert test_object.sample_schema["sample_id"]["valid_types"] == test_object.sample_schema["field1"]["valid_types"] == set([str]) def test_confirm_schema(test_init): test_object = test_init - test_object.create_schema() - test_object.confirm_schema() + test_object._create_schema() + test_object._confirm_schema() assert test_object.schema_errors["sample_1"] == {"sample_id": "SchemaMatch", "field1": "SchemaMatch", "field2": "SchemaMatch", "field3": "SchemaMatch"} def test_compare_values(test_init): test_object = test_init - test_object.create_schema() test_object.compare_values() assert test_object.value_mismatches["sample_1"] == {"sample_id": "ValueMatch", "field1": "ValueMatch", "field2": "ValueMatch", "field3": "ValueMatch"} def test_confirm_schema_fail(): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "query_toy_json_schemamismatch.json", sample_id_key="sample_id") - test_object_lenient = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) - test_object_lenient.create_schema() + test_object_lenient = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_schemamismatch.json", l_sample_id_key="sample_id") + test_object_lenient._create_schema() ## there should be a schema check failure here with pytest.warns(SchemaMismatchWarning): - test_object_lenient.confirm_schema() + test_object_lenient._confirm_schema() ## using strict mode, this warning should now be an exception -- pytest.raises needed to catch it - test_object_strict = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader, strict=True) - test_object_strict.create_schema() + test_object_strict = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_schemamismatch.json", l_sample_id_key="sample_id", strict=True) + test_object_strict._create_schema() with pytest.raises(SchemaMismatchWarning) as error_info: - test_object_strict.confirm_schema() + test_object_strict._confirm_schema() def test_compare_values_fail(): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "query_toy_json_valmismatch.json", sample_id_key="sample_id") - test_object = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) - test_object.create_schema() - test_object.confirm_schema() + + test_object = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_valmismatch.json", l_sample_id_key="sample_id") + test_object._create_schema() + test_object._confirm_schema() ## there should be a value mismatch flagged here with pytest.warns(ValueMismatchWarning, match="Mismatch in value of field field2 for sample sample_4. Expected: 42 Found: fourtwo"): test_object.compare_values() def test_compare_values_specific_fields(): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "query_toy_json_valmismatch.json", sample_id_key="sample_id") - test_object = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) - test_object.create_schema() - test_object.confirm_schema() + + test_object = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_valmismatch.json", l_sample_id_key="sample_id") + test_object._create_schema() + test_object._confirm_schema() ## even though this test uses a value_mismatch test fixture ## there should be no warnings as the field in question is not included in the provided list of field names to check test_object.compare_values(field_names=["sample_id", "field1", "field3"]) @@ -88,14 +78,13 @@ def test_compare_values_specific_fields(): def test_compare_fail_with_no_config(test_init_config): test_object_no_config, _ = test_init_config - test_object_no_config.create_schema() + test_object_no_config._create_schema() with pytest.raises(Exception): - test_object_no_config.confirm_schema() + test_object_no_config._confirm_schema() def test_compare_values_config(test_init_config): _, test_object_w_config = test_init_config - test_object_w_config.create_schema() - test_object_w_config.confirm_schema() + test_object_w_config.compare_values() assert test_object_w_config.schema_errors["sample1"] == {"sample_id": "sample1", "col_right_1": "SchemaMatch", "col_right_2": "SchemaMatch", "col_right_3": "SchemaMatch"} assert test_object_w_config.value_mismatches["sample1"] == {"sample_id": "sample1", "col_right_1": "ValueMatch", "col_right_2": "ValueMatch", "col_right_3": "ValueMatch"} @@ -118,8 +107,7 @@ def _files_are_identical(file1, file2): def test_output_files_json(test_init, tmp_path): test_object = test_init - test_object.create_schema() - test_object.confirm_schema() + test_object.compare_values() metadata = { "prog": "TableValidator_pytest", @@ -131,8 +119,7 @@ def test_output_files_json(test_init, tmp_path): def test_output_files_csv(test_init, tmp_path): test_object = test_init - test_object.create_schema() - test_object.confirm_schema() + test_object.compare_values() metadata = { "prog": "TableValidator_pytest", @@ -144,23 +131,20 @@ def test_output_files_csv(test_init, tmp_path): assert _files_are_identical(file1= TEST_DATA_DIR / "pytest_out_all_match_value_check.csv", file2=f"{tmp_path}/test_out_all_match_value_check.csv")[0], "Files should be identical!" def test_output_files_schema_mismatch(): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "query_toy_json_schemamismatch.json", sample_id_key="sample_id") - test_object_lenient = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) - test_object_lenient.create_schema() + + test_object_lenient = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_schemamismatch.json", l_sample_id_key="sample_id") + test_object_lenient._create_schema() ## there should be a schema check failure here with pytest.warns(SchemaMismatchWarning): - test_object_lenient.confirm_schema() + test_object_lenient._confirm_schema() assert test_object_lenient.schema_errors["sample_4"]["field2"] in ["SchemaMismatch. Expected {, } Found ", "SchemaMismatch. Expected {, } Found "] def test_output_files_value_mismatch(tmp_path): - left_data = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") - right_data = File_IO(input_file= TEST_DATA_DIR / "query_toy_json_valmismatch.json", sample_id_key="sample_id") - test_object = FileCompare(left_data=left_data.data_reader, right_data=right_data.data_reader) - test_object.create_schema() - test_object.confirm_schema() + + test_object = FileCompare(left_file=TEST_DATA_DIR / "ref_toy_csv.csv", right_file=TEST_DATA_DIR / "query_toy_json_valmismatch.json", l_sample_id_key="sample_id") + ## there should be a value mismatch flagged here with pytest.warns(ValueMismatchWarning, match="Mismatch in value of field field2 for sample sample_4. Expected: 42 Found: fourtwo"): test_object.compare_values() From 043f285c4b80246a16d32773db3451f354c9f235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:27:39 +0100 Subject: [PATCH 4/9] add parent_parser for common params, update FileCompare invocation, reorganise output format logic --- validation_toolkit/cli.py | 94 +++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/validation_toolkit/cli.py b/validation_toolkit/cli.py index fc38ff8..0c9a357 100644 --- a/validation_toolkit/cli.py +++ b/validation_toolkit/cli.py @@ -10,11 +10,16 @@ def collect_args(): parser = argparse.ArgumentParser(description="A tool to compare tabular data against specified reference data.") + parser.add_argument("-v", "--version", action="version", version=f"TableValidator: {__version__}") - subparsers = parser.add_subparsers(title="subcommands", help="TableValidator sub-commands", dest="run_mode") + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument("-o", "--output_filename", type=str, required=False, help="File path to write output data to.") + parent_parser.add_argument("-f", "--write_format", type=str, required=False, default="json", help="File format to write output to. Valid formats are ['csv', 'tsv', 'json', 'yaml'] for mode 'convert_format'; ['csv', 'json'] for mode 'compare'.") + + subparsers = parser.add_subparsers(title="subcommands", dest="run_mode") - file_compare_mode = subparsers.add_parser("compare") + file_compare_mode = subparsers.add_parser("compare", parents=[parent_parser]) file_compare_mode.add_argument("-r", "--ref_table", type=str, required=True, help="Reference tabular dataset.") file_compare_mode.add_argument("-q", "--query_table", type=str, required=True, help="Subject tabular dataset to compare.") file_compare_mode.add_argument("-rs", "--ref_sampleid_key", type=str, required=True, help="The column/field name that contains sample IDs in the reference table.") @@ -23,16 +28,10 @@ def collect_args(): file_compare_mode.add_argument("-m", "--mode", type=str, required=False, default="quick_compare", help="How to execute comparison. [quick_compare, compare_subset]") file_compare_mode.add_argument("-l", "--field_list", type=str, required=False, help="Comma-separated list of column names/fields to compare, required and used ONLY if using mode 'compare_subset'.") file_compare_mode.add_argument("-c", "--custom_config", type=str, required=False, help="Custom config file containing col_name and value mappings in .yaml format.") - file_compare_mode.add_argument("-o", "--output_filename", type=str, required=True, help="Path to output file to be written.") - file_compare_mode.add_argument("-f", "--write_format", type=str, required=False, default="json", help="File format to write comparison output to ['json', 'csv'].") - format_convert_mode = subparsers.add_parser("convert_format") + format_convert_mode = subparsers.add_parser("convert_format", parents=[parent_parser]) format_convert_mode.add_argument("-i", "--input_filepath", type=str, required=False, help="File path to read data from for conversion to new format.") - ## code duplication, consider alternatives format_convert_mode.add_argument("-s", "--sampleid_key", type=str, required=True, help="The column/field name that contains sample IDs.") - format_convert_mode.add_argument("-o", "--output_filename", type=str, required=False, help="File name to write converted data to.") - format_convert_mode.add_argument("-d", "--output_dir", type=str, required=False, default=".", help="Directory path to write converted data to.") - format_convert_mode.add_argument("-f", "--write_format", type=str, required=False, help="File format to convert input data to.") args = parser.parse_args() return args @@ -50,36 +49,38 @@ def main(): args = collect_args() - ## confirm accepted mode - if args.mode not in ["quick_compare", "compare_subset"]: - raise ValueError("args.mode should be in [quick_compare, compare_subset]") - ## set up output file + output_file = os.path.abspath(args.output_filename) + + ## when run_mode is "compare", valid output formats are csv and json only + if args.run_mode == "compare": ## if file with given name exists, use timestamp to prevent overwriting - if args.write_format == "json": - if not os.path.exists(args.output_filename): - validator_output_file = args.output_filename - else: - print(f"Path: {args.output_filename} already exists. Using timestamp to prevent overwriting.") - validator_output_file = args.output_filename.strip(f".json") + "_" + ".".join(NOW.split(" ")[1].split(":")) + ".json" - - ## if output format is csv, 2 files are generated, so two filenames need to be checked - elif args.write_format == "csv": - base = args.output_filename.strip(f".csv") - files_to_check = [f"{base}_schema_check.csv", f"{base}_value_check.csv"] - files_exist = any(os.path.exists(file) for file in files_to_check) - if not files_exist: - validator_output_file = args.output_filename + if args.write_format == "json": + if not os.path.exists(output_file): + validator_output_file = output_file + else: + print(f"Path: {output_file} already exists. Using timestamp to prevent overwriting.") + validator_output_file = output_file.strip(f".json") + "_" + ".".join(NOW.split(" ")[1].split(":")) + ".json" + + ## if output format is csv, 2 files are generated, so two filenames need to be checked + elif args.write_format == "csv": + base = output_file.strip(f".csv") + files_to_check = [f"{base}_schema_check.csv", f"{base}_value_check.csv"] + files_exist = any(os.path.exists(file) for file in files_to_check) + if not files_exist: + validator_output_file = output_file + else: + print(f"Paths using basename: {base} already exist. Using timestamp to prevent overwriting.") + validator_output_file = f"{base}_" + ".".join(NOW.split(" ")[1].split(":")) + ".csv" + print(validator_output_file) + else: - print(f"Paths using basename: {base} already exist. Using timestamp to prevent overwriting.") - validator_output_file = f"{base}_" + ".".join(NOW.split(" ")[1].split(":")) + ".csv" - print(validator_output_file) + raise ValueError(f"\n===\nUnsupported output file format! Supported formats: csv, json. You provided: {args.write_format}\n===\n") - else: - raise ValueError(f"Unsupported output file format! Supported formats: csv, json. You provided: {args.write_format}") + ## confirm accepted mode + if args.mode not in ["quick_compare", "compare_subset"]: + raise ValueError("args.mode should be in [quick_compare, compare_subset]") - ## when run_mode is to compare, collect ref and query filepaths - if args.run_mode == "compare": r_file = os.path.abspath(args.ref_table) q_file = os.path.abspath(args.query_table) @@ -87,29 +88,14 @@ def main(): metadata["ref"] = r_file metadata["query"] = q_file - ## if only ref sample_id_key provided, assume it is the same in query - r_key = args.ref_sampleid_key - if r_key and not args.query_sampleid_key: - q_key = r_key - else: - q_key = args.query_sampleid_key - - print("Reading data from files...") ## if custom config given, use it to map colnames and values onto reference file ## colnames and values in ref should now match representations in the query if args.custom_config: print("Custom config found...") - r_table = File_IO(input_file=r_file, sample_id_key=r_key, config=args.custom_config) - q_table = File_IO(input_file=q_file, sample_id_key=q_key) ## initialise validator object - validator = FileCompare(left_data=r_table.data_reader, right_data=q_table.data_reader, strict=args.strict) - - print("Setting up and validating schema...") - ## create and confirm schema - validator.create_schema() - validator.confirm_schema() + validator = FileCompare(left_file=r_file, right_file=q_file, l_sample_id_key=args.ref_sampleid_key, r_sample_id_key=args.query_sampleid_key, config=args.custom_config, strict=args.strict) ## in quick_compare, nothing else needs to be done before comparing values if args.mode == "quick_compare": @@ -132,12 +118,16 @@ def main(): ## when run_mode is "convert_format" ## simply read data in, and write out to provided format + ## allowed output formats are csv, tsv, json, and yaml elif args.run_mode == "convert_format": + if args.write_format not in ["csv", "tsv", "json", "yaml"]: + raise ValueError(f"\n===\nUnsupported output file format! Supported formats: csv, tsv, json, yaml. You provided: {args.write_format}\n===\n") ## admin input_file = os.path.abspath(args.input_filepath) - output_dir = os.path.abspath(args.output_dir) - output_file = os.path.join(output_dir, args.output_filepath) + # output_dir = os.path.abspath(args.output_dir) + # output_file = os.path.abspath(args.outp) reader_object = File_IO(input_file=input_file, sample_id_key=args.sampleid_key) + print(f"Writing output to: {output_file} in {args.write_format} format") match args.write_format: case "csv": reader_object.write_to_csv(output_file=output_file) case "tsv": reader_object.write_to_tsv(output_file=output_file) From ca163c2b48d27374fb16d005a5a3d2a5a8ae88a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:31:18 +0100 Subject: [PATCH 5/9] cleanup --- validation_toolkit/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/validation_toolkit/cli.py b/validation_toolkit/cli.py index 0c9a357..3b15f8b 100644 --- a/validation_toolkit/cli.py +++ b/validation_toolkit/cli.py @@ -124,8 +124,6 @@ def main(): raise ValueError(f"\n===\nUnsupported output file format! Supported formats: csv, tsv, json, yaml. You provided: {args.write_format}\n===\n") ## admin input_file = os.path.abspath(args.input_filepath) - # output_dir = os.path.abspath(args.output_dir) - # output_file = os.path.abspath(args.outp) reader_object = File_IO(input_file=input_file, sample_id_key=args.sampleid_key) print(f"Writing output to: {output_file} in {args.write_format} format") match args.write_format: From 66f39b3bc2112c3b1f967a21a3001a295ce0b7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:36:16 +0100 Subject: [PATCH 6/9] fix ci --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44746e4..8f12d40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,7 @@ name: Run PyTest on Push on: - push: - branches: [ "*" ] + push jobs: build: From 4b9d08ef8bd61057a34438078d5ecbd8c3d55799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Thu, 26 Jun 2025 12:46:24 +0100 Subject: [PATCH 7/9] update README --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4ecdfe4..0da7121 100644 --- a/README.md +++ b/README.md @@ -128,13 +128,13 @@ Metadata lines are formatted as comments starting with two hashes; these are wri # List of Options - `-h`, `--help`: show this help message and exit -- `-r`, `--ref_table`: Reference tabular dataset. -- `-q`, `--query_table`: Subject tabular dataset to compare. -- `-rs`, `--ref_sampleid_key`: The column/field name that contains sample IDs in the reference table. -- `-qs`, `--query_sampleid_key`: The column/field name that contains sample IDs in the query table. +- `-r`, `--ref_table`: Reference tabular dataset. [REQUIRED: TRUE] +- `-q`, `--query_table`: Subject tabular dataset to compare. [REQUIRED: TRUE] +- `-rs`, `--ref_sampleid_key`: The column/field name that contains sample IDs in the reference table. [REQUIRED: TRUE] +- `-qs`, `--query_sampleid_key`: The column/field name that contains sample IDs in the query table. [OPTIONAL] [DEFAULT: same as provided value for `--ref_sampleid_key`] - `-e`, `--strict`: Whether to raise warnings as exceptions or not. **NB** Using this option will cause the program to quit upon encountering differences in the datasets and consequenctly, no output files will be generated. Ordinarily, this flag should never be used. -- `-m`, `--mode`: How to execute comparison. [quick_compare, compare_subset] +- `-m`, `--mode`: How to execute comparison. [VALID: quick_compare, compare_subset] [DEFAULT: quick_compare] - `-l`, `--field_list`: Comma-separated list of column names/fields to compare, required and used ONLY if using mode 'compare_subset'. - `-c`, `--custom_config`: Custom config file containing col_name and value mappings in .yaml format. -- `-o`, `--output_filename`: Path to output file to be written. -- `-f`, `--write_format`: File format to write comparison output to ['json', 'csv']. +- `-o`, `--output_filename`: Path to output file to be written. [REQUIRED: TRUE] +- `-f`, `--write_format`: File format to write comparison output to. [VALID: 'csv', 'json'] From 431717505346ff7e6c7275737fbf1fe775631860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Fri, 27 Jun 2025 12:54:54 +0100 Subject: [PATCH 8/9] change data_reader attr name to data_dict --- tests/table_validator_tests/test_file_io.py | 10 +++++----- validation_toolkit/TableValidator/file_compare.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/table_validator_tests/test_file_io.py b/tests/table_validator_tests/test_file_io.py index 1fb995a..5d523b1 100644 --- a/tests/table_validator_tests/test_file_io.py +++ b/tests/table_validator_tests/test_file_io.py @@ -10,28 +10,28 @@ def test_init(): my_fileio_object = File_IO(input_file= TEST_DATA_DIR / "ref_toy_csv.csv", sample_id_key="sample_id") assert my_fileio_object.input_file_type == "csv", "Wrong file type found" - assert len(my_fileio_object.data_reader.keys()) == 5, "Expected 5 sample_ids in the test data" + assert len(my_fileio_object.data_dict.keys()) == 5, "Expected 5 sample_ids in the test data" assert len(my_fileio_object.fields) == 4, f"Expected 4 columns/fields in data" assert sorted(my_fileio_object.fields) == sorted(["sample_id", "field1", "field2", "field3"]), "Did not find all expected fields" def test_read_from_tsv(): my_fileio_object = File_IO(input_file= TEST_DATA_DIR / "ref_toy_tsv.tsv", sample_id_key="sample_id") assert my_fileio_object.input_file_type == "tsv", "Wrong file type found" - assert len(my_fileio_object.data_reader.keys()) == 5, "Expected 5 sample_ids in the test data" + assert len(my_fileio_object.data_dict.keys()) == 5, "Expected 5 sample_ids in the test data" assert len(my_fileio_object.fields) == 4, f"Expected 4 columns/fields in data" assert sorted(my_fileio_object.fields) == sorted(["sample_id", "field1", "field2", "field3"]), "Did not find all expected fields" def test_read_from_json(): my_fileio_object = File_IO(input_file= TEST_DATA_DIR / "ref_toy_json.json", sample_id_key="sample_id") assert my_fileio_object.input_file_type == "json", "Wrong file type found" - assert len(my_fileio_object.data_reader.keys()) == 5, "Expected 5 sample_ids in the test data" + assert len(my_fileio_object.data_dict.keys()) == 5, "Expected 5 sample_ids in the test data" assert len(my_fileio_object.fields) == 4, f"Expected 4 columns/fields in data" assert sorted(my_fileio_object.fields) == sorted(["sample_id", "field1", "field2", "field3"]), "Did not find all expected fields" def test_read_from_yaml(): my_fileio_object = File_IO(input_file= TEST_DATA_DIR / "ref_toy_yaml.yaml", sample_id_key="sample_id") assert my_fileio_object.input_file_type == "yaml", "Wrong file type found" - assert len(my_fileio_object.data_reader.keys()) == 5, "Expected 5 sample_ids in the test data" + assert len(my_fileio_object.data_dict.keys()) == 5, "Expected 5 sample_ids in the test data" assert len(my_fileio_object.fields) == 4, f"Expected 4 columns/fields in data" assert sorted(my_fileio_object.fields) == sorted(["sample_id", "field1", "field2", "field3"]), "Did not find all expected fields" @@ -72,6 +72,6 @@ def test_data_mapping(): my_ref_fileio_object_no_config = File_IO(input_file= TEST_DATA_DIR / "data_map_test_left.csv", sample_id_key="col_left_1") my_query_fileio_object = File_IO(input_file= TEST_DATA_DIR / "data_map_test_right.csv", sample_id_key="col_right_1") - assert list(my_ref_fileio_object.data_reader.keys()) == list(my_query_fileio_object.data_reader.keys()), "Config mapping erroneously altered sample_ids" + assert list(my_ref_fileio_object.data_dict.keys()) == list(my_query_fileio_object.data_dict.keys()), "Config mapping erroneously altered sample_ids" assert sorted(my_ref_fileio_object.fields) == sorted(my_query_fileio_object.fields), "Config mapping on column names went wrong" assert not sorted(my_ref_fileio_object_no_config.fields) == sorted(my_query_fileio_object.fields), "Column names should not match when config is not used" diff --git a/validation_toolkit/TableValidator/file_compare.py b/validation_toolkit/TableValidator/file_compare.py index ffe86f7..912a9de 100644 --- a/validation_toolkit/TableValidator/file_compare.py +++ b/validation_toolkit/TableValidator/file_compare.py @@ -19,17 +19,17 @@ def __init__(self, left_file: str, right_file: str, l_sample_id_key: str, r_samp ## load reference data ## if config provided then this will be used to map column names - self.ref_table = File_IO(input_file=left_file, sample_id_key=l_sample_id_key, config=config).data_reader + self.ref_table = File_IO(input_file=left_file, sample_id_key=l_sample_id_key, config=config).data_dict ## load query data ## if unique key provided for sample_id column, use that ## else assume same sampl_id_key as reference data if r_sample_id_key: - self.query_table = File_IO(input_file=right_file, sample_id_key=r_sample_id_key).data_reader + self.query_table = File_IO(input_file=right_file, sample_id_key=r_sample_id_key).data_dict else: print("No sample_id_key provided for query table, assuming it is the same as sample_id_key for reference table.") try: - self.query_table = File_IO(input_file=right_file, sample_id_key=l_sample_id_key).data_reader + self.query_table = File_IO(input_file=right_file, sample_id_key=l_sample_id_key).data_dict except KeyError as ke: raise ValueError(f"\n==\nDid not find any column {l_sample_id_key} in query table. Please check that you have provided a config file and a valid query sample_id_key on the command line.\n==\n") From 6bb6d22a4646457d5c4ba63a130581d79370ff7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbd8=E2=80=9D?= <“bd8@sanger.ac.uk”> Date: Fri, 27 Jun 2025 12:55:26 +0100 Subject: [PATCH 9/9] change data_reader attr name to data_dict, use self.data_dict in fn map_data --- validation_toolkit/TableValidator/file_io.py | 40 ++++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/validation_toolkit/TableValidator/file_io.py b/validation_toolkit/TableValidator/file_io.py index 4747f23..06216a3 100644 --- a/validation_toolkit/TableValidator/file_io.py +++ b/validation_toolkit/TableValidator/file_io.py @@ -8,12 +8,12 @@ class File_IO(): """Read, write and intercovert between a range of file types. Currently supports csv, tsv, json, and yaml """ - def __init__(self, input_file: str, sample_id_key: str, config: str = None, data_reader: dict = None): + def __init__(self, input_file: str, sample_id_key: str, config: str = None, data_dict: dict = None): """Initialising a File_IO object Args: input_file (str/pathlike): File to read from. - data_reader (dict, optional): Dictionary formatted as {sample_id: {field1: val1, field2: val2...}}. Defaults to None. + data_dict (dict, optional): Dictionary formatted as {sample_id: {field1: val1, field2: val2...}}. Defaults to None. This is the output/internal structure of data in File_IO File_IO can alternatively start with such a dict sample_id_key (str, optional): The name of the field or column in the input file corresponding to sample IDs. @@ -34,13 +34,13 @@ def __init__(self, input_file: str, sample_id_key: str, config: str = None, data "yaml": self._read_serialized, } - ## if data_reader is provided, use that to initialise + ## if data_dict is provided, use that to initialise ## if input file is ALSO provided, it is ignored - if data_reader or data_reader and input_file: - self.data_reader = data_reader + if data_dict or data_dict and input_file: + self.data_dict = data_dict ## if ONLY input file provided, read it in - elif input_file and not data_reader: + elif input_file and not data_dict: self.full_path_input_file = os.path.abspath(input_file) ## get file type of input file from its extension @@ -48,20 +48,20 @@ def __init__(self, input_file: str, sample_id_key: str, config: str = None, data ## if csv or tsv, pass to tabular file reader function if self.input_file_type in ["csv", "tsv"]: - tmp_data_reader = read_functions[self.input_file_type](self.full_path_input_file, sample_id_key) - self.data_reader = self._recover_types_in_data(tmp_data_reader) + tmp_data_dict = read_functions[self.input_file_type](self.full_path_input_file, sample_id_key) + self.data_dict = self._recover_types_in_data(tmp_data_dict) ## if json or yaml, pass to serialized file reader function else: - self.data_reader = read_functions[self.input_file_type](self.full_path_input_file) + self.data_dict = read_functions[self.input_file_type](self.full_path_input_file) if config: - self.data_reader = self.map_data(data_reader_dict=self.data_reader, config=config) + self.data_dict = self.map_data(config=config) ## collect a list of fields/column names from the input self.fields = [] - example_key = list(self.data_reader.keys())[0] - self.fields.extend(list(self.data_reader[example_key].keys())) + example_key = list(self.data_dict.keys())[0] + self.fields.extend(list(self.data_dict[example_key].keys())) def _get_file_type(self, input_file: str): """Use file extension to infer file type @@ -256,9 +256,9 @@ def write_to_csv(self, output_file: str): output_file (str): Path to output file """ rows_to_write = [] - for sample_id in self.data_reader.keys(): + for sample_id in self.data_dict.keys(): row = {"sample_id": sample_id} - row.update(self.data_reader[sample_id]) + row.update(self.data_dict[sample_id]) rows_to_write.append(row) with open(output_file, "w") as out_handle: writer = csv.DictWriter(out_handle, fieldnames=self.fields, delimiter=",") @@ -273,9 +273,9 @@ def write_to_tsv(self, output_file: str): output_file (str): Path to output file """ rows_to_write = [] - for sample_id in self.data_reader.keys(): + for sample_id in self.data_dict.keys(): row = {"sample_id": sample_id} - row.update(self.data_reader[sample_id]) + row.update(self.data_dict[sample_id]) rows_to_write.append(row) with open(output_file, "w") as out_handle: writer = csv.DictWriter(out_handle, fieldnames=self.fields, delimiter="\t") @@ -290,7 +290,7 @@ def write_to_json(self, output_file: str): output_file (str): Path to output file """ with open(output_file, "w") as outhandle: - json.dump(self.data_reader, outhandle, indent=4) + json.dump(self.data_dict, outhandle, indent=4) def write_to_yaml(self, output_file: str): """Function to write data to yaml @@ -299,9 +299,9 @@ def write_to_yaml(self, output_file: str): output_file (str): Path to output file """ with open(output_file, "w") as outhandle: - yaml.dump(self.data_reader, outhandle, indent=4) + yaml.dump(self.data_dict, outhandle, indent=4) - def map_data(self, data_reader_dict, config): + def map_data(self, config): with open(config, "rt") as config_handle: config_dict = yaml.safe_load(config_handle) @@ -309,7 +309,7 @@ def map_data(self, data_reader_dict, config): val_mappings = {k: v["value_map"] for k, v in config_dict.items()} mapped = {} - for k, v in data_reader_dict.items(): + for k, v in self.data_dict.items(): mapped_vals = {} for val_k, val_v in v.items(): if val_mappings[val_k]: