diff --git a/isatools/convert/isatab2cedar.py b/isatools/convert/isatab2cedar.py index 066fd043f..233bcd8b2 100644 --- a/isatools/convert/isatab2cedar.py +++ b/isatools/convert/isatab2cedar.py @@ -8,10 +8,13 @@ import os from os import listdir from os.path import isdir, join +from pathlib import Path from uuid import uuid4 -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, FormatChecker from jsonschema.exceptions import ValidationError +from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools.io import isatab_parser @@ -38,11 +41,24 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): log.info("Converting ISA to CEDAR model for {}".format(work_dir)) schema_file = "investigation_template.json" with open(join(CEDAR_SCHEMA_PATH, schema_file)) as json_fp: - schema = json.load(json_fp) - if schema is None: + investigation_schema = json.load(json_fp) + if investigation_schema is None: raise IOError("Could not load schema from {}".format(join(CEDAR_SCHEMA_PATH, schema_file))) - resolver = RefResolver("file://{}".format(join(CEDAR_SCHEMA_PATH, schema_file)), schema) - validator = Draft4Validator(schema, resolver=resolver) + + resources = [] + schemas_dir = Path(CEDAR_SCHEMA_PATH) + investigation_schema_path = Path(join(CEDAR_SCHEMA_PATH, schema_file)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) isa_tab = isatab_parser.parse(work_dir) @@ -121,7 +137,7 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier): study_identifier = "" try: - validator.validate(cedar_json, schema) + validator.validate(cedar_json) except ValidationError as e: error_file_name = os.path.join(json_dir, "error.log") with open(error_file_name, "w") as errorfile: diff --git a/isatools/convert/isatab2json.py b/isatools/convert/isatab2json.py index 193029770..c770e60e1 100644 --- a/isatools/convert/isatab2json.py +++ b/isatools/convert/isatab2json.py @@ -8,9 +8,12 @@ import re from enum import Enum from os.path import join +from pathlib import Path from uuid import uuid4 -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft202012Validator, FormatChecker +from referencing import Registry +from referencing.jsonschema import DRAFT202012 from isatools import isatab from isatools.io.isatab_parser import parse @@ -139,10 +142,27 @@ def convert(self, work_dir): # validate json with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp: - schema = json.load(json_fp) - resolver = RefResolver("file://" + join(SCHEMAS_PATH, INVESTIGATION_SCHEMA), schema) - validator = Draft4Validator(schema, resolver=resolver) - validator.validate(isa_json, schema) + # investigation_schema = json.load(json_fp) + # schema = DRAFT4.create_resource(investigation_schema) + # registry = Registry.with_resource(investigation_schema['id'], schema) + # validator = Draft4Validator(investigation_schema, registry=registry) + # validator.validate(isa_json) + + resources = [] + schemas_dir = Path(SCHEMAS_PATH) + investigation_schema_path = Path(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT202012.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"} + validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + validator.validate(isa_json) log.info("Conversion finished") return isa_json diff --git a/isatools/isajson/validate.py b/isatools/isajson/validate.py index b875ddc07..4ca152f80 100644 --- a/isatools/isajson/validate.py +++ b/isatools/isajson/validate.py @@ -13,8 +13,11 @@ import os import re from io import StringIO +from pathlib import Path -from jsonschema import Draft4Validator, RefResolver, ValidationError +from jsonschema import Draft202012Validator, FormatChecker, ValidationError +from referencing import Registry +from referencing.jsonschema import DRAFT202012 from isatools.isajson.load import load @@ -547,10 +550,21 @@ def check_isa_schemas(isa_json, investigation_schema_path): """Used for rule 0003 and 4003""" try: with open(investigation_schema_path) as fp: - investigation_schema = json.load(fp) - resolver = RefResolver("file://" + investigation_schema_path, investigation_schema) - validator = Draft4Validator(investigation_schema, resolver=resolver) + resources = [] + investigation_schema_path = Path(investigation_schema_path) + schemas_dir = investigation_schema_path.parent + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT202012.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"} + validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(isa_json) + except ValidationError as ve: errors.append({"message": "Invalid JSON against ISA-JSON schemas", "supplemental": str(ve), "code": 3}) log.fatal("(F) The JSON does not validate against the provided ISA-JSON schemas!") diff --git a/isatools/net/ols.py b/isatools/net/ols.py index dc447c834..4f3afa3ed 100644 --- a/isatools/net/ols.py +++ b/isatools/net/ols.py @@ -39,6 +39,7 @@ def get_ols_ontologies(): file=file, ) ) + return ontology_sources @@ -47,6 +48,7 @@ def get_ols_ontology(ontology_name): ontologiesUri = OLS_API_BASE_URI + "/ontologies?size=" + str(OLS_PAGINATION_SIZE) log.debug(ontologiesUri) J = json.loads(urlopen(ontologiesUri).read().decode("utf-8")) + print("EMBEDDED: ", J["_embedded"]["ontologies"]) ontology_sources = [] for ontology_source_json in J["_embedded"]["ontologies"]: ontology_sources.append( @@ -57,7 +59,10 @@ def get_ols_ontology(ontology_name): file=ontology_source_json["_links"]["self"]["href"], ) ) + print("NAME: ", ontology_name) + print("SOURCES: ", [o.name for o in ontology_sources]) hits = [o for o in ontology_sources if o.name == ontology_name] + print("HITS: ", hits) if len(hits) == 1: return hits[0] return None diff --git a/isatools/net/storage_adapter.py b/isatools/net/storage_adapter.py index 19f9b4fb1..6e1d50c20 100644 --- a/isatools/net/storage_adapter.py +++ b/isatools/net/storage_adapter.py @@ -5,15 +5,18 @@ import json import logging import os -import pathlib from abc import ABCMeta, abstractmethod from io import BytesIO, StringIO +from os.path import join +from pathlib import Path from urllib.parse import urljoin from zipfile import ZipFile import requests -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, FormatChecker # RefResolver from lxml import etree +from referencing import Registry +from referencing.jsonschema import DRAFT4 log = logging.getLogger("isatools") @@ -58,8 +61,22 @@ def validate_json_against_schema(json_dict, schema_src): """ with open(schema_src) as schema_file: schema = json.load(schema_file) - resolver = RefResolver(pathlib.Path(os.path.abspath(schema_src)).as_uri(), schema) - validator = Draft4Validator(schema, resolver=resolver) + resources = [] + schemas_dir = Path(schema_src) + investigation_schema_path = Path(join(schema_src, schema_file)) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = investigation_schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + + # validator = Draft4Validator(schema) return validator.validate(json_dict, schema) diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json index 43a463809..91b97edf5 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Assay JSON Schema", "name": "ISA Assay JSON Schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json index 43689513c..be22d89da 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Comment schema - it corresponds to ISA Comment[] construct", "name" : "ISA Comment schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json index 5914d6cbc..ad430361f 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Data schema", "name" : "ISA Data schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json index b85d1bcf7..8c4fd04d3 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Factor schema", "name": "ISA Factor schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json index 05383d12a..cf646e6f7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Factor Value schema", "name": "ISA Factor Value schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json index 3b2e1a7f0..50d92a901 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA investigation schema", "name" : "ISA Investigation schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json index c7b566486..933dd108d 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA material attribute schema", "name" : "ISA Material Attribute schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json index 3f67a82f5..f3336196a 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Material Attribute schema", "name" : "ISA Material Attribute schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json index c00f7216d..de0b30169 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Material schema", "name" : "ISA Material schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json index 25c5cd045..7848e6ea7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA ontology reference schema", "name" : "ISA ontology reference schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json index 5e920666a..6663bb3dd 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA ontology source reference schema", "name" : "ISA ontology source reference schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json index 99fae9d17..309e2ea97 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Organization schema", "name" : "ISA Organization schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json index b81a3ed89..7a928a4d1 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Person schema", "name" : "ISA Person schema", @@ -12,7 +12,13 @@ "lastName" : { "type" : "string"}, "firstName" : { "type" : "string"}, "midInitials" : { "type" : "string" }, - "email" : { "type" : "string", "format" : "email"}, + "email" : { + "anyOf": [ + { "type" : "string", "format": "email" }, + { "type" : "string", "maxLength": 0 }, + { "type": "null" } + ] + }, "phone" : { "type": "string"}, "fax" : { "type" : "string" }, "address" : { "type" : "string" }, diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json index f9b63386c..a298ede16 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA process parameter value schema", "name" : "ISA Process Parameter Value schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json index 919ccda3d..476869120 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA process or protocol application schema, corresponds to 'Protocol REF' columns in the study and assay files", "name" : "ISA Process schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json index 74680a630..114accf77 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Protocol Parameter schema", "name" : "ISA Protocol Parameter schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json index ed625dba7..2775631b7 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Protocol schema", "name": "ISA Protocol schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json index c1c7fcdad..dd9495f24 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Publication schema", "name" : "ISA Publication schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json index b83f7b47e..571c1d82b 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Sample schema", "name" : "ISA Sample schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json index 3ba7f804f..b8f59d1f2 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title" : "ISA Source schema", "name" : "ISA Source schema", diff --git a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json index 884b8e779..93232552c 100644 --- a/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json +++ b/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json @@ -1,5 +1,5 @@ { - "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json", + "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ISA Study JSON schema", "name" : "ISA Study JSON schema", diff --git a/isatools/utils.py b/isatools/utils.py index 45da0e013..a4e0c28aa 100644 --- a/isatools/utils.py +++ b/isatools/utils.py @@ -14,7 +14,6 @@ from zipfile import ZipFile import pandas as pd -import yaml from isatools import isatab from isatools.model import Process diff --git a/pyproject.toml b/pyproject.toml index 3021f289c..b40b7f30d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "pandas>=2.2.3,<3", "openpyxl>=3.1.5", "lxml~=6.0.2", - "requests~=2.32.3", + "requests~=2.32.4", "iso8601~=2.1.0", "chardet~=5.2.0", "jinja2~=3.1.4", @@ -51,12 +51,13 @@ dependencies = [ "PyYAML~=6.0.2", "rdflib~=7.2.1", "python-dateutil~=2.9.0.post0", - "Flask>=3.1.0", + "Flask>=3.1.1", "flask_sqlalchemy>=3.0.2", "SQLAlchemy~=2.0.31", #1.4.54 "graphene~=3.4.3", "graphql-core~=3.2.6", "ruff>=0.14.1", + "pytest-timeout>=2.4.0", ] [project.optional-dependencies] @@ -85,7 +86,7 @@ dev = [ "coveralls>=4.0.1", "sure>=2.0.1", "ddt>=1.7.2", - "deepdiff>=8.4.2", + "deepdiff>=8.6.1", "behave>=1.2.6", "httpretty>=1.1.4", "import-linter>=2.5.2", diff --git a/tests/utils/test_isatools_utils.py b/tests/utils/test_isatools_utils.py index 69d4b18b7..1b9366458 100644 --- a/tests/utils/test_isatools_utils.py +++ b/tests/utils/test_isatools_utils.py @@ -139,6 +139,7 @@ def test_get_ontologies(self): self.assertIsInstance(ontology_sources, list) self.assertIsInstance(ontology_sources[0], OntologySource) + @unittest.skip("efo is not available from https://www.ebi.ac.uk/ols4/api/ontologies") def test_get_ontology(self): ontology_source = ols.get_ols_ontology("efo") self.assertIsInstance(ontology_source, OntologySource) @@ -148,6 +149,7 @@ def test_get_ontology(self): self.assertIsInstance(ontology_source.version, str) self.assertEqual(ontology_source.description, "Experimental Factor Ontology") + @unittest.skip("efo is not available from https://www.ebi.ac.uk/ols4/api/ontologies") def test_search_for_term(self): ontology_source = ols.get_ols_ontology("efo") ontology_annotations = ols.search_ols("cell type", ontology_source) diff --git a/tests/validators/test_validate_test_data.py b/tests/validators/test_validate_test_data.py index 1a330b46e..4033afe33 100644 --- a/tests/validators/test_validate_test_data.py +++ b/tests/validators/test_validate_test_data.py @@ -3,8 +3,12 @@ import os import pathlib import unittest +from os.path import join +from pathlib import Path -from jsonschema import Draft4Validator, RefResolver +from jsonschema import Draft4Validator, FormatChecker +from referencing import Registry +from referencing.jsonschema import DRAFT4 from isatools import isajson, isatab from isatools.tests import utils @@ -356,34 +360,68 @@ def test_validate_testdata_sampleassayplan_json(self): with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: sample_assay_plan_schema = json.load(fp) - res_path = pathlib.Path( - "file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json" - ).as_uri() - resolver = RefResolver(res_path, sample_assay_plan_schema) + resources = [] + res_path = ( + pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json") + .resolve() + .as_uri() + ) + schemas_dir = pathlib.Path("file://", self.v2_create_schemas_path) + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) - validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) - validator.validate(json.load(test_case_fp)) + registry = Registry().with_resources(resources) + schema_ref = {"$ref": res_path, "$schema": "https://json-schema.org/draft-04/schema"} + + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + validator.validate(json.load(test_case_fp)) def test_validate_testdata_sampleassayplan_qc_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "sampleassayplan_qc_test.json")) as test_case_fp: with open(os.path.join(self.v2_create_schemas_path, "sample_assay_plan_schema.json")) as fp: sample_assay_plan_schema = json.load(fp) - # resolver = RefResolver('file://{}'.format( - # os.path.join(self.v2_create_schemas_path, - # 'sample_assay_plan_schema.json')), - # sample_assay_plan_schema) - res_path = str(pathlib.Path("file://", self.v2_create_schemas_path, "sample_assay_plan_schema.json")) - resolver = RefResolver(res_path, sample_assay_plan_schema) - validator = Draft4Validator(sample_assay_plan_schema, resolver=resolver) + + resources = [] + schemas_dir = Path("file://", self.v2_create_schemas_path) + schema_path = Path(join(schemas_dir, "sample_assay_plan_schema.json")) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) validator.validate(json.load(test_case_fp)) def test_validate_testdata_treatment_sequence_json(self): with open(os.path.join(utils.JSON_DATA_DIR, "create", "treatment_sequence_test.json")) as test_case_fp: with open(os.path.join(self.v2_create_schemas_path, "treatment_sequence_schema.json")) as fp: treatment_sequence_schema = json.load(fp) - res_path = pathlib.Path("file://", self.v2_create_schemas_path, "treatment_sequence_schema.json").as_uri() - resolver = RefResolver(res_path, treatment_sequence_schema) - validator = Draft4Validator(treatment_sequence_schema, resolver=resolver) + #res_path = pathlib.Path("file://", self.v2_create_schemas_path, "treatment_sequence_schema.json").as_uri() + # resolver = RefResolver(res_path, treatment_sequence_schema) + # validator = Draft4Validator(treatment_sequence_schema, resolver=resolver) + resources = [] + schemas_dir = Path("file://", self.v2_create_schemas_path) + schema_path = Path(join(schemas_dir, "treatment_sequence_schema.json")) + + for p in sorted(schemas_dir.glob("*.json")): + contents = json.loads(p.read_text(encoding="utf-8")) + resource = DRAFT4.create_resource(contents) + resources.append((p.resolve().as_uri(), resource)) + + registry = Registry().with_resources(resources) + main_uri = schema_path.resolve().as_uri() + print(registry.contents(main_uri)) + schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"} + validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker()) + validator.validate(json.load(test_case_fp)) diff --git a/tests/validators/test_validators.py b/tests/validators/test_validators.py index eea316f91..1c73753ac 100644 --- a/tests/validators/test_validators.py +++ b/tests/validators/test_validators.py @@ -484,7 +484,6 @@ def test_info_reporting_mtbls1_isatab(self): report = isatab.validate( fp=test_case_fp, config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level ) - print(report) # self.assertIn( # {'supplemental': 'Found 4 study groups in s_MTBLS1.txt', # 'code': 5001, diff --git a/uv.lock b/uv.lock index d3db8632b..20397fbd1 100644 --- a/uv.lock +++ b/uv.lock @@ -1288,6 +1288,7 @@ dependencies = [ { name = "openpyxl" }, { name = "pandas" }, { name = "progressbar2" }, + { name = "pytest-timeout" }, { name = "python-dateutil" }, { name = "pyyaml" }, { name = "rdflib" }, @@ -1322,7 +1323,7 @@ requires-dist = [ { name = "biopython", specifier = ">=1.85,<1.86" }, { name = "bokeh", marker = "extra == 'notebook'", specifier = "~=3.4.2" }, { name = "chardet", specifier = "~=5.2.0" }, - { name = "flask", specifier = ">=3.1.0" }, + { name = "flask", specifier = ">=3.1.1" }, { name = "flask-sqlalchemy", specifier = ">=3.0.2" }, { name = "graphene", specifier = "~=3.4.3" }, { name = "graphql-core", specifier = "~=3.2.6" }, @@ -1338,10 +1339,11 @@ requires-dist = [ { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pandas", specifier = ">=2.2.3,<3" }, { name = "progressbar2", specifier = "~=4.5.0" }, + { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "python-dateutil", specifier = "~=2.9.0.post0" }, { name = "pyyaml", specifier = "~=6.0.2" }, { name = "rdflib", specifier = "~=7.2.1" }, - { name = "requests", specifier = "~=2.32.3" }, + { name = "requests", specifier = "~=2.32.4" }, { name = "ruff", specifier = ">=0.14.1" }, { name = "setuptools", specifier = ">=77.0.3,<81" }, { name = "sqlalchemy", specifier = "~=2.0.31" }, @@ -1353,7 +1355,7 @@ dev = [ { name = "behave", specifier = ">=1.2.6" }, { name = "coveralls", specifier = ">=4.0.1" }, { name = "ddt", specifier = ">=1.7.2" }, - { name = "deepdiff", specifier = ">=8.4.2" }, + { name = "deepdiff", specifier = ">=8.6.1" }, { name = "httpretty", specifier = ">=1.1.4" }, { name = "import-linter", specifier = ">=2.5.2" }, { name = "pre-commit", specifier = ">=4.0.1,<5" }, @@ -2301,6 +2303,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"