diff --git a/pyproject.toml b/pyproject.toml index 2db36637..9845c380 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,6 @@ pythonpath = [ [tool.coverage.run] relative_files = true + +[tool.black] +include = '(\.pyi?$)|(python\.j2$)' diff --git a/src/shacl2code/lang/templates/python.j2 b/src/shacl2code/lang/templates/python.j2 index 4139f093..92d54e33 100644 --- a/src/shacl2code/lang/templates/python.j2 +++ b/src/shacl2code/lang/templates/python.j2 @@ -8,6 +8,7 @@ from __future__ import annotations +import decimal import functools import hashlib import json @@ -43,7 +44,9 @@ def check_type(obj: Any, types: Union[Type[Any], Tuple[Type[Any], ...]]) -> None raise TypeError( f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj).__name__}" ) - raise TypeError(f"Value must be of type {types.__name__}. Got {type(obj).__name__}") + raise TypeError( + f"Value must be of type {types.__name__}. Got {type(obj).__name__}" + ) class Property(ABC): @@ -84,10 +87,14 @@ class Property(ABC): def walk(self, value, callback: Callable, path: List[str]) -> None: callback(value, path) - def iter_objects(self, value, recursive: bool, visited: Set[SHACLObject]) -> Iterable[Any]: + def iter_objects( + self, value, recursive: bool, visited: Set[SHACLObject] + ) -> Iterable[Any]: return [] - def link_prop(self, value, objectset, missing: Optional[Set[str]], visited: Set[SHACLObject]): + def link_prop( + self, value, objectset, missing: Optional[Set[str]], visited: Set[SHACLObject] + ): return value def to_string(self, value) -> str: @@ -95,15 +102,11 @@ class Property(ABC): @abstractmethod def encode(self, encoder, value, state) -> None: - raise NotImplementedError( - "Subclasses must implement encode method" - ) + raise NotImplementedError("Subclasses must implement encode method") @abstractmethod def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Any: - raise NotImplementedError( - "Subclasses must implement decode method" - ) + raise NotImplementedError("Subclasses must implement decode method") class StringProp(Property): @@ -119,7 +122,9 @@ class StringProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_string(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[str]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[str]: return decoder.read_string() @@ -127,7 +132,9 @@ class AnyURIProp(StringProp): def encode(self, encoder, value, state) -> None: encoder.write_iri(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[str]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[str]: return decoder.read_iri() @@ -146,22 +153,36 @@ class DateTimeProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_datetime(self.to_string(value)) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[datetime]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[datetime]: s = decoder.read_datetime() if s is None: return None + if isinstance(s, datetime): + return self._normalize(s) v = self.from_string(s) return self._normalize(v) def _normalize(self, value: datetime) -> datetime: if value.utcoffset() is None: value = value.astimezone() + + # Remove seconds from timezone offset offset = value.utcoffset() if offset is not None: - seconds = offset % timedelta(minutes=-1 if offset.total_seconds() < 0 else 1) + seconds = offset % timedelta( + minutes=-1 if offset.total_seconds() < 0 else 1 + ) if seconds: offset = offset - seconds value = value.replace(tzinfo=timezone(offset)) + + # Convert 00:00 timezone offset to UTC + offset = value.utcoffset() + if offset is not None and offset.seconds == 0: + value = value.astimezone(timezone.utc) + value = value.replace(microsecond=0) return value @@ -202,7 +223,9 @@ class IntegerProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_integer(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[int]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[int]: return decoder.read_integer() @@ -229,7 +252,9 @@ class BooleanProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_bool(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[bool]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[bool]: return decoder.read_bool() @@ -242,7 +267,9 @@ class FloatProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_float(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[float]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[float]: return decoder.read_float() @@ -322,11 +349,12 @@ class ObjectProp(IRIProp): return value.encode(encoder, state) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] =None): - iri = decoder.read_iri() - if iri is None: + def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None): + if decoder.is_object(): return self.cls.decode(decoder, objectset=objectset) + iri = decoder.read_iri() + iri = self.expand(iri) or iri if objectset is None: @@ -398,7 +426,9 @@ class ListProxy(object): self.__prop.validate(value) self.__data[key] = self.__prop.set(value) else: - raise TypeError(f"ListProxy indices must be integers or slices. Got {type(key).__name__}") + raise TypeError( + f"ListProxy indices must be integers or slices. Got {type(key).__name__}" + ) def __delitem__(self, key) -> None: del self.__data[key] @@ -489,7 +519,9 @@ class ListProp(Property): with list_s.write_list_item() as item_s: self.prop.encode(item_s, v, state) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> ListProxy: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> ListProxy: data = [] for val_d in decoder.read_list(): v = self.prop.decode(val_d, objectset=objectset) @@ -546,7 +578,9 @@ def is_blank_node(s: Any) -> bool: return True -def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: bool = False): +def register( + type_iri: str, *, compact_type: Optional[str] = None, abstract: bool = False +): def add_class(key: str, c: Type[SHACLObject]) -> None: assert ( key not in SHACLObject.CLASSES @@ -560,9 +594,7 @@ def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: boo """ global NAMED_INDIVIDUALS - assert issubclass( - c, SHACLObject - ), f"{c} is not derived from SHACLObject" + assert issubclass(c, SHACLObject), f"{c} is not derived from SHACLObject" c._OBJ_TYPE = type_iri c.IS_ABSTRACT = abstract @@ -585,6 +617,7 @@ def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: boo register_lock = threading.Lock() NAMED_INDIVIDUALS: Set[str] = set() + @functools.total_ordering class SHACLObject(object): CLASSES: Dict[str, Type] = {} @@ -615,7 +648,9 @@ class SHACLObject(object): ) if self.__class__.IS_DEPRECATED: - warnings.warn(f"{self.__class__.__name__} is deprecated", DeprecationWarning) + warnings.warn( + f"{self.__class__.__name__} is deprecated", DeprecationWarning + ) with register_lock: cls = self.__class__ @@ -718,7 +753,9 @@ class SHACLObject(object): f"'{name}' is not a valid property of {self.__class__.__name__}" ) - def __get_prop(self, iri: str) -> Tuple[Property, Optional[int], Optional[int], str, Optional[str]]: + def __get_prop( + self, iri: str + ) -> Tuple[Property, Optional[int], Optional[int], str, Optional[str]]: if iri not in self._OBJ_PROPERTIES: raise KeyError( f"'{iri}' is not a valid property of {self.__class__.__name__}" @@ -726,7 +763,11 @@ class SHACLObject(object): return self._OBJ_PROPERTIES[iri] - def __iter_props(self) -> Iterator[Tuple[str, Property, Optional[int], Optional[int], str, Optional[str]]]: + def __iter_props( + self, + ) -> Iterator[ + Tuple[str, Property, Optional[int], Optional[int], str, Optional[str]] + ]: for iri, v in self._OBJ_PROPERTIES.items(): yield iri, *v @@ -738,23 +779,25 @@ class SHACLObject(object): if self.NODE_KIND == NodeKind.BlankNode: if not is_blank_node(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{iri}' cannot be set to '{value}' and must start with '_:'" + f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{iri}' cannot be set to {value!r} and must start with '_:'" ) elif self.NODE_KIND == NodeKind.IRI: if not is_IRI(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{iri}' cannot be set to '{value}'" + f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{iri}' cannot be set to {value!r}" ) else: if not is_blank_node(value) and not is_IRI(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{iri}' '{value}'. Must be a blank node or IRI" + f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{iri}' {value!r}. Must be a blank node or IRI" ) prop, _, _, pyname, _ = self.__get_prop(iri) prop.validate(value) if iri in self._OBJ_DEPRECATED: - warnings.warn(f"{self.__class__.__name__}.{pyname} is deprecated", DeprecationWarning) + warnings.warn( + f"{self.__class__.__name__}.{pyname} is deprecated", DeprecationWarning + ) self.__dict__["_obj_data"][iri] = prop.set(value) def __delitem__(self, iri: str) -> None: @@ -860,18 +903,9 @@ class SHACLObject(object): raise TypeError("Unable to determine type for object") obj = cls._make_object(typ) - _id = None - for key in (obj.ID_ALIAS, obj._OBJ_IRIS["_id"]): - with obj_d.read_property(key) as prop_d: - if prop_d is None: - continue - - _id = prop_d.read_iri() - if _id is None: - raise TypeError(f"Object key '{key}' is the wrong type") - - obj._id = _id - break + _id = obj_d.read_object_id(obj.ID_ALIAS) + if _id is not None: + obj._id = _id if obj.NODE_KIND == NodeKind.IRI and not obj._id: raise ValueError("Object is missing required IRI") @@ -881,11 +915,10 @@ class SHACLObject(object): v = objectset.find_by_id(obj._id) if v is not None: return v + objectset.add_index(obj) obj._decode_properties(obj_d, objectset=objectset) - if objectset is not None: - objectset.add_index(obj) return obj def _decode_properties(self, decoder, objectset: Optional[SHACLObjectSet] = None): @@ -913,7 +946,9 @@ class SHACLObject(object): return False - def link_helper(self, objectset: Optional[SHACLObjectSet], missing, visited) -> None: + def link_helper( + self, objectset: Optional[SHACLObjectSet], missing, visited + ) -> None: if self in visited: return @@ -1008,25 +1043,6 @@ class SHACLExtensibleObject(SHACLObject): self.__dict__["_obj_data"][key] = decode_value(prop_d) def _encode_properties(self, encoder, state): - def encode_value(encoder, v): - if isinstance(v, bool): - encoder.write_bool(v) - elif isinstance(v, str): - encoder.write_string(v) - elif isinstance(v, int): - encoder.write_integer(v) - elif isinstance(v, float): - encoder.write_float(v) - elif isinstance(v, list): - with encoder.write_list() as list_s: - for i in v: - with list_s.write_list_item() as item_s: - encode_value(item_s, i) - else: - raise TypeError( - f"Unsupported serialized type {type(v)} with value '{v}'" - ) - super()._encode_properties(encoder, state) if self.CLOSED: return @@ -1036,7 +1052,25 @@ class SHACLExtensibleObject(SHACLObject): continue with encoder.write_property(iri) as prop_s: - encode_value(prop_s, value) + if isinstance(value, list): + v = value + else: + v = [value] + with prop_s.write_list() as list_s: + for i in v: + with list_s.write_list_item() as item_s: + if isinstance(i, bool): + item_s.write_bool(i) + elif isinstance(i, str): + item_s.write_string(i) + elif isinstance(i, int): + item_s.write_integer(i) + elif isinstance(i, float): + item_s.write_float(i) + else: + raise TypeError( + f"Unsupported serialized type {type(i)} with value {i!r}" + ) def __setitem__(self, iri: str, value) -> None: try: @@ -1214,7 +1248,9 @@ class SHACLObjectSet(object): return self.missing_ids - def find_by_id(self, _id: str, default: Optional[SHACLObject] = None) -> Optional[SHACLObject]: + def find_by_id( + self, _id: str, default: Optional[SHACLObject] = None + ) -> Optional[SHACLObject]: """ Find object by ID @@ -1238,7 +1274,9 @@ class SHACLObjectSet(object): for child in o.iter_objects(recursive=True, visited=visited): yield child - def foreach_type(self, typ: Union[str, SHACLObject], *, match_subclass: bool = True) -> Iterable[SHACLObject]: + def foreach_type( + self, typ: Union[str, SHACLObject], *, match_subclass: bool = True + ) -> Iterable[SHACLObject]: """ Iterate over each object of a specified type (or subclass there of) @@ -1273,6 +1311,49 @@ class SHACLObjectSet(object): return SHACLObjectSet(new_objects, link=True) + def inline_blank_nodes(self) -> None: + """ + Removes (inlines) blank node objects from the root object set if they + are referenced in only one other location besides the root. + + Deserializers that do not preserve the tree-like structure of the + objects (e.g. RDF) should call this to ensure that blank nodes are + inline correctly + """ + ref_counts: Dict[SHACLObject, int] = {} + + def walk_callback(value: SHACLObject, path: List[str]) -> bool: + nonlocal ref_counts + + if not isinstance(value, SHACLObject): + return True + + ref_counts.setdefault(value, 0) + ref_counts[value] += 1 + if ref_counts[value] > 1: + return False + + return True + + for o in self.objects: + # Note that every object in the root object set gets at least one + # reference + o.walk(walk_callback) + + new_objects = set() + for o in self.objects: + if is_IRI(o._id): + new_objects.add(o) + # If the object is a blank node and is only referenced by this + # root list and one other location, remove it from the root list + # + # A count of 1 means the object is only referenced by the root, and + # therefore must be kept + elif ref_counts[o] != 2: + new_objects.add(o) + + self.objects = new_objects + def encode(self, encoder, force_list: bool = False, *, key=None) -> None: """ Serialize a list of objects to a serialization encoder @@ -1290,7 +1371,7 @@ class SHACLObjectSet(object): return True # Remove blank node ID for re-assignment - if value._id and value._id.startswith("_:"): + if is_blank_node(value._id): del value._id if value._id: @@ -1348,7 +1429,7 @@ class SHACLObjectSet(object): self.create_index() for obj_d in decoder.read_list(): - o = SHACLObject.decode(obj_d, objectset=self) + o = SHACLExtensibleObject.decode(obj_d, objectset=self) self.objects.add(o) self._link() @@ -1391,9 +1472,7 @@ class Decoder(ABC): Consumes the next item of any type """ - raise NotImplementedError( - "Subclasses must implement read_value method" - ) + raise NotImplementedError("Subclasses must implement read_value method") @abstractmethod def read_string(self) -> Optional[str]: @@ -1403,9 +1482,7 @@ class Decoder(ABC): Returns the string value of the next item, or `None` if the next item is not a string """ - raise NotImplementedError( - "Subclasses must implement read_string method" - ) + raise NotImplementedError("Subclasses must implement read_string method") @abstractmethod def read_datetime(self) -> Optional[str]: @@ -1419,9 +1496,7 @@ class Decoder(ABC): implementation can just check if the next item is a string without worrying about the format """ - raise NotImplementedError( - "Subclasses must implement read_datetime method" - ) + raise NotImplementedError("Subclasses must implement read_datetime method") @abstractmethod def read_integer(self) -> Optional[int]: @@ -1431,9 +1506,7 @@ class Decoder(ABC): Returns the integer value of the next item, or `None` if the next item is not an integer """ - raise NotImplementedError( - "Subclasses must implement read_integer method" - ) + raise NotImplementedError("Subclasses must implement read_integer method") @abstractmethod def read_iri(self) -> Optional[str]: @@ -1446,9 +1519,7 @@ class Decoder(ABC): The returned string should be either a fully-qualified IRI, or a blank node ID """ - raise NotImplementedError( - "Subclasses must implement read_iri method" - ) + raise NotImplementedError("Subclasses must implement read_iri method") @abstractmethod def read_enum(self, e) -> Optional[str]: @@ -1462,9 +1533,7 @@ class Decoder(ABC): actually a member of the specified Enum, so the `Decoder` does not need to check that, but can if it wishes """ - raise NotImplementedError( - "Subclasses must implement read_enum method" - ) + raise NotImplementedError("Subclasses must implement read_enum method") @abstractmethod def read_bool(self) -> Optional[bool]: @@ -1474,9 +1543,7 @@ class Decoder(ABC): Returns the boolean value of the next item, or `None` if the next item is not a boolean """ - raise NotImplementedError( - "Subclasses must implement read_bool method" - ) + raise NotImplementedError("Subclasses must implement read_bool method") @abstractmethod def read_float(self) -> Optional[float]: @@ -1486,9 +1553,7 @@ class Decoder(ABC): Returns the float value of the next item, or `None` if the next item is not a float """ - raise NotImplementedError( - "Subclasses must implement read_float method" - ) + raise NotImplementedError("Subclasses must implement read_float method") @abstractmethod def read_list(self) -> Iterator["Decoder"]: @@ -1499,9 +1564,7 @@ class Decoder(ABC): generated `Decoder` can be used to read the corresponding item from the list """ - raise NotImplementedError( - "Subclasses must implement read_list method" - ) + raise NotImplementedError("Subclasses must implement read_list method") @abstractmethod def is_list(self) -> bool: @@ -1510,9 +1573,7 @@ class Decoder(ABC): Returns True if the next item is a list, or False if it is a scalar """ - raise NotImplementedError( - "Subclasses must implement is_list method" - ) + raise NotImplementedError("Subclasses must implement is_list method") @abstractmethod def read_object(self) -> Tuple[Any, "Decoder"]: @@ -1526,9 +1587,7 @@ class Decoder(ABC): Properties will be read out of the object using `read_property` and `read_object_id` """ - raise NotImplementedError( - "Subclasses must implement read_object method" - ) + raise NotImplementedError("Subclasses must implement read_object method") @abstractmethod @contextmanager @@ -1540,9 +1599,16 @@ class Decoder(ABC): value of the property with the given key in current object, or `None` if the property does not exist in the current object. """ - raise NotImplementedError( - "Subclasses must implement read_property method" - ) + raise NotImplementedError("Subclasses must implement read_property method") + + @abstractmethod + def is_object(self) -> bool: + """ + Checks if the item is an object + + Returns True if the item is an object, or False if is not + """ + raise NotImplementedError("Subclasses must implement is_object method") @abstractmethod def object_keys(self) -> Iterator[str]: @@ -1551,9 +1617,7 @@ class Decoder(ABC): Iterates over all the serialized keys for the current object """ - raise NotImplementedError( - "Subclasses must implement object_keys method" - ) + raise NotImplementedError("Subclasses must implement object_keys method") @abstractmethod def read_object_id(self, alias=None) -> Optional[Any]: @@ -1568,9 +1632,7 @@ class Decoder(ABC): If `alias` is provided, is is a hint as to another name by which the ID might be found, if the `Decoder` supports aliases for an ID """ - raise NotImplementedError( - "Subclasses must implement read_object_id method" - ) + raise NotImplementedError("Subclasses must implement read_object_id method") class JSONLDDecoder(Decoder): @@ -1643,6 +1705,9 @@ class JSONLDDecoder(Decoder): else: yield None + def is_object(self) -> bool: + return isinstance(self.data, dict) + def object_keys(self) -> Iterator[str]: for key in self.data.keys(): if key in ("@type", "{{ context.compact_iri('@type') }}"): @@ -1684,9 +1749,7 @@ class Encoder(ABC): Encodes the value as a string in the output """ - raise NotImplementedError( - "Subclasses must implement write_string method" - ) + raise NotImplementedError("Subclasses must implement write_string method") @abstractmethod def write_datetime(self, v) -> None: @@ -1697,9 +1760,7 @@ class Encoder(ABC): Note: The provided string is already correctly encoded as an ISO datetime """ - raise NotImplementedError( - "Subclasses must implement write_datetime method" - ) + raise NotImplementedError("Subclasses must implement write_datetime method") @abstractmethod def write_integer(self, v) -> None: @@ -1708,9 +1769,7 @@ class Encoder(ABC): Encodes the value as an integer in the output """ - raise NotImplementedError( - "Subclasses must implement write_integer method" - ) + raise NotImplementedError("Subclasses must implement write_integer method") @abstractmethod def write_iri(self, v, compact=None) -> None: @@ -1722,9 +1781,7 @@ class Encoder(ABC): the serialization supports compacted IRIs, it should be preferred to the full IRI """ - raise NotImplementedError( - "Subclasses must implement write_iri method" - ) + raise NotImplementedError("Subclasses must implement write_iri method") @abstractmethod def write_enum(self, v, e, compact=None) -> None: @@ -1735,9 +1792,7 @@ class Encoder(ABC): qualified IRI. If `compact` is provided and the serialization supports compacted IRIs, it should be preferred to the full IRI. """ - raise NotImplementedError( - "Subclasses must implement write_enum method" - ) + raise NotImplementedError("Subclasses must implement write_enum method") @abstractmethod def write_bool(self, v) -> None: @@ -1746,9 +1801,7 @@ class Encoder(ABC): Encodes the value as a boolean in the output """ - raise NotImplementedError( - "Subclasses must implement write_bool method" - ) + raise NotImplementedError("Subclasses must implement write_bool method") @abstractmethod def write_float(self, v) -> None: @@ -1757,13 +1810,11 @@ class Encoder(ABC): Encodes the value as a floating point number in the output """ - raise NotImplementedError( - "Subclasses must implement write_float method" - ) + raise NotImplementedError("Subclasses must implement write_float method") @abstractmethod @contextmanager - def write_object(self, o, _id, needs_id: bool): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): """ Write object @@ -1781,9 +1832,7 @@ class Encoder(ABC): Properties will be written the object using `write_property` """ - raise NotImplementedError( - "Subclasses must implement write_object method" - ) + raise NotImplementedError("Subclasses must implement write_object method") @abstractmethod @contextmanager @@ -1798,9 +1847,7 @@ class Encoder(ABC): the serialization supports compacted IRIs, it should be preferred to the full IRI. """ - raise NotImplementedError( - "Subclasses must implement write_property method" - ) + raise NotImplementedError("Subclasses must implement write_property method") @abstractmethod @contextmanager @@ -1813,9 +1860,7 @@ class Encoder(ABC): Each item of the list will be added using `write_list_item` """ - raise NotImplementedError( - "Subclasses must implement write_list method" - ) + raise NotImplementedError("Subclasses must implement write_list method") @abstractmethod @contextmanager @@ -1826,9 +1871,7 @@ class Encoder(ABC): A context manager that yields an `Encoder` that can be used to encode the value for a list item """ - raise NotImplementedError( - "Subclasses must implement write_list_item method" - ) + raise NotImplementedError("Subclasses must implement write_list_item method") class JSONLDEncoder(Encoder): @@ -1864,7 +1907,7 @@ class JSONLDEncoder(Encoder): self.data[compact or iri] = s.data # type: ignore # within write_object() context, self.data is always dict or None @contextmanager - def write_object(self, o, _id, needs_id): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): self.data = { "{{ context.compact_iri('@type') }}": o.COMPACT_TYPE or o.TYPE, } @@ -1898,7 +1941,7 @@ class JSONLDSerializer(object): ): h = JSONLDEncoder() objectset.encode(h, force_at_graph) - data: Dict[str, Union[str, List[str]]] = {} + data: Dict[str, Any] = {} if len(CONTEXT_URLS) == 1: data["@context"] = CONTEXT_URLS[0] elif CONTEXT_URLS: @@ -1991,7 +2034,7 @@ class JSONLDInlineEncoder(Encoder): self.comma = True @contextmanager - def write_object(self, o, _id, needs_id): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): self._write_comma() self.write("{") @@ -2059,6 +2102,249 @@ class JSONLDInlineSerializer(object): return sha1.hexdigest() +try: + import rdflib + import rdflib.term + from rdflib.namespace import RDF + + class RDFDecoder(Decoder): + def __init__( + self, + graph: rdflib.Graph, + subject: Optional[rdflib.term.Node] = None, + predicate: Optional[rdflib.term.Node] = None, + value: Optional[rdflib.term.Node] = None, + ): + self.graph = graph + self.subject = subject + self.predicate = predicate + self.value = value + + def __read_node(self): + if self.value is not None: + return self.value + if self.predicate is None: + return None + return self.graph.value(self.subject, self.predicate) + + def read_value(self) -> Optional[Any]: + v = self.__read_node() + if isinstance(v, rdflib.term.Literal): + return v.toPython() + return None + + def read_string(self) -> Optional[str]: + v = self.read_value() + if isinstance(v, str): + return v + return None + + def read_datetime(self) -> Optional[str]: + return self.read_value() + + def read_integer(self) -> Optional[int]: + v = self.read_value() + if isinstance(v, (int, decimal.Decimal)): + return int(v) + return None + + def read_bool(self) -> Optional[bool]: + v = self.read_value() + if isinstance(v, bool): + return v + return None + + def read_float(self) -> Optional[float]: + v = self.read_value() + if isinstance(v, (int, float, str, decimal.Decimal)): + return float(v) + return None + + def read_iri(self) -> Optional[str]: + v = self.__read_node() + if isinstance(v, rdflib.term.URIRef): + return v.toPython() + elif isinstance(v, rdflib.term.Literal): + v = v.toPython() + if isinstance(v, str): + return v + elif isinstance(v, rdflib.term.BNode): + return v.n3() + return None + + def read_enum(self, e) -> Optional[str]: + v = self.__read_node() + if isinstance(v, rdflib.term.URIRef): + return v.toPython() + return None + + def read_list(self) -> Iterator["RDFDecoder"]: + if not self.subject: + blank_nodes = set() + for s in self.graph.subjects(unique=True): + # type: ignore # RDF.type is dynamic + if (s, RDF.type, None) not in self.graph: + continue + + if isinstance(s, rdflib.term.BNode): + blank_nodes.add(s) + continue + yield self.__class__(self.graph, s) + + for s in blank_nodes: + yield self.__class__(self.graph, s) + else: + for o in self.graph.objects(self.subject, self.predicate): + # type: ignore # RDF.type is dynamic + if (o, RDF.type, None) in self.graph: + yield self.__class__(self.graph, o) + else: + yield self.__class__( + self.graph, + self.subject, + self.predicate, + o, + ) + + def is_list(self) -> bool: + if not self.subject: + return True + if self.value is not None: + return False + return len(list(self.graph.objects(self.subject, self.predicate))) > 1 + + @contextmanager + def read_property(self, key) -> Iterator[Optional["RDFDecoder"]]: + if key == "@id": + yield self.__class__(self.graph, value=self.subject) + else: + yield self.__class__(self.graph, self.subject, rdflib.term.URIRef(key)) + + def is_object(self) -> bool: + n = self.__read_node() or self.subject + # type: ignore # RDF.type is dynamic + return (n, RDF.type, None) in self.graph + + def object_keys(self) -> Iterator[str]: + for p in self.graph.predicates(self.subject, unique=True): + # type: ignore # RDF.type is dynamic + if p == RDF.type: + continue + + if not isinstance(p, rdflib.term.IdentifiedNode): + raise TypeError(f"Predicate is of unknown type {type(p)}") + + yield p.toPython() + + def read_object(self) -> Tuple[Any, "RDFDecoder"]: + s = self.__read_node() + if s is None: + s = self.subject + + # type: ignore # RDF.type is dynamic + typ = self.graph.value(s, RDF.type) + if typ is None: + return None, self + + if not isinstance(typ, rdflib.term.IdentifiedNode): + raise TypeError(f"Type value is of unknown type {type(typ)}") + + return typ.toPython(), self.__class__(self.graph, s) + + def read_object_id(self, alias=None) -> Optional[Any]: + if isinstance(self.subject, rdflib.term.BNode): + return self.subject.n3() + if not isinstance(self.subject, rdflib.term.IdentifiedNode): + raise TypeError(f"Subject is of unknown type {type(self.subject)}") + return self.subject.toPython() + + class RDFDeserializer(object): + def read(self, graph: rdflib.Graph, objset: SHACLObjectSet) -> None: + d = RDFDecoder(graph) + objset.decode(d) + objset.inline_blank_nodes() + + class RDFEncoder(Encoder): + def __init__( + self, + graph: rdflib.Graph, + subject: Optional[rdflib.term.Node] = None, + predicate: Optional[rdflib.term.Node] = None, + ): + self.graph = graph + self.subject = subject + self.predicate = predicate + + def __add_literal(self, v): + if self.subject is None or self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, rdflib.Literal(v))) + + def __add_uriref(self, v): + if self.subject is None or self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, rdflib.URIRef(v))) + + def write_string(self, v): + self.__add_literal(v) + + def write_datetime(self, v): + self.__add_literal(v) + + def write_integer(self, v): + self.__add_literal(v) + + def write_iri(self, v, compact=None): + self.__add_uriref(v) + + def write_enum(self, v, e, compact=None): + self.__add_uriref(v) + + def write_bool(self, v): + self.__add_literal(v) + + def write_float(self, v): + self.__add_literal(v) + + @contextmanager + def write_property(self, iri: str, compact: Optional[str] = None): + yield self.__class__(self.graph, self.subject, rdflib.URIRef(iri)) + + @contextmanager + def write_object(self, o, _id, needs_id: bool): + obj: rdflib.term.Node + if _id.startswith("_:"): + obj = rdflib.BNode(_id[2:]) + else: + obj = rdflib.URIRef(_id) + + if self.subject is not None: + if self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, obj)) + self.graph.add((obj, RDF.type, rdflib.URIRef(o.TYPE))) # type: ignore # RDF.type is dynamic + yield self.__class__(self.graph, obj) + + @contextmanager + def write_list(self): + yield self + + @contextmanager + def write_list_item(self): + yield self + + class RDFSerializer(object): + def write(self, objset: SHACLObjectSet, g: rdflib.Graph): + """ + Write a SHACLObjectSet to an RDF graph + """ + e = RDFEncoder(g) + objset.encode(e) + +except ImportError: + pass + + def print_tree(objects, all_fields: bool = False) -> None: """ Print object tree diff --git a/tests/data/roundtrip.json b/tests/data/roundtrip.json index 33bc8da7..32c735d5 100644 --- a/tests/data/roundtrip.json +++ b/tests/data/roundtrip.json @@ -6,7 +6,6 @@ "link-class-link-prop": "http://serialize.example.com/link-derived-target", "link-class-link-prop-no-class": "http://serialize.example.com/link-derived-target", "link-class-link-list-prop": [ - "http://serialize.example.com/link-derived-target", "http://serialize.example.com/link-derived-target" ] }, @@ -59,8 +58,8 @@ "test-class/class-prop": "http://serialize.example.com/test-derived", "test-class/class-prop-no-class": "http://serialize.example.com/test-derived", "test-class/class-list-prop": [ - "http://serialize.example.com/test-derived", - "http://serialize.example.com/test" + "http://serialize.example.com/test", + "http://serialize.example.com/test-derived" ], "test-class/enum-prop": "foo", "test-class/enum-prop-no-class": "bar", @@ -90,10 +89,12 @@ "@type": "uses-extensible-abstract-class", "@id": "http://serialize.example.com/test-uses-extensible-abstract", "uses-extensible-abstract-class/prop": { - "@type": " http://serialize.example.com/custom-extensible", - "http://custom-prop": [ - "abc" - ] + "@type": "http://serialize.example.com/custom-extensible", + "http://custom-list-prop": [ + "abc", + "def" + ], + "http://custom-scalar-prop": ["abc"] } } ] diff --git a/tests/test_python.py b/tests/test_python.py index cbbe6e32..96f3f905 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -32,23 +32,36 @@ TEST_TZ = timezone(timedelta(hours=-2), name="TST") +def shacl2code_generate(args, outfile): + return subprocess.run( + [ + "shacl2code", + "generate", + ] + + args + + [ + "python", + "--output", + outfile, + ], + check=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + + @pytest.fixture(scope="module") def test_context(tmp_path_factory, model_server): tmp_directory = tmp_path_factory.mktemp("pythontestcontext") outfile = tmp_directory / "model.py" - subprocess.run( + shacl2code_generate( [ - "shacl2code", - "generate", "--input", TEST_MODEL, "--context", model_server + "/test-context.json", - "python", - "--output", - outfile, ], - check=True, + outfile, ) outfile.chmod(0o755) yield (tmp_directory, outfile) @@ -92,19 +105,7 @@ def test_output_syntax(self, tmp_path, args): Checks that the output file is valid python syntax by executing it" """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run([sys.executable, outfile, "--help"], check=True) @@ -112,21 +113,7 @@ def test_trailing_whitespace(self, args): """ Tests that the generated file does not have trailing whitespace """ - p = subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - "-", - ], - check=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) + p = shacl2code_generate(args, "-") for num, line in enumerate(p.stdout.splitlines()): assert ( @@ -137,21 +124,7 @@ def test_tabs(self, args): """ Tests that the output file doesn't contain tabs """ - p = subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - "-", - ], - check=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) + p = shacl2code_generate(args, "-") for num, line in enumerate(p.stdout.splitlines()): assert "\t" not in line, f"Line {num + 1} has tabs" @@ -174,19 +147,7 @@ def test_mypy(self, tmp_path, args): Mypy static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["mypy", outfile], encoding="utf-8", @@ -198,19 +159,7 @@ def test_pyrefly(self, tmp_path, args): Pyrefly static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["pyrefly", "check", outfile], encoding="utf-8", @@ -222,19 +171,7 @@ def test_pyright(self, tmp_path, args): Pyright static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["pyright", outfile], encoding="utf-8", @@ -242,23 +179,24 @@ def test_pyright(self, tmp_path, args): ) -def test_roundtrip(model, tmp_path, roundtrip): - def check_file(p, expect, digest): - sha1 = hashlib.sha1() - with p.open("rb") as f: - while True: - d = f.read(4096) - if not d: - break - sha1.update(d) +def check_file(p, expect, digest): + sha1 = hashlib.sha1() + with p.open("rb") as f: + while True: + d = f.read(4096) + if not d: + break + sha1.update(d) - assert sha1.hexdigest() == digest + assert sha1.hexdigest() == digest - with p.open("r") as f: - data = json.load(f) + with p.open("r") as f: + data = json.load(f) - assert data == expect + assert data == expect + +def test_roundtrip(model, tmp_path, roundtrip): doc = model.SHACLObjectSet() with roundtrip.open("r") as f: d = model.JSONLDDeserializer() @@ -298,6 +236,51 @@ def test_script_roundtrip(test_script, tmp_path, roundtrip): assert data == expect_data +def test_from_rdf_roundtrip(model, tmp_path, roundtrip): + with roundtrip.open("r") as f: + expect_data = json.load(f) + + # Parse data using RDF + g = rdflib.Graph() + g.parse(roundtrip) + + # Convert to SHACL objects + objset = model.SHACLObjectSet() + model.RDFDeserializer().read(g, objset) + + # Write out + outfile = tmp_path / "out.json" + with outfile.open("wb") as f: + digest = model.JSONLDInlineSerializer().write(objset, f) + + check_file(outfile, expect_data, digest) + + +def test_to_rdf_roundtrip(model, tmp_path, roundtrip): + with roundtrip.open("r") as f: + expect_data = json.load(f) + + # Read JSON data + objset = model.SHACLObjectSet() + with roundtrip.open("r") as f: + model.JSONLDDeserializer().read(f, objset) + + # Convert to RDF + g = rdflib.Graph() + model.RDFSerializer().write(objset, g) + + # Convert from RDF to new object set + objset = model.SHACLObjectSet() + model.RDFDeserializer().read(g, objset) + + # Write out + outfile = tmp_path / "out.json" + with outfile.open("wb") as f: + digest = model.JSONLDInlineSerializer().write(objset, f) + + check_file(outfile, expect_data, digest) + + def test_jsonschema_validation(roundtrip, test_jsonschema): with roundtrip.open("r") as f: data = json.load(f) @@ -1157,58 +1140,70 @@ def test_extensible_prop(model, test_context_url, prop, serkey, value, expect): @pytest.mark.parametrize( - "iri,value,expect,ser_expect", + "iri,value,expect,ser_data,ser_expect", [ ( "extensible_class_property", "foo", KeyError, None, + None, ), ( "http://example.org/extensible-test-prop", "foo", SAME_AS_VALUE, - SAME_AS_VALUE, + ["foo"], + ["foo"], ), ( "http://example.org/extensible-test-prop", 1, SAME_AS_VALUE, - SAME_AS_VALUE, + [1], + [1], ), ( "http://example.org/extensible-test-prop", 1.123, SAME_AS_VALUE, - "1.123", + ["1.123"], + [1.123], ), ( "http://example.org/extensible-test-prop", object(), SAME_AS_VALUE, TypeError, + None, ), ( "http://example.org/extensible-test-prop", [1, "foo", 1.123], SAME_AS_VALUE, [1, "foo", "1.123"], + [1, "foo", 1.123], ), ( "http://example.org/extensible-test-prop", [object()], SAME_AS_VALUE, TypeError, + None, ), ], ) -def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect): +def test_extensible_iri( + model, test_context_url, iri, value, expect, ser_data, ser_expect +): e = model.extensible_class(extensible_class_required="required") if expect is SAME_AS_VALUE: expect = value + if ser_data is SAME_AS_VALUE: + ser_data = value + if ser_expect is SAME_AS_VALUE: ser_expect = value @@ -1224,8 +1219,8 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) objset = model.SHACLObjectSet() objset.add(e) - if isinstance(ser_expect, type) and issubclass(ser_expect, Exception): - with pytest.raises(ser_expect): + if isinstance(ser_data, type) and issubclass(ser_data, Exception): + with pytest.raises(ser_data): data = s.serialize_data(objset) else: data = s.serialize_data(objset) @@ -1233,7 +1228,7 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) "@context": test_context_url, "@type": "extensible-class", "extensible-class/required": "required", - iri: ser_expect, + iri: ser_data, } objset = model.SHACLObjectSet() @@ -1241,7 +1236,7 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) d.deserialize_data(data, objset) e = objset.objects.pop() - assert e[iri] == expect + assert e[iri] == ser_expect del e[iri] with pytest.raises(KeyError):