From f4ac7d468bf2a17038ba539d16133cbd1c0358f2 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 20 Aug 2025 08:18:53 -0600 Subject: [PATCH 1/5] tests: python: Factor out generation Many tests are now invoking shacl2code to generate bindings, so factor out the code to make it simpler --- tests/test_python.py | 115 ++++++++++--------------------------------- 1 file changed, 26 insertions(+), 89 deletions(-) diff --git a/tests/test_python.py b/tests/test_python.py index cbbe6e32..97b723bf 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -32,23 +32,36 @@ TEST_TZ = timezone(timedelta(hours=-2), name="TST") +def shacl2code_generate(args, outfile): + return subprocess.run( + [ + "shacl2code", + "generate", + ] + + args + + [ + "python", + "--output", + outfile, + ], + check=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + + @pytest.fixture(scope="module") def test_context(tmp_path_factory, model_server): tmp_directory = tmp_path_factory.mktemp("pythontestcontext") outfile = tmp_directory / "model.py" - subprocess.run( + shacl2code_generate( [ - "shacl2code", - "generate", "--input", TEST_MODEL, "--context", model_server + "/test-context.json", - "python", - "--output", - outfile, ], - check=True, + outfile, ) outfile.chmod(0o755) yield (tmp_directory, outfile) @@ -92,19 +105,7 @@ def test_output_syntax(self, tmp_path, args): Checks that the output file is valid python syntax by executing it" """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run([sys.executable, outfile, "--help"], check=True) @@ -112,21 +113,7 @@ def test_trailing_whitespace(self, args): """ Tests that the generated file does not have trailing whitespace """ - p = subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - "-", - ], - check=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) + p = shacl2code_generate(args, "-") for num, line in enumerate(p.stdout.splitlines()): assert ( @@ -137,21 +124,7 @@ def test_tabs(self, args): """ Tests that the output file doesn't contain tabs """ - p = subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - "-", - ], - check=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) + p = shacl2code_generate(args, "-") for num, line in enumerate(p.stdout.splitlines()): assert "\t" not in line, f"Line {num + 1} has tabs" @@ -174,19 +147,7 @@ def test_mypy(self, tmp_path, args): Mypy static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["mypy", outfile], encoding="utf-8", @@ -198,19 +159,7 @@ def test_pyrefly(self, tmp_path, args): Pyrefly static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["pyrefly", "check", outfile], encoding="utf-8", @@ -222,19 +171,7 @@ def test_pyright(self, tmp_path, args): Pyright static type checking """ outfile = tmp_path / "output.py" - subprocess.run( - [ - "shacl2code", - "generate", - ] - + args - + [ - "python", - "--output", - outfile, - ], - check=True, - ) + shacl2code_generate(args, outfile) subprocess.run( ["pyright", outfile], encoding="utf-8", From 8c98eec0d62c2f0c309ab60bf28278a1ef8081af Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 20 Aug 2025 10:20:51 -0600 Subject: [PATCH 2/5] python: Reformat Reformats the python code. Also configures black to correctly reformat the python template file. --- pyproject.toml | 3 + src/shacl2code/lang/templates/python.j2 | 197 +++++++++++------------- 2 files changed, 95 insertions(+), 105 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2db36637..9845c380 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,6 @@ pythonpath = [ [tool.coverage.run] relative_files = true + +[tool.black] +include = '(\.pyi?$)|(python\.j2$)' diff --git a/src/shacl2code/lang/templates/python.j2 b/src/shacl2code/lang/templates/python.j2 index 4139f093..c7e7a5e3 100644 --- a/src/shacl2code/lang/templates/python.j2 +++ b/src/shacl2code/lang/templates/python.j2 @@ -43,7 +43,9 @@ def check_type(obj: Any, types: Union[Type[Any], Tuple[Type[Any], ...]]) -> None raise TypeError( f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj).__name__}" ) - raise TypeError(f"Value must be of type {types.__name__}. Got {type(obj).__name__}") + raise TypeError( + f"Value must be of type {types.__name__}. Got {type(obj).__name__}" + ) class Property(ABC): @@ -84,10 +86,14 @@ class Property(ABC): def walk(self, value, callback: Callable, path: List[str]) -> None: callback(value, path) - def iter_objects(self, value, recursive: bool, visited: Set[SHACLObject]) -> Iterable[Any]: + def iter_objects( + self, value, recursive: bool, visited: Set[SHACLObject] + ) -> Iterable[Any]: return [] - def link_prop(self, value, objectset, missing: Optional[Set[str]], visited: Set[SHACLObject]): + def link_prop( + self, value, objectset, missing: Optional[Set[str]], visited: Set[SHACLObject] + ): return value def to_string(self, value) -> str: @@ -95,15 +101,11 @@ class Property(ABC): @abstractmethod def encode(self, encoder, value, state) -> None: - raise NotImplementedError( - "Subclasses must implement encode method" - ) + raise NotImplementedError("Subclasses must implement encode method") @abstractmethod def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Any: - raise NotImplementedError( - "Subclasses must implement decode method" - ) + raise NotImplementedError("Subclasses must implement decode method") class StringProp(Property): @@ -119,7 +121,9 @@ class StringProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_string(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[str]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[str]: return decoder.read_string() @@ -127,7 +131,9 @@ class AnyURIProp(StringProp): def encode(self, encoder, value, state) -> None: encoder.write_iri(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[str]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[str]: return decoder.read_iri() @@ -146,7 +152,9 @@ class DateTimeProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_datetime(self.to_string(value)) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[datetime]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[datetime]: s = decoder.read_datetime() if s is None: return None @@ -158,7 +166,9 @@ class DateTimeProp(Property): value = value.astimezone() offset = value.utcoffset() if offset is not None: - seconds = offset % timedelta(minutes=-1 if offset.total_seconds() < 0 else 1) + seconds = offset % timedelta( + minutes=-1 if offset.total_seconds() < 0 else 1 + ) if seconds: offset = offset - seconds value = value.replace(tzinfo=timezone(offset)) @@ -202,7 +212,9 @@ class IntegerProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_integer(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[int]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[int]: return decoder.read_integer() @@ -229,7 +241,9 @@ class BooleanProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_bool(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[bool]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[bool]: return decoder.read_bool() @@ -242,7 +256,9 @@ class FloatProp(Property): def encode(self, encoder, value, state) -> None: encoder.write_float(value) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> Optional[float]: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> Optional[float]: return decoder.read_float() @@ -322,7 +338,7 @@ class ObjectProp(IRIProp): return value.encode(encoder, state) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] =None): + def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None): iri = decoder.read_iri() if iri is None: return self.cls.decode(decoder, objectset=objectset) @@ -398,7 +414,9 @@ class ListProxy(object): self.__prop.validate(value) self.__data[key] = self.__prop.set(value) else: - raise TypeError(f"ListProxy indices must be integers or slices. Got {type(key).__name__}") + raise TypeError( + f"ListProxy indices must be integers or slices. Got {type(key).__name__}" + ) def __delitem__(self, key) -> None: del self.__data[key] @@ -489,7 +507,9 @@ class ListProp(Property): with list_s.write_list_item() as item_s: self.prop.encode(item_s, v, state) - def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None) -> ListProxy: + def decode( + self, decoder, *, objectset: Optional[SHACLObjectSet] = None + ) -> ListProxy: data = [] for val_d in decoder.read_list(): v = self.prop.decode(val_d, objectset=objectset) @@ -546,7 +566,9 @@ def is_blank_node(s: Any) -> bool: return True -def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: bool = False): +def register( + type_iri: str, *, compact_type: Optional[str] = None, abstract: bool = False +): def add_class(key: str, c: Type[SHACLObject]) -> None: assert ( key not in SHACLObject.CLASSES @@ -560,9 +582,7 @@ def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: boo """ global NAMED_INDIVIDUALS - assert issubclass( - c, SHACLObject - ), f"{c} is not derived from SHACLObject" + assert issubclass(c, SHACLObject), f"{c} is not derived from SHACLObject" c._OBJ_TYPE = type_iri c.IS_ABSTRACT = abstract @@ -585,6 +605,7 @@ def register(type_iri: str, *, compact_type: Optional[str] = None, abstract: boo register_lock = threading.Lock() NAMED_INDIVIDUALS: Set[str] = set() + @functools.total_ordering class SHACLObject(object): CLASSES: Dict[str, Type] = {} @@ -615,7 +636,9 @@ class SHACLObject(object): ) if self.__class__.IS_DEPRECATED: - warnings.warn(f"{self.__class__.__name__} is deprecated", DeprecationWarning) + warnings.warn( + f"{self.__class__.__name__} is deprecated", DeprecationWarning + ) with register_lock: cls = self.__class__ @@ -718,7 +741,9 @@ class SHACLObject(object): f"'{name}' is not a valid property of {self.__class__.__name__}" ) - def __get_prop(self, iri: str) -> Tuple[Property, Optional[int], Optional[int], str, Optional[str]]: + def __get_prop( + self, iri: str + ) -> Tuple[Property, Optional[int], Optional[int], str, Optional[str]]: if iri not in self._OBJ_PROPERTIES: raise KeyError( f"'{iri}' is not a valid property of {self.__class__.__name__}" @@ -726,7 +751,11 @@ class SHACLObject(object): return self._OBJ_PROPERTIES[iri] - def __iter_props(self) -> Iterator[Tuple[str, Property, Optional[int], Optional[int], str, Optional[str]]]: + def __iter_props( + self, + ) -> Iterator[ + Tuple[str, Property, Optional[int], Optional[int], str, Optional[str]] + ]: for iri, v in self._OBJ_PROPERTIES.items(): yield iri, *v @@ -754,7 +783,9 @@ class SHACLObject(object): prop, _, _, pyname, _ = self.__get_prop(iri) prop.validate(value) if iri in self._OBJ_DEPRECATED: - warnings.warn(f"{self.__class__.__name__}.{pyname} is deprecated", DeprecationWarning) + warnings.warn( + f"{self.__class__.__name__}.{pyname} is deprecated", DeprecationWarning + ) self.__dict__["_obj_data"][iri] = prop.set(value) def __delitem__(self, iri: str) -> None: @@ -913,7 +944,9 @@ class SHACLObject(object): return False - def link_helper(self, objectset: Optional[SHACLObjectSet], missing, visited) -> None: + def link_helper( + self, objectset: Optional[SHACLObjectSet], missing, visited + ) -> None: if self in visited: return @@ -1214,7 +1247,9 @@ class SHACLObjectSet(object): return self.missing_ids - def find_by_id(self, _id: str, default: Optional[SHACLObject] = None) -> Optional[SHACLObject]: + def find_by_id( + self, _id: str, default: Optional[SHACLObject] = None + ) -> Optional[SHACLObject]: """ Find object by ID @@ -1238,7 +1273,9 @@ class SHACLObjectSet(object): for child in o.iter_objects(recursive=True, visited=visited): yield child - def foreach_type(self, typ: Union[str, SHACLObject], *, match_subclass: bool = True) -> Iterable[SHACLObject]: + def foreach_type( + self, typ: Union[str, SHACLObject], *, match_subclass: bool = True + ) -> Iterable[SHACLObject]: """ Iterate over each object of a specified type (or subclass there of) @@ -1391,9 +1428,7 @@ class Decoder(ABC): Consumes the next item of any type """ - raise NotImplementedError( - "Subclasses must implement read_value method" - ) + raise NotImplementedError("Subclasses must implement read_value method") @abstractmethod def read_string(self) -> Optional[str]: @@ -1403,9 +1438,7 @@ class Decoder(ABC): Returns the string value of the next item, or `None` if the next item is not a string """ - raise NotImplementedError( - "Subclasses must implement read_string method" - ) + raise NotImplementedError("Subclasses must implement read_string method") @abstractmethod def read_datetime(self) -> Optional[str]: @@ -1419,9 +1452,7 @@ class Decoder(ABC): implementation can just check if the next item is a string without worrying about the format """ - raise NotImplementedError( - "Subclasses must implement read_datetime method" - ) + raise NotImplementedError("Subclasses must implement read_datetime method") @abstractmethod def read_integer(self) -> Optional[int]: @@ -1431,9 +1462,7 @@ class Decoder(ABC): Returns the integer value of the next item, or `None` if the next item is not an integer """ - raise NotImplementedError( - "Subclasses must implement read_integer method" - ) + raise NotImplementedError("Subclasses must implement read_integer method") @abstractmethod def read_iri(self) -> Optional[str]: @@ -1446,9 +1475,7 @@ class Decoder(ABC): The returned string should be either a fully-qualified IRI, or a blank node ID """ - raise NotImplementedError( - "Subclasses must implement read_iri method" - ) + raise NotImplementedError("Subclasses must implement read_iri method") @abstractmethod def read_enum(self, e) -> Optional[str]: @@ -1462,9 +1489,7 @@ class Decoder(ABC): actually a member of the specified Enum, so the `Decoder` does not need to check that, but can if it wishes """ - raise NotImplementedError( - "Subclasses must implement read_enum method" - ) + raise NotImplementedError("Subclasses must implement read_enum method") @abstractmethod def read_bool(self) -> Optional[bool]: @@ -1474,9 +1499,7 @@ class Decoder(ABC): Returns the boolean value of the next item, or `None` if the next item is not a boolean """ - raise NotImplementedError( - "Subclasses must implement read_bool method" - ) + raise NotImplementedError("Subclasses must implement read_bool method") @abstractmethod def read_float(self) -> Optional[float]: @@ -1486,9 +1509,7 @@ class Decoder(ABC): Returns the float value of the next item, or `None` if the next item is not a float """ - raise NotImplementedError( - "Subclasses must implement read_float method" - ) + raise NotImplementedError("Subclasses must implement read_float method") @abstractmethod def read_list(self) -> Iterator["Decoder"]: @@ -1499,9 +1520,7 @@ class Decoder(ABC): generated `Decoder` can be used to read the corresponding item from the list """ - raise NotImplementedError( - "Subclasses must implement read_list method" - ) + raise NotImplementedError("Subclasses must implement read_list method") @abstractmethod def is_list(self) -> bool: @@ -1510,9 +1529,7 @@ class Decoder(ABC): Returns True if the next item is a list, or False if it is a scalar """ - raise NotImplementedError( - "Subclasses must implement is_list method" - ) + raise NotImplementedError("Subclasses must implement is_list method") @abstractmethod def read_object(self) -> Tuple[Any, "Decoder"]: @@ -1526,9 +1543,7 @@ class Decoder(ABC): Properties will be read out of the object using `read_property` and `read_object_id` """ - raise NotImplementedError( - "Subclasses must implement read_object method" - ) + raise NotImplementedError("Subclasses must implement read_object method") @abstractmethod @contextmanager @@ -1540,9 +1555,7 @@ class Decoder(ABC): value of the property with the given key in current object, or `None` if the property does not exist in the current object. """ - raise NotImplementedError( - "Subclasses must implement read_property method" - ) + raise NotImplementedError("Subclasses must implement read_property method") @abstractmethod def object_keys(self) -> Iterator[str]: @@ -1551,9 +1564,7 @@ class Decoder(ABC): Iterates over all the serialized keys for the current object """ - raise NotImplementedError( - "Subclasses must implement object_keys method" - ) + raise NotImplementedError("Subclasses must implement object_keys method") @abstractmethod def read_object_id(self, alias=None) -> Optional[Any]: @@ -1568,9 +1579,7 @@ class Decoder(ABC): If `alias` is provided, is is a hint as to another name by which the ID might be found, if the `Decoder` supports aliases for an ID """ - raise NotImplementedError( - "Subclasses must implement read_object_id method" - ) + raise NotImplementedError("Subclasses must implement read_object_id method") class JSONLDDecoder(Decoder): @@ -1684,9 +1693,7 @@ class Encoder(ABC): Encodes the value as a string in the output """ - raise NotImplementedError( - "Subclasses must implement write_string method" - ) + raise NotImplementedError("Subclasses must implement write_string method") @abstractmethod def write_datetime(self, v) -> None: @@ -1697,9 +1704,7 @@ class Encoder(ABC): Note: The provided string is already correctly encoded as an ISO datetime """ - raise NotImplementedError( - "Subclasses must implement write_datetime method" - ) + raise NotImplementedError("Subclasses must implement write_datetime method") @abstractmethod def write_integer(self, v) -> None: @@ -1708,9 +1713,7 @@ class Encoder(ABC): Encodes the value as an integer in the output """ - raise NotImplementedError( - "Subclasses must implement write_integer method" - ) + raise NotImplementedError("Subclasses must implement write_integer method") @abstractmethod def write_iri(self, v, compact=None) -> None: @@ -1722,9 +1725,7 @@ class Encoder(ABC): the serialization supports compacted IRIs, it should be preferred to the full IRI """ - raise NotImplementedError( - "Subclasses must implement write_iri method" - ) + raise NotImplementedError("Subclasses must implement write_iri method") @abstractmethod def write_enum(self, v, e, compact=None) -> None: @@ -1735,9 +1736,7 @@ class Encoder(ABC): qualified IRI. If `compact` is provided and the serialization supports compacted IRIs, it should be preferred to the full IRI. """ - raise NotImplementedError( - "Subclasses must implement write_enum method" - ) + raise NotImplementedError("Subclasses must implement write_enum method") @abstractmethod def write_bool(self, v) -> None: @@ -1746,9 +1745,7 @@ class Encoder(ABC): Encodes the value as a boolean in the output """ - raise NotImplementedError( - "Subclasses must implement write_bool method" - ) + raise NotImplementedError("Subclasses must implement write_bool method") @abstractmethod def write_float(self, v) -> None: @@ -1757,9 +1754,7 @@ class Encoder(ABC): Encodes the value as a floating point number in the output """ - raise NotImplementedError( - "Subclasses must implement write_float method" - ) + raise NotImplementedError("Subclasses must implement write_float method") @abstractmethod @contextmanager @@ -1781,9 +1776,7 @@ class Encoder(ABC): Properties will be written the object using `write_property` """ - raise NotImplementedError( - "Subclasses must implement write_object method" - ) + raise NotImplementedError("Subclasses must implement write_object method") @abstractmethod @contextmanager @@ -1798,9 +1791,7 @@ class Encoder(ABC): the serialization supports compacted IRIs, it should be preferred to the full IRI. """ - raise NotImplementedError( - "Subclasses must implement write_property method" - ) + raise NotImplementedError("Subclasses must implement write_property method") @abstractmethod @contextmanager @@ -1813,9 +1804,7 @@ class Encoder(ABC): Each item of the list will be added using `write_list_item` """ - raise NotImplementedError( - "Subclasses must implement write_list method" - ) + raise NotImplementedError("Subclasses must implement write_list method") @abstractmethod @contextmanager @@ -1826,9 +1815,7 @@ class Encoder(ABC): A context manager that yields an `Encoder` that can be used to encode the value for a list item """ - raise NotImplementedError( - "Subclasses must implement write_list_item method" - ) + raise NotImplementedError("Subclasses must implement write_list_item method") class JSONLDEncoder(Encoder): From ea518e5130ed63cef12c6aeb46412ee1562e1782 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 21 Aug 2025 09:25:43 -0600 Subject: [PATCH 3/5] python: Convert 00:00 timezone to UTC If the timezone offset is 00:00, convert the datetime to explicitly use the UTC timezone. --- src/shacl2code/lang/templates/python.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/shacl2code/lang/templates/python.j2 b/src/shacl2code/lang/templates/python.j2 index c7e7a5e3..f4746b68 100644 --- a/src/shacl2code/lang/templates/python.j2 +++ b/src/shacl2code/lang/templates/python.j2 @@ -164,6 +164,8 @@ class DateTimeProp(Property): def _normalize(self, value: datetime) -> datetime: if value.utcoffset() is None: value = value.astimezone() + + # Remove seconds from timezone offset offset = value.utcoffset() if offset is not None: seconds = offset % timedelta( @@ -172,6 +174,12 @@ class DateTimeProp(Property): if seconds: offset = offset - seconds value = value.replace(tzinfo=timezone(offset)) + + # Convert 00:00 timezone offset to UTC + offset = value.utcoffset() + if offset is not None and offset.seconds == 0: + value = value.astimezone(timezone.utc) + value = value.replace(microsecond=0) return value From b1a4338ef8464aaaf81df7ee9a75d68eeaf68901 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 21 Aug 2025 09:27:20 -0600 Subject: [PATCH 4/5] python: Encode extensible properties as lists All other language bindings (C++, go) will encode extensible properties as a list, even if they only contain one item, so make python match this behavior. --- src/shacl2code/lang/templates/python.j2 | 39 ++++++++++++------------- tests/data/roundtrip.json | 10 ++++--- tests/test_python.py | 30 +++++++++++++------ 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/shacl2code/lang/templates/python.j2 b/src/shacl2code/lang/templates/python.j2 index f4746b68..c9c1758f 100644 --- a/src/shacl2code/lang/templates/python.j2 +++ b/src/shacl2code/lang/templates/python.j2 @@ -1049,25 +1049,6 @@ class SHACLExtensibleObject(SHACLObject): self.__dict__["_obj_data"][key] = decode_value(prop_d) def _encode_properties(self, encoder, state): - def encode_value(encoder, v): - if isinstance(v, bool): - encoder.write_bool(v) - elif isinstance(v, str): - encoder.write_string(v) - elif isinstance(v, int): - encoder.write_integer(v) - elif isinstance(v, float): - encoder.write_float(v) - elif isinstance(v, list): - with encoder.write_list() as list_s: - for i in v: - with list_s.write_list_item() as item_s: - encode_value(item_s, i) - else: - raise TypeError( - f"Unsupported serialized type {type(v)} with value '{v}'" - ) - super()._encode_properties(encoder, state) if self.CLOSED: return @@ -1077,7 +1058,25 @@ class SHACLExtensibleObject(SHACLObject): continue with encoder.write_property(iri) as prop_s: - encode_value(prop_s, value) + if isinstance(value, list): + v = value + else: + v = [value] + with prop_s.write_list() as list_s: + for i in v: + with list_s.write_list_item() as item_s: + if isinstance(i, bool): + item_s.write_bool(i) + elif isinstance(i, str): + item_s.write_string(i) + elif isinstance(i, int): + item_s.write_integer(i) + elif isinstance(i, float): + item_s.write_float(i) + else: + raise TypeError( + f"Unsupported serialized type {type(i)} with value {i!r}" + ) def __setitem__(self, iri: str, value) -> None: try: diff --git a/tests/data/roundtrip.json b/tests/data/roundtrip.json index 33bc8da7..c04071e0 100644 --- a/tests/data/roundtrip.json +++ b/tests/data/roundtrip.json @@ -90,10 +90,12 @@ "@type": "uses-extensible-abstract-class", "@id": "http://serialize.example.com/test-uses-extensible-abstract", "uses-extensible-abstract-class/prop": { - "@type": " http://serialize.example.com/custom-extensible", - "http://custom-prop": [ - "abc" - ] + "@type": "http://serialize.example.com/custom-extensible", + "http://custom-list-prop": [ + "abc", + "def" + ], + "http://custom-scalar-prop": ["abc"] } } ] diff --git a/tests/test_python.py b/tests/test_python.py index 97b723bf..769ac7d4 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -1094,58 +1094,70 @@ def test_extensible_prop(model, test_context_url, prop, serkey, value, expect): @pytest.mark.parametrize( - "iri,value,expect,ser_expect", + "iri,value,expect,ser_data,ser_expect", [ ( "extensible_class_property", "foo", KeyError, None, + None, ), ( "http://example.org/extensible-test-prop", "foo", SAME_AS_VALUE, - SAME_AS_VALUE, + ["foo"], + ["foo"], ), ( "http://example.org/extensible-test-prop", 1, SAME_AS_VALUE, - SAME_AS_VALUE, + [1], + [1], ), ( "http://example.org/extensible-test-prop", 1.123, SAME_AS_VALUE, - "1.123", + ["1.123"], + [1.123], ), ( "http://example.org/extensible-test-prop", object(), SAME_AS_VALUE, TypeError, + None, ), ( "http://example.org/extensible-test-prop", [1, "foo", 1.123], SAME_AS_VALUE, [1, "foo", "1.123"], + [1, "foo", 1.123], ), ( "http://example.org/extensible-test-prop", [object()], SAME_AS_VALUE, TypeError, + None, ), ], ) -def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect): +def test_extensible_iri( + model, test_context_url, iri, value, expect, ser_data, ser_expect +): e = model.extensible_class(extensible_class_required="required") if expect is SAME_AS_VALUE: expect = value + if ser_data is SAME_AS_VALUE: + ser_data = value + if ser_expect is SAME_AS_VALUE: ser_expect = value @@ -1161,8 +1173,8 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) objset = model.SHACLObjectSet() objset.add(e) - if isinstance(ser_expect, type) and issubclass(ser_expect, Exception): - with pytest.raises(ser_expect): + if isinstance(ser_data, type) and issubclass(ser_data, Exception): + with pytest.raises(ser_data): data = s.serialize_data(objset) else: data = s.serialize_data(objset) @@ -1170,7 +1182,7 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) "@context": test_context_url, "@type": "extensible-class", "extensible-class/required": "required", - iri: ser_expect, + iri: ser_data, } objset = model.SHACLObjectSet() @@ -1178,7 +1190,7 @@ def test_extensible_iri(model, test_context_url, iri, value, expect, ser_expect) d.deserialize_data(data, objset) e = objset.objects.pop() - assert e[iri] == expect + assert e[iri] == ser_expect del e[iri] with pytest.raises(KeyError): From 6a00eeeedd9be9036e121daa0ec5b2e9ac0281b1 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 21 Aug 2025 09:29:29 -0600 Subject: [PATCH 5/5] python: Add RDF serializer and deserializer Adds support for an optional serializer and deserializer that can go to and from anf rdflib Graph. This should make it possible to correctly encode an RDF graph in a format that can be accepted by the JSON schema --- src/shacl2code/lang/templates/python.j2 | 342 ++++++++++++++++++++++-- tests/data/roundtrip.json | 5 +- tests/test_python.py | 74 ++++- 3 files changed, 379 insertions(+), 42 deletions(-) diff --git a/src/shacl2code/lang/templates/python.j2 b/src/shacl2code/lang/templates/python.j2 index c9c1758f..92d54e33 100644 --- a/src/shacl2code/lang/templates/python.j2 +++ b/src/shacl2code/lang/templates/python.j2 @@ -8,6 +8,7 @@ from __future__ import annotations +import decimal import functools import hashlib import json @@ -158,6 +159,8 @@ class DateTimeProp(Property): s = decoder.read_datetime() if s is None: return None + if isinstance(s, datetime): + return self._normalize(s) v = self.from_string(s) return self._normalize(v) @@ -347,10 +350,11 @@ class ObjectProp(IRIProp): return value.encode(encoder, state) def decode(self, decoder, *, objectset: Optional[SHACLObjectSet] = None): - iri = decoder.read_iri() - if iri is None: + if decoder.is_object(): return self.cls.decode(decoder, objectset=objectset) + iri = decoder.read_iri() + iri = self.expand(iri) or iri if objectset is None: @@ -775,17 +779,17 @@ class SHACLObject(object): if self.NODE_KIND == NodeKind.BlankNode: if not is_blank_node(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{iri}' cannot be set to '{value}' and must start with '_:'" + f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{iri}' cannot be set to {value!r} and must start with '_:'" ) elif self.NODE_KIND == NodeKind.IRI: if not is_IRI(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{iri}' cannot be set to '{value}'" + f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{iri}' cannot be set to {value!r}" ) else: if not is_blank_node(value) and not is_IRI(value): raise ValueError( - f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{iri}' '{value}'. Must be a blank node or IRI" + f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{iri}' {value!r}. Must be a blank node or IRI" ) prop, _, _, pyname, _ = self.__get_prop(iri) @@ -899,18 +903,9 @@ class SHACLObject(object): raise TypeError("Unable to determine type for object") obj = cls._make_object(typ) - _id = None - for key in (obj.ID_ALIAS, obj._OBJ_IRIS["_id"]): - with obj_d.read_property(key) as prop_d: - if prop_d is None: - continue - - _id = prop_d.read_iri() - if _id is None: - raise TypeError(f"Object key '{key}' is the wrong type") - - obj._id = _id - break + _id = obj_d.read_object_id(obj.ID_ALIAS) + if _id is not None: + obj._id = _id if obj.NODE_KIND == NodeKind.IRI and not obj._id: raise ValueError("Object is missing required IRI") @@ -920,11 +915,10 @@ class SHACLObject(object): v = objectset.find_by_id(obj._id) if v is not None: return v + objectset.add_index(obj) obj._decode_properties(obj_d, objectset=objectset) - if objectset is not None: - objectset.add_index(obj) return obj def _decode_properties(self, decoder, objectset: Optional[SHACLObjectSet] = None): @@ -1317,6 +1311,49 @@ class SHACLObjectSet(object): return SHACLObjectSet(new_objects, link=True) + def inline_blank_nodes(self) -> None: + """ + Removes (inlines) blank node objects from the root object set if they + are referenced in only one other location besides the root. + + Deserializers that do not preserve the tree-like structure of the + objects (e.g. RDF) should call this to ensure that blank nodes are + inline correctly + """ + ref_counts: Dict[SHACLObject, int] = {} + + def walk_callback(value: SHACLObject, path: List[str]) -> bool: + nonlocal ref_counts + + if not isinstance(value, SHACLObject): + return True + + ref_counts.setdefault(value, 0) + ref_counts[value] += 1 + if ref_counts[value] > 1: + return False + + return True + + for o in self.objects: + # Note that every object in the root object set gets at least one + # reference + o.walk(walk_callback) + + new_objects = set() + for o in self.objects: + if is_IRI(o._id): + new_objects.add(o) + # If the object is a blank node and is only referenced by this + # root list and one other location, remove it from the root list + # + # A count of 1 means the object is only referenced by the root, and + # therefore must be kept + elif ref_counts[o] != 2: + new_objects.add(o) + + self.objects = new_objects + def encode(self, encoder, force_list: bool = False, *, key=None) -> None: """ Serialize a list of objects to a serialization encoder @@ -1334,7 +1371,7 @@ class SHACLObjectSet(object): return True # Remove blank node ID for re-assignment - if value._id and value._id.startswith("_:"): + if is_blank_node(value._id): del value._id if value._id: @@ -1392,7 +1429,7 @@ class SHACLObjectSet(object): self.create_index() for obj_d in decoder.read_list(): - o = SHACLObject.decode(obj_d, objectset=self) + o = SHACLExtensibleObject.decode(obj_d, objectset=self) self.objects.add(o) self._link() @@ -1564,6 +1601,15 @@ class Decoder(ABC): """ raise NotImplementedError("Subclasses must implement read_property method") + @abstractmethod + def is_object(self) -> bool: + """ + Checks if the item is an object + + Returns True if the item is an object, or False if is not + """ + raise NotImplementedError("Subclasses must implement is_object method") + @abstractmethod def object_keys(self) -> Iterator[str]: """ @@ -1659,6 +1705,9 @@ class JSONLDDecoder(Decoder): else: yield None + def is_object(self) -> bool: + return isinstance(self.data, dict) + def object_keys(self) -> Iterator[str]: for key in self.data.keys(): if key in ("@type", "{{ context.compact_iri('@type') }}"): @@ -1765,7 +1814,7 @@ class Encoder(ABC): @abstractmethod @contextmanager - def write_object(self, o, _id, needs_id: bool): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): """ Write object @@ -1858,7 +1907,7 @@ class JSONLDEncoder(Encoder): self.data[compact or iri] = s.data # type: ignore # within write_object() context, self.data is always dict or None @contextmanager - def write_object(self, o, _id, needs_id): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): self.data = { "{{ context.compact_iri('@type') }}": o.COMPACT_TYPE or o.TYPE, } @@ -1892,7 +1941,7 @@ class JSONLDSerializer(object): ): h = JSONLDEncoder() objectset.encode(h, force_at_graph) - data: Dict[str, Union[str, List[str]]] = {} + data: Dict[str, Any] = {} if len(CONTEXT_URLS) == 1: data["@context"] = CONTEXT_URLS[0] elif CONTEXT_URLS: @@ -1985,7 +2034,7 @@ class JSONLDInlineEncoder(Encoder): self.comma = True @contextmanager - def write_object(self, o, _id, needs_id): + def write_object(self, o: SHACLObject, _id: str, needs_id: bool): self._write_comma() self.write("{") @@ -2053,6 +2102,249 @@ class JSONLDInlineSerializer(object): return sha1.hexdigest() +try: + import rdflib + import rdflib.term + from rdflib.namespace import RDF + + class RDFDecoder(Decoder): + def __init__( + self, + graph: rdflib.Graph, + subject: Optional[rdflib.term.Node] = None, + predicate: Optional[rdflib.term.Node] = None, + value: Optional[rdflib.term.Node] = None, + ): + self.graph = graph + self.subject = subject + self.predicate = predicate + self.value = value + + def __read_node(self): + if self.value is not None: + return self.value + if self.predicate is None: + return None + return self.graph.value(self.subject, self.predicate) + + def read_value(self) -> Optional[Any]: + v = self.__read_node() + if isinstance(v, rdflib.term.Literal): + return v.toPython() + return None + + def read_string(self) -> Optional[str]: + v = self.read_value() + if isinstance(v, str): + return v + return None + + def read_datetime(self) -> Optional[str]: + return self.read_value() + + def read_integer(self) -> Optional[int]: + v = self.read_value() + if isinstance(v, (int, decimal.Decimal)): + return int(v) + return None + + def read_bool(self) -> Optional[bool]: + v = self.read_value() + if isinstance(v, bool): + return v + return None + + def read_float(self) -> Optional[float]: + v = self.read_value() + if isinstance(v, (int, float, str, decimal.Decimal)): + return float(v) + return None + + def read_iri(self) -> Optional[str]: + v = self.__read_node() + if isinstance(v, rdflib.term.URIRef): + return v.toPython() + elif isinstance(v, rdflib.term.Literal): + v = v.toPython() + if isinstance(v, str): + return v + elif isinstance(v, rdflib.term.BNode): + return v.n3() + return None + + def read_enum(self, e) -> Optional[str]: + v = self.__read_node() + if isinstance(v, rdflib.term.URIRef): + return v.toPython() + return None + + def read_list(self) -> Iterator["RDFDecoder"]: + if not self.subject: + blank_nodes = set() + for s in self.graph.subjects(unique=True): + # type: ignore # RDF.type is dynamic + if (s, RDF.type, None) not in self.graph: + continue + + if isinstance(s, rdflib.term.BNode): + blank_nodes.add(s) + continue + yield self.__class__(self.graph, s) + + for s in blank_nodes: + yield self.__class__(self.graph, s) + else: + for o in self.graph.objects(self.subject, self.predicate): + # type: ignore # RDF.type is dynamic + if (o, RDF.type, None) in self.graph: + yield self.__class__(self.graph, o) + else: + yield self.__class__( + self.graph, + self.subject, + self.predicate, + o, + ) + + def is_list(self) -> bool: + if not self.subject: + return True + if self.value is not None: + return False + return len(list(self.graph.objects(self.subject, self.predicate))) > 1 + + @contextmanager + def read_property(self, key) -> Iterator[Optional["RDFDecoder"]]: + if key == "@id": + yield self.__class__(self.graph, value=self.subject) + else: + yield self.__class__(self.graph, self.subject, rdflib.term.URIRef(key)) + + def is_object(self) -> bool: + n = self.__read_node() or self.subject + # type: ignore # RDF.type is dynamic + return (n, RDF.type, None) in self.graph + + def object_keys(self) -> Iterator[str]: + for p in self.graph.predicates(self.subject, unique=True): + # type: ignore # RDF.type is dynamic + if p == RDF.type: + continue + + if not isinstance(p, rdflib.term.IdentifiedNode): + raise TypeError(f"Predicate is of unknown type {type(p)}") + + yield p.toPython() + + def read_object(self) -> Tuple[Any, "RDFDecoder"]: + s = self.__read_node() + if s is None: + s = self.subject + + # type: ignore # RDF.type is dynamic + typ = self.graph.value(s, RDF.type) + if typ is None: + return None, self + + if not isinstance(typ, rdflib.term.IdentifiedNode): + raise TypeError(f"Type value is of unknown type {type(typ)}") + + return typ.toPython(), self.__class__(self.graph, s) + + def read_object_id(self, alias=None) -> Optional[Any]: + if isinstance(self.subject, rdflib.term.BNode): + return self.subject.n3() + if not isinstance(self.subject, rdflib.term.IdentifiedNode): + raise TypeError(f"Subject is of unknown type {type(self.subject)}") + return self.subject.toPython() + + class RDFDeserializer(object): + def read(self, graph: rdflib.Graph, objset: SHACLObjectSet) -> None: + d = RDFDecoder(graph) + objset.decode(d) + objset.inline_blank_nodes() + + class RDFEncoder(Encoder): + def __init__( + self, + graph: rdflib.Graph, + subject: Optional[rdflib.term.Node] = None, + predicate: Optional[rdflib.term.Node] = None, + ): + self.graph = graph + self.subject = subject + self.predicate = predicate + + def __add_literal(self, v): + if self.subject is None or self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, rdflib.Literal(v))) + + def __add_uriref(self, v): + if self.subject is None or self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, rdflib.URIRef(v))) + + def write_string(self, v): + self.__add_literal(v) + + def write_datetime(self, v): + self.__add_literal(v) + + def write_integer(self, v): + self.__add_literal(v) + + def write_iri(self, v, compact=None): + self.__add_uriref(v) + + def write_enum(self, v, e, compact=None): + self.__add_uriref(v) + + def write_bool(self, v): + self.__add_literal(v) + + def write_float(self, v): + self.__add_literal(v) + + @contextmanager + def write_property(self, iri: str, compact: Optional[str] = None): + yield self.__class__(self.graph, self.subject, rdflib.URIRef(iri)) + + @contextmanager + def write_object(self, o, _id, needs_id: bool): + obj: rdflib.term.Node + if _id.startswith("_:"): + obj = rdflib.BNode(_id[2:]) + else: + obj = rdflib.URIRef(_id) + + if self.subject is not None: + if self.predicate is None: + raise TypeError() + self.graph.add((self.subject, self.predicate, obj)) + self.graph.add((obj, RDF.type, rdflib.URIRef(o.TYPE))) # type: ignore # RDF.type is dynamic + yield self.__class__(self.graph, obj) + + @contextmanager + def write_list(self): + yield self + + @contextmanager + def write_list_item(self): + yield self + + class RDFSerializer(object): + def write(self, objset: SHACLObjectSet, g: rdflib.Graph): + """ + Write a SHACLObjectSet to an RDF graph + """ + e = RDFEncoder(g) + objset.encode(e) + +except ImportError: + pass + + def print_tree(objects, all_fields: bool = False) -> None: """ Print object tree diff --git a/tests/data/roundtrip.json b/tests/data/roundtrip.json index c04071e0..32c735d5 100644 --- a/tests/data/roundtrip.json +++ b/tests/data/roundtrip.json @@ -6,7 +6,6 @@ "link-class-link-prop": "http://serialize.example.com/link-derived-target", "link-class-link-prop-no-class": "http://serialize.example.com/link-derived-target", "link-class-link-list-prop": [ - "http://serialize.example.com/link-derived-target", "http://serialize.example.com/link-derived-target" ] }, @@ -59,8 +58,8 @@ "test-class/class-prop": "http://serialize.example.com/test-derived", "test-class/class-prop-no-class": "http://serialize.example.com/test-derived", "test-class/class-list-prop": [ - "http://serialize.example.com/test-derived", - "http://serialize.example.com/test" + "http://serialize.example.com/test", + "http://serialize.example.com/test-derived" ], "test-class/enum-prop": "foo", "test-class/enum-prop-no-class": "bar", diff --git a/tests/test_python.py b/tests/test_python.py index 769ac7d4..96f3f905 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -179,23 +179,24 @@ def test_pyright(self, tmp_path, args): ) -def test_roundtrip(model, tmp_path, roundtrip): - def check_file(p, expect, digest): - sha1 = hashlib.sha1() - with p.open("rb") as f: - while True: - d = f.read(4096) - if not d: - break - sha1.update(d) - - assert sha1.hexdigest() == digest +def check_file(p, expect, digest): + sha1 = hashlib.sha1() + with p.open("rb") as f: + while True: + d = f.read(4096) + if not d: + break + sha1.update(d) + + assert sha1.hexdigest() == digest + + with p.open("r") as f: + data = json.load(f) - with p.open("r") as f: - data = json.load(f) + assert data == expect - assert data == expect +def test_roundtrip(model, tmp_path, roundtrip): doc = model.SHACLObjectSet() with roundtrip.open("r") as f: d = model.JSONLDDeserializer() @@ -235,6 +236,51 @@ def test_script_roundtrip(test_script, tmp_path, roundtrip): assert data == expect_data +def test_from_rdf_roundtrip(model, tmp_path, roundtrip): + with roundtrip.open("r") as f: + expect_data = json.load(f) + + # Parse data using RDF + g = rdflib.Graph() + g.parse(roundtrip) + + # Convert to SHACL objects + objset = model.SHACLObjectSet() + model.RDFDeserializer().read(g, objset) + + # Write out + outfile = tmp_path / "out.json" + with outfile.open("wb") as f: + digest = model.JSONLDInlineSerializer().write(objset, f) + + check_file(outfile, expect_data, digest) + + +def test_to_rdf_roundtrip(model, tmp_path, roundtrip): + with roundtrip.open("r") as f: + expect_data = json.load(f) + + # Read JSON data + objset = model.SHACLObjectSet() + with roundtrip.open("r") as f: + model.JSONLDDeserializer().read(f, objset) + + # Convert to RDF + g = rdflib.Graph() + model.RDFSerializer().write(objset, g) + + # Convert from RDF to new object set + objset = model.SHACLObjectSet() + model.RDFDeserializer().read(g, objset) + + # Write out + outfile = tmp_path / "out.json" + with outfile.open("wb") as f: + digest = model.JSONLDInlineSerializer().write(objset, f) + + check_file(outfile, expect_data, digest) + + def test_jsonschema_validation(roundtrip, test_jsonschema): with roundtrip.open("r") as f: data = json.load(f)