diff --git a/README.md b/README.md index 241dc2b..c033ecf 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,13 @@ helpers. A derived class must either: -- have a member data matching the name of each argument to `__init__` +- have a member data matching the name of each positional argument to `__init__` - provide a user-defined `to_dict()` method which returns a dictionary whose keys match the arguments to `__init__` +Keyword-only arguments to `__init__` (those after `*`) are left out of the default `to_dict()`, so they can be used for constructor options which aren't part of an object's serialized state. + +Pickling goes through the same `to_dict()` / `from_dict()` pair, so pickles also survive renamed keywords (see below). Pickles written by versions before 1.2.0 can still be loaded. + If you change the keyword arguments to a class which derives from `Serializable` but would like to be able to deserialize older JSON representations then you can define a class-level dictionary called `_SERIALIZABLE_KEYWORD_ALIASES` which maps old keywords to new names (or `None` if a keyword was removed). ## Limitations diff --git a/docs/guides/objects.md b/docs/guides/objects.md index dfb998d..4a9313c 100644 --- a/docs/guides/objects.md +++ b/docs/guides/objects.md @@ -3,8 +3,10 @@ ## Constructor-based classes Inherit from `Serializable` when you already have a normal constructor. Store -an attribute for each constructor argument, or implement `to_dict()` with keys -that the constructor accepts. +an attribute for each positional constructor argument, or implement `to_dict()` +with keys that the constructor accepts. Keyword-only arguments (after `*`) are +left out of the default `to_dict()`, so they can hold constructor options which +aren't part of the saved state. ```python from serializable import Serializable @@ -52,6 +54,23 @@ True For a dataclass, write `object.to_json()` with `Path.write_text()` and reload the text with `YourClass.from_json()`. +## Pickle objects + +Pickling uses the same `to_dict()` and `from_dict()` methods as JSON, so +[renamed fields](migration.md) are handled the same way. Pickles written by +versions before 1.2.0 can still be loaded. + +```python +import pickle + +restored = pickle.loads(pickle.dumps(sample)) +print(restored == sample) +``` + +```text +True +``` + ## Nested objects and collections Module-level helpers serialize supported containers as well as individual diff --git a/serializable/dataclass_serializable.py b/serializable/dataclass_serializable.py index 9187a73..e6247b3 100644 --- a/serializable/dataclass_serializable.py +++ b/serializable/dataclass_serializable.py @@ -43,7 +43,7 @@ class Point(DataclassSerializable): from dataclasses import fields from typing import Any, ClassVar -from .helpers import from_json, from_serializable_repr, to_json, to_serializable_repr +from .helpers import from_json, to_json class DataclassSerializable: @@ -89,5 +89,7 @@ def from_json(cls, json_string: str): def __reduce__(self): """Pickle via the same to_dict / from_dict path used for JSON so pickled objects round-trip even when field order or internal - representation changes between releases.""" - return (from_serializable_repr, (to_serializable_repr(self),)) + representation changes between releases. Pickles written before + serializable 1.2.0 call from_serializable_repr instead, which remains + supported.""" + return (type(self).from_dict, (self.to_dict(),)) diff --git a/serializable/helpers.py b/serializable/helpers.py index 2d0939a..4c7a0be 100644 --- a/serializable/helpers.py +++ b/serializable/helpers.py @@ -15,36 +15,42 @@ objects into serializable types. """ +from functools import lru_cache from types import BuiltinFunctionType, FunctionType import simplejson as json -from .primitive_types import return_primitive +from .primitive_types import PRIMITIVE_TYPES def init_arg_names(obj): """ - Names of arguments to __init__ method of this object's class. + Names of the positional arguments to the __init__ method of this + object's class (excluding self). + + Keyword-only arguments are deliberately excluded, so a class can accept + constructor options which aren't part of its serialized state. """ + return _class_init_arg_names(type(obj)) + + +# Cached per class since this is called on every to_dict/__eq__/__hash__ of +# a Serializable which doesn't override to_dict. Assumes a class's __init__ +# isn't replaced after its first instance has been serialized. +@lru_cache(maxsize=1024) +def _class_init_arg_names(cls): # doing something wildly hacky by pulling out the arguments to - # __init__ or __new__ and hoping that they match fields defined on the - # object + # __init__ and hoping that they match fields defined on the object try: - init_code = obj.__init__.__func__.__code__ - except AttributeError: - try: - init_code = obj.__new__.__func__.__code__ - except AttributeError as exc: - # if object is a namedtuple then we can return its fields - # as the required initial args - if hasattr(obj, "_fields"): - return obj._fields - raise ValueError(f"Cannot determine args to {obj}.__init__") from exc - - arg_names = init_code.co_varnames[: init_code.co_argcount] + init_code = cls.__init__.__code__ + except AttributeError as exc: + # if object is a namedtuple then we can return its fields + # as the required initial args + if hasattr(cls, "_fields"): + return cls._fields + raise ValueError(f"Cannot determine args to {cls.__qualname__}.__init__") from exc # drop self argument - nonself_arg_names = arg_names[1:] - return nonself_arg_names + return init_code.co_varnames[1 : init_code.co_argcount] def simple_object_to_dict(self): @@ -185,10 +191,12 @@ def dict_to_serializable_repr(x): return result -def from_serializable_dict(x): +def _from_reconstructed_dict(x): """ - Reconstruct a dictionary by recursively reconstructing all its keys and - values. + Given a dictionary from a serializable representation whose values have + already been reconstructed, return the object it represents: a class or + function, an instance of a user-defined class, or a dictionary (restoring + any non-string keys). May modify `x`. This is the most hackish part since we rely on key names such as __name__, __class__, __module__ as metadata about how to reconstruct @@ -200,26 +208,38 @@ def from_serializable_dict(x): flattened result of to_dict() for user-defined objects. """ if "__name__" in x: - return _lookup_value(x.pop("__module__"), x.pop("__name__")) + return _lookup_value(x["__module__"], x["__name__"]) + + serialized_keys = x.pop(SERIALIZED_DICTIONARY_KEYS_FIELD, None) + if serialized_keys: + non_string_key_objects = [from_json(serialized_key) for serialized_key in serialized_keys] + converted_dict = type(x)() + for k, v in x.items(): + serialized_key_index = parse_serialized_keys_index(k) + if serialized_key_index is not None: + k = non_string_key_objects[serialized_key_index] + converted_dict[k] = v + x = converted_dict + + if "__class__" not in x: + return x + class_object = x.pop("__class__") + if "__value__" in x: + return class_object(x["__value__"]) + if hasattr(class_object, "from_dict"): + return class_object.from_dict(x) + return class_object(**x) + - non_string_key_objects = [ - from_json(serialized_key) for serialized_key in x.pop(SERIALIZED_DICTIONARY_KEYS_FIELD, []) - ] +def from_serializable_dict(x): + """ + Reconstruct a dictionary, or the object it represents, by recursively + reconstructing all its keys and values. Does not modify `x`. + """ converted_dict = type(x)() for k, v in x.items(): - serialized_key_index = parse_serialized_keys_index(k) - if serialized_key_index is not None: - k = non_string_key_objects[serialized_key_index] - converted_dict[k] = from_serializable_repr(v) - if "__class__" in converted_dict: - class_object = converted_dict.pop("__class__") - if "__value__" in converted_dict: - return class_object(converted_dict["__value__"]) - if hasattr(class_object, "from_dict"): - return class_object.from_dict(converted_dict) - return class_object(**converted_dict) - return converted_dict + return _from_reconstructed_dict(converted_dict) def list_to_serializable_repr(x): @@ -242,12 +262,13 @@ def to_dict(obj): raise ValueError(f"Cannot convert {obj} : {type(obj)} to dictionary") from exc -@return_primitive def to_serializable_repr(x): """ Convert an instance of Serializable or a primitive collection containing such instances into serializable types. """ + if isinstance(x, PRIMITIVE_TYPES): + return x t = type(x) if isinstance(x, list): return list_to_serializable_repr(x) @@ -267,8 +288,12 @@ def to_serializable_repr(x): return state_dictionary -@return_primitive def from_serializable_repr(x): + """ + Inverse of to_serializable_repr. Does not modify `x`. + """ + if isinstance(x, PRIMITIVE_TYPES): + return x t = type(x) if isinstance(x, list): return t([from_serializable_repr(element) for element in x]) @@ -286,4 +311,8 @@ def to_json(x): def from_json(json_string): - return from_serializable_repr(json.loads(json_string)) + """ + Inverse of to_json. Objects are reconstructed bottom-up while parsing, + equivalent to (but faster than) from_serializable_repr(json.loads(...)). + """ + return json.loads(json_string, object_hook=_from_reconstructed_dict) diff --git a/serializable/serializable.py b/serializable/serializable.py index 385d856..232bb9d 100644 --- a/serializable/serializable.py +++ b/serializable/serializable.py @@ -13,13 +13,7 @@ from typing import ClassVar -from .helpers import ( - from_json, - from_serializable_repr, - simple_object_to_dict, - to_json, - to_serializable_repr, -) +from .helpers import from_json, simple_object_to_dict, to_json class Serializable: @@ -48,8 +42,10 @@ def to_dict(self): (bool, string, int, float), primitive collections (int, list, tuple, set) or instances of Serializable. - The default implementation is to assume all the arguments to __init__ - have fields of the same name on a serializable object. + The default implementation is to assume all the positional arguments + to __init__ have fields of the same name on a serializable object. + Keyword-only arguments are excluded, so they can be used for + constructor options which aren't part of the serialized state. """ return simple_object_to_dict(self) @@ -90,9 +86,9 @@ def _update_kwargs(cls, kwargs): def from_dict(cls, state_dict): """ Given a dictionary of flattened fields (result of calling to_dict()), - returns an instance. + returns an instance. Does not modify `state_dict`. """ - state_dict = cls._reconstruct_nested_objects(state_dict) + state_dict = cls._reconstruct_nested_objects(dict(state_dict)) return cls(**cls._update_kwargs(state_dict)) def to_json(self): @@ -127,10 +123,11 @@ def read_json_file(cls, path): def __reduce__(self): """ Overriding this method directs the default pickler to reconstruct - this object using our from_dict method. - """ + this object using our from_dict method, so pickles survive the same + changes to a class (e.g. renamed keywords) as JSON does. Values in + to_dict() are pickled natively. - # I wish I could just return (self.from_dict, (self.to_dict(),) but - # Python 2 won't pickle the class method from_dict so instead have to - # use globally defined functions. - return (from_serializable_repr, (to_serializable_repr(self),)) + Pickles written before serializable 1.2.0 call from_serializable_repr + instead, which remains supported. + """ + return (type(self).from_dict, (self.to_dict(),)) diff --git a/serializable/version.py b/serializable/version.py index e52182c..858266d 100644 --- a/serializable/version.py +++ b/serializable/version.py @@ -1,4 +1,4 @@ -__version__ = "1.1.2" +__version__ = "1.2.0" def print_version(): diff --git a/tests/test_default_to_dict_implementation.py b/tests/test_default_to_dict_implementation.py index 8fa422b..d73f26b 100644 --- a/tests/test_default_to_dict_implementation.py +++ b/tests/test_default_to_dict_implementation.py @@ -25,3 +25,17 @@ def __init__(self, x, y=1): def test_serializable_default_to_dict(): a = A(10, 1) eq_(a, A.from_dict(a.to_dict())) + + +class WithKeywordOnlyOption(Serializable): + def __init__(self, x, *, scale=1): + # the keyword-only option isn't stored, so it can't be serialized + self.x = x * scale + + +def test_keyword_only_args_excluded_from_default_to_dict(): + # keyword-only args are constructor options rather than serialized state + # (varcode's SpliceOutcomeSet relies on this) + obj = WithKeywordOnlyOption(10, scale=2) + eq_(obj.to_dict(), {"x": 20}) + eq_(obj, WithKeywordOnlyOption.from_json(obj.to_json())) diff --git a/tests/test_keyword_aliases.py b/tests/test_keyword_aliases.py index 6bc1f8c..c67e7d7 100644 --- a/tests/test_keyword_aliases.py +++ b/tests/test_keyword_aliases.py @@ -96,3 +96,10 @@ def test_renamed_keyword_inheritance(): eq_(obj.y, 2) assert not hasattr(obj, "old_x") assert not hasattr(obj, "old_y") + + +def test_from_dict_does_not_modify_input(): + d = {"old_x": 1, "old_gone": 2} + obj = TestClassWithKeywordAliases.from_dict(d) + eq_(obj.x, 1) + eq_(d, {"old_x": 1, "old_gone": 2}) diff --git a/tests/test_nesting.py b/tests/test_nesting.py index ae2ed29..9dabe62 100644 --- a/tests/test_nesting.py +++ b/tests/test_nesting.py @@ -1,4 +1,13 @@ -from serializable import Serializable, from_serializable_repr, to_serializable_repr +import copy +import json + +from serializable import ( + Serializable, + from_json, + from_serializable_repr, + to_json, + to_serializable_repr, +) from .common import eq_ @@ -42,3 +51,27 @@ def test_object_with_dict_values(): def test_object_with_set_values(): x = B() eq_(x, from_serializable_repr(to_serializable_repr(x))) + + +nested = { + (1, 2): [A(1), B()], + "class": A, + "inner": {"t": (A({(3,): {4}}),), 5: "int key"}, +} + + +def test_from_serializable_repr_does_not_modify_input(): + serialized = to_serializable_repr(nested) + snapshot = copy.deepcopy(serialized) + eq_(nested, from_serializable_repr(serialized)) + eq_(serialized, snapshot) + # decoding the same representation twice used to fail + eq_(nested, from_serializable_repr(serialized)) + + +def test_from_json_matches_from_serializable_repr(): + # from_json reconstructs objects while parsing; it must agree with + # decoding the parsed JSON in a separate pass + json_string = to_json(nested) + eq_(nested, from_json(json_string)) + eq_(from_json(json_string), from_serializable_repr(json.loads(json_string))) diff --git a/tests/test_pickle.py b/tests/test_pickle.py new file mode 100644 index 0000000..2daae2d --- /dev/null +++ b/tests/test_pickle.py @@ -0,0 +1,102 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import datetime +import io +import pickle +from dataclasses import dataclass + +import pytest + +from serializable import ( + DataclassSerializable, + Serializable, + from_serializable_repr, + to_serializable_repr, +) + +from .common import eq_ + + +class Point(Serializable): + def __init__(self, x, y): + self.x = x + self.y = y + + +@dataclass(frozen=True) +class Pair(DataclassSerializable): + left: object + right: object + + +class Interned(Serializable): + """from_dict returns a cached instance, like pyensembl's EnsemblRelease""" + + _instances = {} + + def __init__(self, name): + self.name = name + + @classmethod + def from_dict(cls, state_dict): + name = state_dict["name"] + if name not in cls._instances: + cls._instances[name] = cls(name) + return cls._instances[name] + + +nested = Point([Point(1, 2), (3, 4)], {(5, 6): {7, 8}, "pair": Pair("a", Point(9, 10))}) + + +class Pre12Pickler(pickle.Pickler): + """Pickles objects the way serializable < 1.2.0 did.""" + + def reducer_override(self, obj): + if isinstance(obj, (Serializable, DataclassSerializable)): + return (from_serializable_repr, (to_serializable_repr(obj),)) + return NotImplemented + + +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +def test_pickle_roundtrip_all_protocols(protocol): + eq_(nested, pickle.loads(pickle.dumps(nested, protocol=protocol))) + pair = Pair(nested, "b") + eq_(pair, pickle.loads(pickle.dumps(pair, protocol=protocol))) + + +def test_load_pickle_written_before_1_2(): + objects = [nested, Pair(1, nested)] + buffer = io.BytesIO() + Pre12Pickler(buffer).dump(objects) + eq_(objects, pickle.loads(buffer.getvalue())) + + +def test_pickle_reconstructs_with_from_dict(): + obj = Interned.from_dict({"name": "a"}) + assert pickle.loads(pickle.dumps(obj)) is obj + + +def test_pickle_value_without_serializable_representation(): + # values in to_dict() are pickled natively, so they don't need a + # JSON-compatible representation + obj = Point(datetime.date(2026, 10, 6), None) + with pytest.raises(ValueError): + obj.to_json() + eq_(obj, pickle.loads(pickle.dumps(obj))) + + +def test_deepcopy(): + copied = copy.deepcopy(nested) + eq_(nested, copied) + assert copied.x is not nested.x