Faster pickling and from_json; stop mutating inputs; bump to 1.2.0

README.md

@@ -41,9 +41,13 @@ helpers.
 
 A derived class must either:
 
-- have a member data matching the name of each argument to `__init__`
+- have a member data matching the name of each positional argument to `__init__`
 - provide a user-defined `to_dict()` method which returns a dictionary whose keys match the arguments to `__init__`
 
+Keyword-only arguments to `__init__` (those after `*`) are left out of the default `to_dict()`, so they can be used for constructor options which aren't part of an object's serialized state.
+
+Pickling goes through the same `to_dict()` / `from_dict()` pair, so pickles also survive renamed keywords (see below). Pickles written by versions before 1.2.0 can still be loaded.
+
 If you change the keyword arguments to a class which derives from `Serializable` but would like to be able to deserialize older JSON representations then you can define a class-level dictionary called `_SERIALIZABLE_KEYWORD_ALIASES` which maps old keywords to new names (or `None` if a keyword was removed).
 
 ## Limitations

docs/guides/objects.md

@@ -3,8 +3,10 @@
 ## Constructor-based classes
 
 Inherit from `Serializable` when you already have a normal constructor. Store
-an attribute for each constructor argument, or implement `to_dict()` with keys
-that the constructor accepts.
+an attribute for each positional constructor argument, or implement `to_dict()`
+with keys that the constructor accepts. Keyword-only arguments (after `*`) are
+left out of the default `to_dict()`, so they can hold constructor options which
+aren't part of the saved state.
 
 ```python
 from serializable import Serializable
@@ -52,6 +54,23 @@ True
 For a dataclass, write `object.to_json()` with `Path.write_text()` and reload
 the text with `YourClass.from_json()`.
 
+## Pickle objects
+
+Pickling uses the same `to_dict()` and `from_dict()` methods as JSON, so
+[renamed fields](migration.md) are handled the same way. Pickles written by
+versions before 1.2.0 can still be loaded.
+
+```python
+import pickle
+
+restored = pickle.loads(pickle.dumps(sample))
+print(restored == sample)
+```
+
+```text
+True
+```
+
 ## Nested objects and collections
 
 Module-level helpers serialize supported containers as well as individual

serializable/dataclass_serializable.py

@@ -43,7 +43,7 @@ class Point(DataclassSerializable):
 from dataclasses import fields
 from typing import Any, ClassVar
 
-from .helpers import from_json, from_serializable_repr, to_json, to_serializable_repr
+from .helpers import from_json, to_json
 
 
 class DataclassSerializable:
@@ -89,5 +89,7 @@ def from_json(cls, json_string: str):
     def __reduce__(self):
         """Pickle via the same to_dict / from_dict path used for JSON so
         pickled objects round-trip even when field order or internal
-        representation changes between releases."""
-        return (from_serializable_repr, (to_serializable_repr(self),))
+        representation changes between releases. Pickles written before
+        serializable 1.2.0 call from_serializable_repr instead, which remains
+        supported."""
+        return (type(self).from_dict, (self.to_dict(),))

serializable/helpers.py

@@ -15,36 +15,42 @@
 objects into serializable types.
 """
 
+from functools import lru_cache
 from types import BuiltinFunctionType, FunctionType
 
 import simplejson as json
 
-from .primitive_types import return_primitive
+from .primitive_types import PRIMITIVE_TYPES
 
 
 def init_arg_names(obj):
     """
-    Names of arguments to __init__ method of this object's class.
+    Names of the positional arguments to the __init__ method of this
+    object's class (excluding self).
+
+    Keyword-only arguments are deliberately excluded, so a class can accept
+    constructor options which aren't part of its serialized state.
     """
+    return _class_init_arg_names(type(obj))
+
+
+# Cached per class since this is called on every to_dict/__eq__/__hash__ of
+# a Serializable which doesn't override to_dict. Assumes a class's __init__
+# isn't replaced after its first instance has been serialized.
+@lru_cache(maxsize=1024)
+def _class_init_arg_names(cls):
     # doing something wildly hacky by pulling out the arguments to
-    # __init__ or __new__ and hoping that they match fields defined on the
-    # object
+    # __init__ and hoping that they match fields defined on the object
     try:
-        init_code = obj.__init__.__func__.__code__
-    except AttributeError:
-        try:
-            init_code = obj.__new__.__func__.__code__
-        except AttributeError as exc:
-            # if object is a namedtuple then we can return its fields
-            # as the required initial args
-            if hasattr(obj, "_fields"):
-                return obj._fields
-            raise ValueError(f"Cannot determine args to {obj}.__init__") from exc
-
-    arg_names = init_code.co_varnames[: init_code.co_argcount]
+        init_code = cls.__init__.__code__
+    except AttributeError as exc:
+        # if object is a namedtuple then we can return its fields
+        # as the required initial args
+        if hasattr(cls, "_fields"):
+            return cls._fields
+        raise ValueError(f"Cannot determine args to {cls.__qualname__}.__init__") from exc
     # drop self argument
-    nonself_arg_names = arg_names[1:]
-    return nonself_arg_names
+    return init_code.co_varnames[1 : init_code.co_argcount]
 
 
 def simple_object_to_dict(self):
@@ -185,10 +191,12 @@ def dict_to_serializable_repr(x):
     return result
 
 
-def from_serializable_dict(x):
+def _from_reconstructed_dict(x):
     """
-    Reconstruct a dictionary by recursively reconstructing all its keys and
-    values.
+    Given a dictionary from a serializable representation whose values have
+    already been reconstructed, return the object it represents: a class or
+    function, an instance of a user-defined class, or a dictionary (restoring
+    any non-string keys). May modify `x`.
 
     This is the most hackish part since we rely on key names such as
     __name__, __class__, __module__ as metadata about how to reconstruct
@@ -200,26 +208,38 @@ def from_serializable_dict(x):
         flattened result of to_dict() for user-defined objects.
     """
     if "__name__" in x:
-        return _lookup_value(x.pop("__module__"), x.pop("__name__"))
+        return _lookup_value(x["__module__"], x["__name__"])
+
+    serialized_keys = x.pop(SERIALIZED_DICTIONARY_KEYS_FIELD, None)
+    if serialized_keys:
+        non_string_key_objects = [from_json(serialized_key) for serialized_key in serialized_keys]
+        converted_dict = type(x)()
+        for k, v in x.items():
+            serialized_key_index = parse_serialized_keys_index(k)
+            if serialized_key_index is not None:
+                k = non_string_key_objects[serialized_key_index]
+            converted_dict[k] = v
+        x = converted_dict
+
+    if "__class__" not in x:
+        return x
+    class_object = x.pop("__class__")
+    if "__value__" in x:
+        return class_object(x["__value__"])
+    if hasattr(class_object, "from_dict"):
+        return class_object.from_dict(x)
+    return class_object(**x)
+
 
-    non_string_key_objects = [
-        from_json(serialized_key) for serialized_key in x.pop(SERIALIZED_DICTIONARY_KEYS_FIELD, [])
-    ]
+def from_serializable_dict(x):
+    """
+    Reconstruct a dictionary, or the object it represents, by recursively
+    reconstructing all its keys and values. Does not modify `x`.
+    """
     converted_dict = type(x)()
     for k, v in x.items():
-        serialized_key_index = parse_serialized_keys_index(k)
-        if serialized_key_index is not None:
-            k = non_string_key_objects[serialized_key_index]
-
         converted_dict[k] = from_serializable_repr(v)
-    if "__class__" in converted_dict:
-        class_object = converted_dict.pop("__class__")
-        if "__value__" in converted_dict:
-            return class_object(converted_dict["__value__"])
-        if hasattr(class_object, "from_dict"):
-            return class_object.from_dict(converted_dict)
-        return class_object(**converted_dict)
-    return converted_dict
+    return _from_reconstructed_dict(converted_dict)
 
 
 def list_to_serializable_repr(x):
@@ -242,12 +262,13 @@ def to_dict(obj):
         raise ValueError(f"Cannot convert {obj} : {type(obj)} to dictionary") from exc
 
 
-@return_primitive
 def to_serializable_repr(x):
     """
     Convert an instance of Serializable or a primitive collection containing
     such instances into serializable types.
     """
+    if isinstance(x, PRIMITIVE_TYPES):
+        return x
     t = type(x)
     if isinstance(x, list):
         return list_to_serializable_repr(x)
@@ -267,8 +288,12 @@ def to_serializable_repr(x):
     return state_dictionary
 
 
-@return_primitive
 def from_serializable_repr(x):
+    """
+    Inverse of to_serializable_repr. Does not modify `x`.
+    """
+    if isinstance(x, PRIMITIVE_TYPES):
+        return x
     t = type(x)
     if isinstance(x, list):
         return t([from_serializable_repr(element) for element in x])
@@ -286,4 +311,8 @@ def to_json(x):
 
 
 def from_json(json_string):
-    return from_serializable_repr(json.loads(json_string))
+    """
+    Inverse of to_json. Objects are reconstructed bottom-up while parsing,
+    equivalent to (but faster than) from_serializable_repr(json.loads(...)).
+    """
+    return json.loads(json_string, object_hook=_from_reconstructed_dict)

serializable/serializable.py

@@ -13,13 +13,7 @@
 
 from typing import ClassVar
 
-from .helpers import (
-    from_json,
-    from_serializable_repr,
-    simple_object_to_dict,
-    to_json,
-    to_serializable_repr,
-)
+from .helpers import from_json, simple_object_to_dict, to_json
 
 
 class Serializable:
@@ -48,8 +42,10 @@ def to_dict(self):
         (bool, string, int, float), primitive collections
         (int, list, tuple, set) or instances of Serializable.
 
-        The default implementation is to assume all the arguments to __init__
-        have fields of the same name on a serializable object.
+        The default implementation is to assume all the positional arguments
+        to __init__ have fields of the same name on a serializable object.
+        Keyword-only arguments are excluded, so they can be used for
+        constructor options which aren't part of the serialized state.
         """
         return simple_object_to_dict(self)
 
@@ -90,9 +86,9 @@ def _update_kwargs(cls, kwargs):
     def from_dict(cls, state_dict):
         """
         Given a dictionary of flattened fields (result of calling to_dict()),
-        returns an instance.
+        returns an instance. Does not modify `state_dict`.
         """
-        state_dict = cls._reconstruct_nested_objects(state_dict)
+        state_dict = cls._reconstruct_nested_objects(dict(state_dict))
         return cls(**cls._update_kwargs(state_dict))
 
     def to_json(self):
@@ -127,10 +123,11 @@ def read_json_file(cls, path):
     def __reduce__(self):
         """
         Overriding this method directs the default pickler to reconstruct
-        this object using our from_dict method.
-        """
+        this object using our from_dict method, so pickles survive the same
+        changes to a class (e.g. renamed keywords) as JSON does. Values in
+        to_dict() are pickled natively.
 
-        # I wish I could just return (self.from_dict, (self.to_dict(),) but
-        # Python 2 won't pickle the class method from_dict so instead have to
-        # use globally defined functions.
-        return (from_serializable_repr, (to_serializable_repr(self),))
+        Pickles written before serializable 1.2.0 call from_serializable_repr
+        instead, which remains supported.
+        """
+        return (type(self).from_dict, (self.to_dict(),))

serializable/version.py

@@ -1,4 +1,4 @@
-__version__ = "1.1.2"
+__version__ = "1.2.0"
 
 
 def print_version():

tests/test_default_to_dict_implementation.py

@@ -25,3 +25,17 @@ def __init__(self, x, y=1):
 def test_serializable_default_to_dict():
     a = A(10, 1)
     eq_(a, A.from_dict(a.to_dict()))
+
+
+class WithKeywordOnlyOption(Serializable):
+    def __init__(self, x, *, scale=1):
+        # the keyword-only option isn't stored, so it can't be serialized
+        self.x = x * scale
+
+
+def test_keyword_only_args_excluded_from_default_to_dict():
+    # keyword-only args are constructor options rather than serialized state
+    # (varcode's SpliceOutcomeSet relies on this)
+    obj = WithKeywordOnlyOption(10, scale=2)
+    eq_(obj.to_dict(), {"x": 20})
+    eq_(obj, WithKeywordOnlyOption.from_json(obj.to_json()))

tests/test_keyword_aliases.py

@@ -96,3 +96,10 @@ def test_renamed_keyword_inheritance():
     eq_(obj.y, 2)
     assert not hasattr(obj, "old_x")
     assert not hasattr(obj, "old_y")
+
+
+def test_from_dict_does_not_modify_input():
+    d = {"old_x": 1, "old_gone": 2}
+    obj = TestClassWithKeywordAliases.from_dict(d)
+    eq_(obj.x, 1)
+    eq_(d, {"old_x": 1, "old_gone": 2})

tests/test_nesting.py

@@ -1,4 +1,13 @@
-from serializable import Serializable, from_serializable_repr, to_serializable_repr
+import copy
+import json
+
+from serializable import (
+    Serializable,
+    from_json,
+    from_serializable_repr,
+    to_json,
+    to_serializable_repr,
+)
 
 from .common import eq_
 
@@ -42,3 +51,27 @@ def test_object_with_dict_values():
 def test_object_with_set_values():
     x = B()
     eq_(x, from_serializable_repr(to_serializable_repr(x)))
+
+
+nested = {
+    (1, 2): [A(1), B()],
+    "class": A,
+    "inner": {"t": (A({(3,): {4}}),), 5: "int key"},
+}
+
+
+def test_from_serializable_repr_does_not_modify_input():
+    serialized = to_serializable_repr(nested)
+    snapshot = copy.deepcopy(serialized)
+    eq_(nested, from_serializable_repr(serialized))
+    eq_(serialized, snapshot)
+    # decoding the same representation twice used to fail
+    eq_(nested, from_serializable_repr(serialized))
+
+
+def test_from_json_matches_from_serializable_repr():
+    # from_json reconstructs objects while parsing; it must agree with
+    # decoding the parsed JSON in a separate pass
+    json_string = to_json(nested)
+    eq_(nested, from_json(json_string))
+    eq_(from_json(json_string), from_serializable_repr(json.loads(json_string)))