Skip to content

Commit 3383010

Browse files
author
thelastbackspace
committed
serialize: handle dicts whose keys can't be sorted
serialize_object/deserialize_object walked small dicts with sorted(), which raises TypeError for mixed key types like {'A': 1, 2: 2}. Fall back to insertion order when sorting fails, and walk the unpickled dict in its own order on the way back, which is the order the buffers were extracted in. Closes #1057
1 parent d724f99 commit 3383010

2 files changed

Lines changed: 33 additions & 2 deletions

File tree

‎ipyparallel/serialize/serialize.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,12 @@ def serialize_object(obj, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):
102102
buffers.extend(_extract_buffers(c, buffer_threshold))
103103
elif istype(obj, dict) and len(obj) < item_threshold:
104104
cobj = {}
105-
for k in sorted(obj):
105+
try:
106+
keys = sorted(obj)
107+
except TypeError:
108+
# keys that can't be ordered (e.g. mixed str and int)
109+
keys = list(obj)
110+
for k in keys:
106111
c = can(obj[k])
107112
buffers.extend(_extract_buffers(c, buffer_threshold))
108113
cobj[k] = c
@@ -135,7 +140,9 @@ def deserialize_object(buffers, g=None):
135140
newobj = uncan_sequence(canned, g)
136141
elif istype(canned, dict) and len(canned) < MAX_ITEMS:
137142
newobj = {}
138-
for k in sorted(canned):
143+
# the unpickled dict keeps the key order serialize_object built it in,
144+
# which is the order its buffers were extracted
145+
for k in canned:
139146
c = canned[k]
140147
_restore_buffers(c, bufs)
141148
newobj[k] = uncan(c, g)

‎ipyparallel/tests/test_serialize.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,30 @@ def test_numpy_in_dict():
148148
assert_array_equal(A, B)
149149

150150

151+
def test_dict_unsortable_keys():
152+
d = {'A': 1, 2: 2}
153+
bufs = serialize_object(d)
154+
d2, r = deserialize_object(bufs)
155+
assert r == []
156+
assert d2 == d
157+
158+
159+
def test_numpy_in_dict_unsortable_keys():
160+
pytest.importorskip('numpy')
161+
from numpy.testing import assert_array_equal
162+
163+
A = new_array((100, 100), dtype='float64')
164+
B = new_array((1000,), dtype='int32')
165+
d = {'b': B, 1: A, 'c': 3}
166+
bufs = serialize_object(d)
167+
assert len(bufs) == 3
168+
d2, r = deserialize_object(bufs)
169+
assert r == []
170+
assert d2['c'] == 3
171+
assert_array_equal(d2[1], A)
172+
assert_array_equal(d2['b'], B)
173+
174+
151175
def test_class():
152176
@interactive
153177
class C:

0 commit comments

Comments
 (0)