diff -Nru cloudpickle-0.2.2/cloudpickle/cloudpickle.py cloudpickle-0.4.0/cloudpickle/cloudpickle.py --- cloudpickle-0.2.2/cloudpickle/cloudpickle.py 2016-12-31 23:47:36.000000000 +0000 +++ cloudpickle-0.4.0/cloudpickle/cloudpickle.py 2017-08-07 18:03:58.000000000 +0000 @@ -47,6 +47,7 @@ import imp import io import itertools +import logging import opcode import operator import pickle @@ -56,6 +57,7 @@ import types import weakref + if sys.version < '3': from pickle import Pickler try: @@ -69,6 +71,92 @@ from io import BytesIO as StringIO PY3 = True + +def _make_cell_set_template_code(): + """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF + + Notes + ----- + In Python 3, we could use an easier function: + + .. code-block:: python + + def f(): + cell = None + + def _stub(value): + nonlocal cell + cell = value + + return _stub + + _cell_set_template_code = f() + + This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is + invalid syntax on Python 2. If we use this function we also don't need + to do the weird freevars/cellvars swap below + """ + def inner(value): + lambda: cell # make ``cell`` a closure so that we get a STORE_DEREF + cell = value + + co = inner.__code__ + + # NOTE: we are marking the cell variable as a free variable intentionally + # so that we simulate an inner function instead of the outer function. This + # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way. + if not PY3: + return types.CodeType( + co.co_argcount, + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # this is the trickery + (), + ) + else: + return types.CodeType( + co.co_argcount, + co.co_kwonlyargcount, + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # this is the trickery + (), + ) + + +_cell_set_template_code = _make_cell_set_template_code() + + +def cell_set(cell, value): + """Set the value of a closure cell. + """ + return types.FunctionType( + _cell_set_template_code, + {}, + '_cell_set_inner', + (), + (cell,), + )(value) + + #relevant opcodes STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] @@ -176,11 +264,14 @@ """ mod_name = obj.__name__ # If module is successfully found then it is not a dynamically created module - try: - _find_module(mod_name) + if hasattr(obj, '__file__'): is_dynamic = False - except ImportError: - is_dynamic = True + else: + try: + _find_module(mod_name) + is_dynamic = False + except ImportError: + is_dynamic = True self.modules.add(obj) if is_dynamic: @@ -219,7 +310,12 @@ if name is None: name = obj.__name__ - modname = pickle.whichmodule(obj, name) + try: + # whichmodule() could fail, see + # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling + modname = pickle.whichmodule(obj, name) + except Exception: + modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] @@ -238,7 +334,7 @@ # a builtin_function_or_method which comes in as an attribute of some # object (e.g., object.__new__, itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions - # have no __code__ attribute in CPython, so the handling for + # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. @@ -282,6 +378,92 @@ self.memoize(obj) dispatch[types.FunctionType] = save_function + def _save_subimports(self, code, top_level_dependencies): + """ + Ensure de-pickler imports any package child-modules that + are needed by the function + """ + # check if any known dependency is an imported package + for x in top_level_dependencies: + if isinstance(x, types.ModuleType) and x.__package__: + # check if the package has any currently loaded sub-imports + prefix = x.__name__ + '.' + for name, module in sys.modules.items(): + # Older versions of pytest will add a "None" module to sys.modules. + if name is not None and name.startswith(prefix): + # check whether the function can address the sub-module + tokens = set(name[len(prefix):].split('.')) + if not tokens - set(code.co_names): + # ensure unpickler executes this import + self.save(module) + # then discards the reference to it + self.write(pickle.POP) + + def save_dynamic_class(self, obj): + """ + Save a class that can't be stored as module global. + + This method is used to serialize classes that are defined inside + functions, or that otherwise can't be serialized as attribute lookups + from global modules. + """ + clsdict = dict(obj.__dict__) # copy dict proxy to a dict + if not isinstance(clsdict.get('__dict__', None), property): + # don't extract dict that are properties + clsdict.pop('__dict__', None) + clsdict.pop('__weakref__', None) + + # hack as __new__ is stored differently in the __dict__ + new_override = clsdict.get('__new__', None) + if new_override: + clsdict['__new__'] = obj.__new__ + + save = self.save + write = self.write + + # We write pickle instructions explicitly here to handle the + # possibility that the type object participates in a cycle with its own + # __dict__. We first write an empty "skeleton" version of the class and + # memoize it before writing the class' __dict__ itself. We then write + # instructions to "rehydrate" the skeleton class by restoring the + # attributes from the __dict__. + # + # A type can appear in a cycle with its __dict__ if an instance of the + # type appears in the type's __dict__ (which happens for the stdlib + # Enum class), or if the type defines methods that close over the name + # of the type, (which is common for Python 2-style super() calls). + + # Push the rehydration function. + save(_rehydrate_skeleton_class) + + # Mark the start of the args for the rehydration function. + write(pickle.MARK) + + # On PyPy, __doc__ is a readonly attribute, so we need to include it in + # the initial skeleton class. This is safe because we know that the + # doc can't participate in a cycle with the original class. + doc_dict = {'__doc__': clsdict.pop('__doc__', None)} + + # Create and memoize an empty class with obj's name and bases. + save(type(obj)) + save(( + obj.__name__, + obj.__bases__, + doc_dict, + )) + write(pickle.REDUCE) + self.memoize(obj) + + # Now save the rest of obj's __dict__. Any references to obj + # encountered while saving will point to the skeleton class. + save(clsdict) + + # Write a tuple of (skeleton_class, clsdict). + write(pickle.TUPLE) + + # Call _rehydrate_skeleton_class(skeleton_class, clsdict) + write(pickle.REDUCE) + def save_function_tuple(self, func): """ Pickles an actual func object. @@ -302,14 +484,23 @@ save = self.save write = self.write - code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func) + code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func) save(_fill_function) # skeleton function updater write(pickle.MARK) # beginning of tuple that _fill_function expects + self._save_subimports( + code, + itertools.chain(f_globals.values(), closure_values or ()), + ) + # create a skeleton function object and memoize it save(_make_skel_func) - save((code, closure, base_globals)) + save(( + code, + len(closure_values) if closure_values is not None else -1, + base_globals, + )) write(pickle.REDUCE) self.memoize(func) @@ -317,6 +508,7 @@ save(f_globals) save(defaults) save(dct) + save(closure_values) write(pickle.TUPLE) write(pickle.REDUCE) # applies _fill_function on the tuple @@ -354,7 +546,7 @@ def extract_func_data(self, func): """ Turn the function into a tuple of data necessary to recreate it: - code, globals, defaults, closure, dict + code, globals, defaults, closure_values, dict """ code = func.__code__ @@ -371,7 +563,11 @@ defaults = func.__defaults__ # process closure - closure = [c.cell_contents for c in func.__closure__] if func.__closure__ else [] + closure = ( + list(map(_get_cell_contents, func.__closure__)) + if func.__closure__ is not None + else None + ) # save the dict dct = func.__dict__ @@ -388,6 +584,12 @@ dispatch[types.BuiltinFunctionType] = save_builtin_function def save_global(self, obj, name=None, pack=struct.pack): + """ + Save a "global". + + The name of this method is somewhat misleading: all types get + dispatched here. + """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) @@ -397,7 +599,12 @@ modname = getattr(obj, "__module__", None) if modname is None: - modname = pickle.whichmodule(obj, name) + try: + # whichmodule() could fail, see + # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling + modname = pickle.whichmodule(obj, name) + except Exception: + modname = '__main__' if modname == '__main__': themodule = None @@ -411,18 +618,7 @@ typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): - d = dict(obj.__dict__) # copy dict proxy to a dict - if not isinstance(d.get('__dict__', None), property): - # don't extract dict that are properties - d.pop('__dict__', None) - d.pop('__weakref__', None) - - # hack as __new__ is stored differently in the __dict__ - new_override = d.get('__new__', None) - if new_override: - d['__new__'] = obj.__new__ - - self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj) + self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj) @@ -442,10 +638,15 @@ dispatch[types.MethodType] = save_instancemethod def save_inst(self, obj): - """Inner logic to save instance. Based off pickle.save_inst - Supports __transient__""" + """Inner logic to save instance. Based off pickle.save_inst""" cls = obj.__class__ + # Try the dispatch table (pickle module doesn't do it) + f = self.dispatch.get(cls) + if f: + f(self, obj) # Call unbound method with explicit self + return + memo = self.memo write = self.write save = self.save @@ -475,13 +676,6 @@ getstate = obj.__getstate__ except AttributeError: stuff = obj.__dict__ - #remove items if transient - if hasattr(obj, '__transient__'): - transient = obj.__transient__ - stuff = stuff.copy() - for k in list(stuff.keys()): - if k in transient: - del stuff[k] else: stuff = getstate() pickle._keep_alive(stuff, memo) @@ -544,8 +738,6 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, obj=None): - """Modified to support __transient__ on new objects - Change only affects protocol level 2 (which is always used by PiCloud""" # Assert that args is a tuple or None if not isinstance(args, tuple): raise pickle.PicklingError("args from reduce() should be a tuple") @@ -559,7 +751,6 @@ # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__": - #Added fix to allow transient cls = args[0] if not hasattr(cls, "__new__"): raise pickle.PicklingError( @@ -570,15 +761,6 @@ args = args[1:] save(cls) - #Don't pickle transient entries - if hasattr(obj, '__transient__'): - transient = obj.__transient__ - state = state.copy() - - for k in list(state.keys()): - if k in transient: - del state[k] - save(args) write(pickle.NEWOBJ) else: @@ -667,11 +849,23 @@ dispatch[type(Ellipsis)] = save_ellipsis dispatch[type(NotImplemented)] = save_not_implemented + # WeakSet was added in 2.7. + if hasattr(weakref, 'WeakSet'): + def save_weakset(self, obj): + self.save_reduce(weakref.WeakSet, (list(obj),)) + + dispatch[weakref.WeakSet] = save_weakset + """Special functions for Add-on libraries""" def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass + def save_logger(self, obj): + self.save_reduce(logging.getLogger, (obj.name,), obj=obj) + + dispatch[logging.Logger] = save_logger + # Tornado support @@ -773,38 +967,91 @@ def _gen_not_implemented(): return NotImplemented -def _fill_function(func, globals, defaults, dict): + +def _get_cell_contents(cell): + try: + return cell.cell_contents + except ValueError: + # sentinel used by ``_fill_function`` which will leave the cell empty + return _empty_cell_value + + +def instance(cls): + """Create a new instance of a class. + + Parameters + ---------- + cls : type + The class to create an instance of. + + Returns + ------- + instance : cls + A new instance of ``cls``. + """ + return cls() + + +@instance +class _empty_cell_value(object): + """sentinel for empty closures + """ + @classmethod + def __reduce__(cls): + return cls.__name__ + + +def _fill_function(func, globals, defaults, dict, closure_values): """ Fills in the rest of function data into the skeleton function object that were created via _make_skel_func(). - """ + """ func.__globals__.update(globals) func.__defaults__ = defaults func.__dict__ = dict - return func + cells = func.__closure__ + if cells is not None: + for cell, value in zip(cells, closure_values): + if value is not _empty_cell_value: + cell_set(cell, value) + return func -def _make_cell(value): - return (lambda: value).__closure__[0] +def _make_empty_cell(): + if False: + # trick the compiler into creating an empty cell in our lambda + cell = None + raise AssertionError('this route should not be executed') -def _reconstruct_closure(values): - return tuple([_make_cell(v) for v in values]) + return (lambda: cell).__closure__[0] -def _make_skel_func(code, closures, base_globals = None): +def _make_skel_func(code, cell_count, base_globals=None): """ Creates a skeleton function object that contains just the provided code and the correct number of cells in func_closure. All other func attributes (e.g. func_globals) are empty. """ - closure = _reconstruct_closure(closures) if closures else None - if base_globals is None: base_globals = {} base_globals['__builtins__'] = __builtins__ - return types.FunctionType(code, base_globals, - None, None, closure) + closure = ( + tuple(_make_empty_cell() for _ in range(cell_count)) + if cell_count >= 0 else + None + ) + return types.FunctionType(code, base_globals, None, None, closure) + + +def _rehydrate_skeleton_class(skeleton_class, class_dict): + """Put attributes from `class_dict` back on `skeleton_class`. + + See CloudPickler.save_dynamic_class for more info. + """ + for attrname, attr in class_dict.items(): + setattr(skeleton_class, attrname, attr) + return skeleton_class def _find_module(mod_name): @@ -817,7 +1064,9 @@ if path is not None: path = [path] file, path, description = imp.find_module(part, path) - return file, path, description + if file is not None: + file.close() + return path, description """Constructors for 3rd party libraries Note: These can never be renamed due to client compatibility issues""" diff -Nru cloudpickle-0.2.2/cloudpickle/__init__.py cloudpickle-0.4.0/cloudpickle/__init__.py --- cloudpickle-0.2.2/cloudpickle/__init__.py 2017-01-02 00:17:21.000000000 +0000 +++ cloudpickle-0.4.0/cloudpickle/__init__.py 2017-08-09 22:45:10.000000000 +0000 @@ -2,4 +2,4 @@ from cloudpickle.cloudpickle import * -__version__ = '0.2.2' +__version__ = '0.4.0' diff -Nru cloudpickle-0.2.2/cloudpickle.egg-info/PKG-INFO cloudpickle-0.4.0/cloudpickle.egg-info/PKG-INFO --- cloudpickle-0.2.2/cloudpickle.egg-info/PKG-INFO 2017-01-02 00:20:05.000000000 +0000 +++ cloudpickle-0.4.0/cloudpickle.egg-info/PKG-INFO 2017-08-09 22:50:46.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cloudpickle -Version: 0.2.2 +Version: 0.4.0 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe diff -Nru cloudpickle-0.2.2/debian/changelog cloudpickle-0.4.0/debian/changelog --- cloudpickle-0.2.2/debian/changelog 2017-04-22 00:12:15.000000000 +0000 +++ cloudpickle-0.4.0/debian/changelog 2017-10-13 15:24:04.000000000 +0000 @@ -1,8 +1,18 @@ -cloudpickle (0.2.2-1) unstable; urgency=medium +cloudpickle (0.4.0-1) unstable; urgency=medium - * New upstream release + * New upstream release. + * Update standards version to 4.1.1. No changes needed. + * Remove unnecessary Testsuite header. - -- Diane Trout Fri, 21 Apr 2017 17:12:15 -0700 + -- Diane Trout Fri, 13 Oct 2017 08:24:04 -0700 + +cloudpickle (0.3.1-1) unstable; urgency=medium + + * remove unnecessary debian/files + * Switch from git-dpm to gbp + * New upstream version 0.3.1 + + -- Diane Trout Thu, 01 Jun 2017 20:53:11 -0700 cloudpickle (0.2.1-1) unstable; urgency=low diff -Nru cloudpickle-0.2.2/debian/control cloudpickle-0.4.0/debian/control --- cloudpickle-0.2.2/debian/control 2017-04-22 00:12:15.000000000 +0000 +++ cloudpickle-0.4.0/debian/control 2017-10-13 15:24:04.000000000 +0000 @@ -10,11 +10,10 @@ python3-pytest, python3-pytest-cov, python3-setuptools -Standards-Version: 3.9.8 +Standards-Version: 4.1.1 Homepage: https://github.com/cloudpipe/cloudpickle Vcs-Git: https://anonscm.debian.org/git/python-modules/packages/cloudpickle.git Vcs-Browser: https://anonscm.debian.org/git/python-modules/packages/cloudpickle.git -Testsuite: autopkgtest-pkg-python X-Python-Version: >= 2.6 X-Python3-Version: >= 3.3 diff -Nru cloudpickle-0.2.2/debian/gbp.conf cloudpickle-0.4.0/debian/gbp.conf --- cloudpickle-0.2.2/debian/gbp.conf 1970-01-01 00:00:00.000000000 +0000 +++ cloudpickle-0.4.0/debian/gbp.conf 2017-06-02 03:43:06.000000000 +0000 @@ -0,0 +1,6 @@ +[DEFAULT] +upstream-branch = upstream +debian-branch = master +upstream-tag = upstream/%(version)s +debian-tag = debian/%(version)s +sign-tags = True diff -Nru cloudpickle-0.2.2/debian/.git-dpm cloudpickle-0.4.0/debian/.git-dpm --- cloudpickle-0.2.2/debian/.git-dpm 2017-04-22 00:12:15.000000000 +0000 +++ cloudpickle-0.4.0/debian/.git-dpm 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -# see git-dpm(1) from git-dpm package -63b00b2a864c5d6dcea5e3750a2d8e6914a81c47 -63b00b2a864c5d6dcea5e3750a2d8e6914a81c47 -63b00b2a864c5d6dcea5e3750a2d8e6914a81c47 -63b00b2a864c5d6dcea5e3750a2d8e6914a81c47 -cloudpickle_0.2.2.orig.tar.gz -c55d53d452b3c51bd67448a12b5e0297c1c1a1d7 -17020 -debianTag="debian/%e%v" -patchedTag="patched/%e%v" -upstreamTag="upstream/%e%u" diff -Nru cloudpickle-0.2.2/PKG-INFO cloudpickle-0.4.0/PKG-INFO --- cloudpickle-0.2.2/PKG-INFO 2017-01-02 00:20:05.000000000 +0000 +++ cloudpickle-0.4.0/PKG-INFO 2017-08-09 22:50:46.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cloudpickle -Version: 0.2.2 +Version: 0.4.0 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe diff -Nru cloudpickle-0.2.2/setup.py cloudpickle-0.4.0/setup.py --- cloudpickle-0.2.2/setup.py 2017-01-02 00:17:25.000000000 +0000 +++ cloudpickle-0.4.0/setup.py 2017-08-09 22:44:58.000000000 +0000 @@ -8,7 +8,7 @@ dist = setup( name='cloudpickle', - version='0.2.2', + version='0.4.0', description='Extended pickling support for Python objects', author='Cloudpipe', author_email='cloudpipe@googlegroups.com', diff -Nru cloudpickle-0.2.2/tests/cloudpickle_file_test.py cloudpickle-0.4.0/tests/cloudpickle_file_test.py --- cloudpickle-0.2.2/tests/cloudpickle_file_test.py 2016-12-31 23:47:36.000000000 +0000 +++ cloudpickle-0.4.0/tests/cloudpickle_file_test.py 2017-05-30 19:03:43.000000000 +0000 @@ -32,7 +32,7 @@ # Empty file open(self.tmpfilepath, 'w').close() with open(self.tmpfilepath, 'r') as f: - self.assertEquals('', pickle.loads(cloudpickle.dumps(f)).read()) + self.assertEqual('', pickle.loads(cloudpickle.dumps(f)).read()) os.remove(self.tmpfilepath) def test_closed_file(self): @@ -51,7 +51,7 @@ # Open for reading with open(self.tmpfilepath, 'r') as f: new_f = pickle.loads(cloudpickle.dumps(f)) - self.assertEquals(self.teststring, new_f.read()) + self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) def test_w_mode(self): @@ -68,7 +68,7 @@ f.write(self.teststring) f.seek(0) new_f = pickle.loads(cloudpickle.dumps(f)) - self.assertEquals(self.teststring, new_f.read()) + self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) def test_seek(self): @@ -78,11 +78,11 @@ f.seek(4) unpickled = pickle.loads(cloudpickle.dumps(f)) # unpickled StringIO is at position 4 - self.assertEquals(4, unpickled.tell()) - self.assertEquals(self.teststring[4:], unpickled.read()) + self.assertEqual(4, unpickled.tell()) + self.assertEqual(self.teststring[4:], unpickled.read()) # but unpickled StringIO also contained the start unpickled.seek(0) - self.assertEquals(self.teststring, unpickled.read()) + self.assertEqual(self.teststring, unpickled.read()) os.remove(self.tmpfilepath) @pytest.mark.skipif(sys.version_info >= (3,), @@ -94,12 +94,12 @@ f = fp.file # FIXME this doesn't work yet: cloudpickle.dumps(fp) newfile = pickle.loads(cloudpickle.dumps(f)) - self.assertEquals(self.teststring, newfile.read()) + self.assertEqual(self.teststring, newfile.read()) def test_pickling_special_file_handles(self): # Warning: if you want to run your tests with nose, add -s option for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415 - self.assertEquals(out, pickle.loads(cloudpickle.dumps(out))) + self.assertEqual(out, pickle.loads(cloudpickle.dumps(out))) self.assertRaises(pickle.PicklingError, lambda: cloudpickle.dumps(sys.stdin)) diff -Nru cloudpickle-0.2.2/tests/cloudpickle_test.py cloudpickle-0.4.0/tests/cloudpickle_test.py --- cloudpickle-0.2.2/tests/cloudpickle_test.py 2016-12-31 23:47:36.000000000 +0000 +++ cloudpickle-0.4.0/tests/cloudpickle_test.py 2017-08-07 18:03:58.000000000 +0000 @@ -1,14 +1,29 @@ from __future__ import division -import imp -import unittest -import pytest -import pickle -import sys -import random + +import abc + +import base64 import functools +import imp +from io import BytesIO import itertools +import logging +from operator import itemgetter, attrgetter +import pickle import platform +import random +import subprocess +import sys import textwrap +import unittest +import weakref + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +import pytest try: # try importing numpy and scipy. These are not hard dependencies and @@ -25,22 +40,15 @@ except ImportError: tornado = None - -from operator import itemgetter, attrgetter - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from io import BytesIO - import cloudpickle -from cloudpickle.cloudpickle import _find_module +from cloudpickle.cloudpickle import _find_module, _make_empty_cell, cell_set from .testutils import subprocess_pickle_echo +HAVE_WEAKSET = hasattr(weakref, 'WeakSet') + + def pickle_depickle(obj): """Helper function to test whether object pickled with cloudpickle can be depickled with pickle @@ -91,7 +99,7 @@ def test_pickling_file_handles(self): out1 = sys.stderr out2 = pickle.loads(cloudpickle.dumps(out1)) - self.assertEquals(out1, out2) + self.assertEqual(out1, out2) def test_func_globals(self): class Unpicklable(object): @@ -131,6 +139,115 @@ f2 = lambda x: f1(x) // b self.assertEqual(pickle_depickle(f2)(1), 1) + def test_recursive_closure(self): + def f1(): + def g(): + return g + return g + + def f2(base): + def g(n): + return base if n <= 1 else n * g(n - 1) + return g + + g1 = pickle_depickle(f1()) + self.assertEqual(g1(), g1) + + g2 = pickle_depickle(f2(2)) + self.assertEqual(g2(5), 240) + + def test_closure_none_is_preserved(self): + def f(): + """a function with no closure cells + """ + + self.assertTrue( + f.__closure__ is None, + msg='f actually has closure cells!', + ) + + g = pickle_depickle(f) + + self.assertTrue( + g.__closure__ is None, + msg='g now has closure cells even though f does not', + ) + + def test_empty_cell_preserved(self): + def f(): + if False: # pragma: no cover + cell = None + + def g(): + cell # NameError, unbound free variable + + return g + + g1 = f() + with pytest.raises(NameError): + g1() + + g2 = pickle_depickle(g1) + with pytest.raises(NameError): + g2() + + def test_unhashable_closure(self): + def f(): + s = set((1, 2)) # mutable set is unhashable + + def g(): + return len(s) + + return g + + g = pickle_depickle(f()) + self.assertEqual(g(), 2) + + def test_dynamically_generated_class_that_uses_super(self): + + class Base(object): + def method(self): + return 1 + + class Derived(Base): + "Derived Docstring" + def method(self): + return super(Derived, self).method() + 1 + + self.assertEqual(Derived().method(), 2) + + # Pickle and unpickle the class. + UnpickledDerived = pickle_depickle(Derived) + self.assertEqual(UnpickledDerived().method(), 2) + + # We have special logic for handling __doc__ because it's a readonly + # attribute on PyPy. + self.assertEqual(UnpickledDerived.__doc__, "Derived Docstring") + + # Pickle and unpickle an instance. + orig_d = Derived() + d = pickle_depickle(orig_d) + self.assertEqual(d.method(), 2) + + def test_cycle_in_classdict_globals(self): + + class C(object): + + def it_works(self): + return "woohoo!" + + C.C_again = C + C.instance_of_C = C() + + depickled_C = pickle_depickle(C) + depickled_instance = pickle_depickle(C()) + + # Test instance of depickled class. + self.assertEqual(depickled_C().it_works(), "woohoo!") + self.assertEqual(depickled_C.C_again().it_works(), "woohoo!") + self.assertEqual(depickled_C.instance_of_C.it_works(), "woohoo!") + self.assertEqual(depickled_instance.it_works(), "woohoo!") + @pytest.mark.skipif(sys.version_info >= (3, 4) and sys.version_info < (3, 4, 3), reason="subprocess has a bug in 3.4.0 to 3.4.2") @@ -360,6 +477,209 @@ self.assertTrue(f2 is f3) self.assertEqual(f2(), res) + def test_submodule(self): + # Function that refers (by attribute) to a sub-module of a package. + + # Choose any module NOT imported by __init__ of its parent package + # examples in standard library include: + # - http.cookies, unittest.mock, curses.textpad, xml.etree.ElementTree + + global xml # imitate performing this import at top of file + import xml.etree.ElementTree + def example(): + x = xml.etree.ElementTree.Comment # potential AttributeError + + s = cloudpickle.dumps(example) + + # refresh the environment, i.e., unimport the dependency + del xml + for item in list(sys.modules): + if item.split('.')[0] == 'xml': + del sys.modules[item] + + # deserialise + f = pickle.loads(s) + f() # perform test for error + + def test_submodule_closure(self): + # Same as test_submodule except the package is not a global + def scope(): + import xml.etree.ElementTree + def example(): + x = xml.etree.ElementTree.Comment # potential AttributeError + return example + example = scope() + + s = cloudpickle.dumps(example) + + # refresh the environment (unimport dependency) + for item in list(sys.modules): + if item.split('.')[0] == 'xml': + del sys.modules[item] + + f = cloudpickle.loads(s) + f() # test + + def test_multiprocess(self): + # running a function pickled by another process (a la dask.distributed) + def scope(): + import curses.textpad + def example(): + x = xml.etree.ElementTree.Comment + x = curses.textpad.Textbox + return example + global xml + import xml.etree.ElementTree + example = scope() + + s = cloudpickle.dumps(example) + + # choose "subprocess" rather than "multiprocessing" because the latter + # library uses fork to preserve the parent environment. + command = ("import pickle, base64; " + "pickle.loads(base64.b32decode('" + + base64.b32encode(s).decode('ascii') + + "'))()") + assert not subprocess.call([sys.executable, '-c', command]) + + def test_import(self): + # like test_multiprocess except subpackage modules referenced directly + # (unlike test_submodule) + global etree + def scope(): + import curses.textpad as foobar + def example(): + x = etree.Comment + x = foobar.Textbox + return example + example = scope() + import xml.etree.ElementTree as etree + + s = cloudpickle.dumps(example) + + command = ("import pickle, base64; " + "pickle.loads(base64.b32decode('" + + base64.b32encode(s).decode('ascii') + + "'))()") + assert not subprocess.call([sys.executable, '-c', command]) + + def test_cell_manipulation(self): + cell = _make_empty_cell() + + with pytest.raises(ValueError): + cell.cell_contents + + ob = object() + cell_set(cell, ob) + self.assertTrue( + cell.cell_contents is ob, + msg='cell contents not set correctly', + ) + + def test_logger(self): + logger = logging.getLogger('cloudpickle.dummy_test_logger') + pickled = pickle_depickle(logger) + self.assertTrue(pickled is logger, (pickled, logger)) + + dumped = cloudpickle.dumps(logger) + + code = """if 1: + import cloudpickle, logging + + logging.basicConfig(level=logging.INFO) + logger = cloudpickle.loads(%(dumped)r) + logger.info('hello') + """ % locals() + proc = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = proc.communicate() + self.assertEqual(proc.wait(), 0) + self.assertEqual(out.strip().decode(), + 'INFO:cloudpickle.dummy_test_logger:hello') + + def test_abc(self): + + @abc.abstractmethod + def foo(self): + raise NotImplementedError('foo') + + # Invoke the metaclass directly rather than using class syntax for + # python 2/3 compat. + AbstractClass = abc.ABCMeta('AbstractClass', (object,), {'foo': foo}) + + class ConcreteClass(AbstractClass): + def foo(self): + return 'it works!' + + depickled_base = pickle_depickle(AbstractClass) + depickled_class = pickle_depickle(ConcreteClass) + depickled_instance = pickle_depickle(ConcreteClass()) + + self.assertEqual(depickled_class().foo(), 'it works!') + self.assertEqual(depickled_instance.foo(), 'it works!') + + # assertRaises doesn't return a contextmanager in python 2.6 :(. + self.failUnlessRaises(TypeError, depickled_base) + + class DepickledBaseSubclass(depickled_base): + def foo(self): + return 'it works for realz!' + + self.assertEqual(DepickledBaseSubclass().foo(), 'it works for realz!') + + @pytest.mark.skipif(not HAVE_WEAKSET, reason="WeakSet doesn't exist") + def test_weakset_identity_preservation(self): + # Test that weaksets don't lose all their inhabitants if they're + # pickled in a larger data structure that includes other references to + # their inhabitants. + + class SomeClass(object): + def __init__(self, x): + self.x = x + + obj1, obj2, obj3 = SomeClass(1), SomeClass(2), SomeClass(3) + + things = [weakref.WeakSet([obj1, obj2]), obj1, obj2, obj3] + result = pickle_depickle(things) + + weakset, depickled1, depickled2, depickled3 = result + + self.assertEqual(depickled1.x, 1) + self.assertEqual(depickled2.x, 2) + self.assertEqual(depickled3.x, 3) + self.assertEqual(len(weakset), 2) + + self.assertEqual(set(weakset), set([depickled1, depickled2])) + + def test_ignoring_whichmodule_exception(self): + class FakeModule(object): + def __getattr__(self, name): + # This throws an exception while looking up within + # pickle.whichimodule. + raise Exception() + + class Foo(object): + __module__ = None + + def foo(self): + return "it works!" + + def foo(): + return "it works!" + + foo.__module__ = None + + sys.modules["_fake_module"] = FakeModule() + try: + # Test whichmodule in save_global. + self.assertEqual(pickle_depickle(Foo()).foo(), "it works!") + + # Test whichmodule in save_function. + self.assertEqual(pickle_depickle(foo)(), "it works!") + finally: + sys.modules.pop("_fake_module", None) + if __name__ == '__main__': unittest.main()