diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index e60cd64d..d5ae330a 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -123,10 +123,18 @@ class CloudPickleConfig: filepath_interceptor: Used to modify filepaths in `co_filename` and function.__globals__['__file__']. + + get_code_object_identifier: Use identifiers derived from code + location when pickling dynamic functions (e.g. lambdas). Enabling + this setting results in pickled payloads becoming more stable to + code changes: when a particular lambda function is slightly + modified but the location of the function in the codebase has not + changed, the pickled representation might stay the same. """ id_generator: typing.Optional[callable] = uuid_generator skip_reset_dynamic_type_state: bool = False filepath_interceptor: typing.Optional[callable] = None + get_code_object_identifier: typing.Optional[callable] = None DEFAULT_CONFIG = CloudPickleConfig() @@ -569,6 +577,12 @@ def _make_function(code, globals, name, argdefs, closure): return types.FunctionType(code, globals, name, argdefs, closure) +def _make_function_from_identifier( + get_code_from_identifier, code_path, globals, name, argdefs, closure): + fcode = get_code_from_identifier(code_path) + return _make_function(fcode, globals, name, argdefs, closure) + + def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda @@ -1307,6 +1321,20 @@ class Pickler(pickle.Pickler): dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) + def _stable_identifier_function_reduce(self, func): + code_path = self.config.get_code_object_identifier(func) + if not code_path: + return self._dynamic_function_reduce(func) + newargs = (code_path, ) + state = _function_getstate(func) + return ( + _make_function_from_identifier, + newargs, + state, + None, + None, + _function_setstate) + # function reducers are defined as instance methods of cloudpickle.Pickler # objects, as they rely on a cloudpickle.Pickler attribute (globals_ref) def _dynamic_function_reduce(self, func): @@ -1326,6 +1354,8 @@ def _function_reduce(self, obj): """ if _should_pickle_by_reference(obj): return NotImplemented + elif self.config.get_code_object_identifier is not None: + return self._stable_identifier_function_reduce(obj) else: return self._dynamic_function_reduce(obj)