Skip to content
Closed
Prev Previous commit
Next Next commit
Fixed named tuple issue
  • Loading branch information
holdenk committed Jul 25, 2017
commit 195cd21ece2036df88a95d2fdc7dfd29c5681efa
11 changes: 11 additions & 0 deletions python/pyspark/cloudpickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -547,6 +547,10 @@ def save_global(self, obj, name=None, pack=struct.pack):
if new_override:
d['__new__'] = obj.__new__

# namedtuple is a special case for Spark
if getattr(obj, '_is_namedtuple_', False):
self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields))
return
self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
Expand Down Expand Up @@ -1000,6 +1004,13 @@ def _find_module(mod_name):
file.close()
return path, description

def _load_namedtuple(name, fields):
"""
Loads a class generated by namedtuple
"""
from collections import namedtuple
return namedtuple(name, fields)

"""Constructors for 3rd party libraries
Note: These can never be renamed due to client compatibility issues"""

Expand Down