Using the Python 2.6 multiprocessing module within a class

If you want to use the new multiprocessing module in Python 2.6 within a class, you might run into some problems. Here's a trick how to do a work-around.

This is how it doesn't work. Instead you get the error message below.
from multiprocessing import Pool
import time

class C:        
    def f(self, name):
        print 'hello %s,'%name
        time.sleep(5)
        print 'nice to meet you.'
    
    def run(self):
        pool = Pool(processes=2)
        pool.map(self.f, ('frank', 'justin', 'osi', 'thomas'))

if __name__ == '__main__':
    c = C()
    c.run()
Exception in thread Thread-1:
Traceback (most recent call last):
File "/sw/lib/python2.6/threading.py", line 522, in __bootstrap_inner
self.run()
File "/sw/lib/python2.6/threading.py", line 477, in run
self.__target(*self.__args, **self.__kwargs)
File "/sw/lib/python2.6/multiprocessing/pool.py", line 225, in _handle_tasks
put(task)
PicklingError: Can't pickle : attribute lookup __builtin__.instancemethod failed

There's a little trick to solve this problem: Use a function outside the class to unpack the self from the arguments and calls f again.
from multiprocessing import Pool
import time

def unwrap_self_f(arg, **kwarg):
    return C.f(*arg, **kwarg)

class C:
    def f(self, name):
        print 'hello %s,'%name
        time.sleep(5)
        print 'nice to meet you.'
    
    def run(self):
        pool = Pool(processes=2)
        names = ('frank', 'justin', 'osi', 'thomas')
        pool.map(unwrap_self_f, zip([self]*len(names), names))

if __name__ == '__main__':
    c = C()
    c.run()



This page has been viewed 14349 times.