Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
720a8ea
Added tests for shared_memory submodule.
applio Feb 2, 2019
29a7f80
Added tests for ShareableList.
applio Feb 3, 2019
c56e29c
Fix bug in allocationn size during creation of empty ShareableList il…
applio Feb 3, 2019
c36de70
Initial set of docs for shared_memory module.
applio Feb 7, 2019
3c89c7c
Added docs for ShareableList, added doctree entry for shared_memory s…
applio Feb 8, 2019
5f4ba8f
Added examples to SharedMemoryManager docs, for ease of documentation…
applio Feb 9, 2019
f9aaa11
Wording tweaks to docs.
applio Feb 9, 2019
2377cfd
Fix test failures on Windows.
applio Feb 9, 2019
6bfa560
Added tests around SharedMemoryManager.
applio Feb 9, 2019
eaf7888
Documentation tweaks.
applio Feb 9, 2019
e166ed9
Fix inappropriate test on Windows.
applio Feb 9, 2019
0f18511
Further documentation tweaks.
applio Feb 11, 2019
a097dbb
Fix bare exception.
applio Feb 11, 2019
7c65017
Removed __copyright__.
applio Feb 11, 2019
da7731d
Fixed typo in doc, removed comment.
applio Feb 11, 2019
242a5e9
Merge remote-tracking branch 'upstream/master' into enh-tests-shmem
applio Feb 11, 2019
7bdfbbb
Updated SharedMemoryManager preliminary tests to reflect change of no…
applio Feb 11, 2019
eec4bb1
Added Sphinx doctest run controls.
applio Feb 11, 2019
1076567
CloseHandle should be in a finally block in case MapViewOfFile fails.
applio Feb 12, 2019
0be0531
Missed opportunity to use with statement.
applio Feb 12, 2019
1e5341e
Switch to self.addCleanup to spare long try/finally blocks and save o…
applio Feb 12, 2019
a5800a9
Simplify the posixshmem extension module.
nascheme Feb 13, 2019
34f1e9a
Added to doc around size parameter of SharedMemory.
applio Feb 16, 2019
9846290
Changed PosixSharedMemory.size to use os.fstat.
applio Feb 16, 2019
1f9bbf2
Change SharedMemory.buf to a read-only property as well as NamedShare…
applio Feb 17, 2019
69dd8a9
Marked as provisional per PEP411 in docstring.
applio Feb 17, 2019
8cf9ba3
Merge branch 'enh-tests-neilsimplify-shmem' into enh-tests-shmem
applio Feb 17, 2019
594140a
Changed SharedMemoryTracker to be private.
applio Feb 17, 2019
395709b
Removed registered Proxy Objects from SharedMemoryManager.
applio Feb 17, 2019
aa4a887
Removed shareable_wrap().
applio Feb 17, 2019
885592b
Removed shareable_wrap() and dangling references to it.
applio Feb 17, 2019
9001b76
Merge remote and local branches regarding elimination of
applio Feb 17, 2019
5848ec4
For consistency added __reduce__ to key classes.
applio Feb 17, 2019
6ff8eed
Fix for potential race condition on Windows for O_CREX.
applio Feb 18, 2019
06620e2
Remove unused imports.
applio Feb 18, 2019
868b83d
Update access to kernel32 on Windows per feedback from eryksun.
applio Feb 19, 2019
9d83b06
Moved kernel32 calls to _winapi.
applio Feb 20, 2019
715ded9
Removed ShareableList.copy as redundant.
applio Feb 20, 2019
6878533
Changes to _winapi use from eryksun feedback.
applio Feb 20, 2019
0d3d06f
Adopt simpler SharedMemory API, collapsing PosixSharedMemory and Wind…
applio Feb 21, 2019
05e26dd
Fix missing docstring on class, add test for ignoring size when attac…
applio Feb 21, 2019
7a3c7e5
Moved SharedMemoryManager to managers module, tweak to fragile test.
applio Feb 21, 2019
caf0a5d
Tweak to exception in OpenFileMapping suggested by eryksun.
applio Feb 21, 2019
12c097d
Mark a few dangling bits as private as suggested by Giampaolo.
applio Feb 22, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updated SharedMemoryManager preliminary tests to reflect change of no…
…t supporting all registered functions on SyncManager.
  • Loading branch information
applio committed Feb 11, 2019
commit 7bdfbbb71908aa7af236058453a0618e64ff1580
97 changes: 52 additions & 45 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -4956,29 +4956,9 @@ def is_alive(self):
any(process.is_alive() for process in forked_processes))


class TestSyncManagerTypes(unittest.TestCase):
"""Test all the types which can be shared between a parent and a
child process by using a manager which acts as an intermediary
between them.

In the following unit-tests the base type is created in the parent
process, the @classmethod represents the worker process and the
shared object is readable and editable between the two.

# The child.
@classmethod
def _test_list(cls, obj):
assert obj[0] == 5
assert obj.append(6)
class _MixinTestCommonManagerTypes(object):

# The parent.
def test_list(self):
o = self.manager.list()
o.append(5)
self.run_worker(self._test_list, o)
assert o[1] == 6
"""
manager_class = multiprocessing.managers.SyncManager
manager_class = None

def setUp(self):
self.manager = self.manager_class()
Expand Down Expand Up @@ -5025,27 +5005,6 @@ def run_worker(self, worker, obj):
self.wait_proc_exit()
self.assertEqual(self.proc.exitcode, 0)

@classmethod
def _test_queue(cls, obj):
assert obj.qsize() == 2
assert obj.full()
assert not obj.empty()
assert obj.get() == 5
assert not obj.empty()
assert obj.get() == 6
assert obj.empty()

def test_queue(self, qname="Queue"):
o = getattr(self.manager, qname)(2)
o.put(5)
o.put(6)
self.run_worker(self._test_queue, o)
assert o.empty()
assert not o.full()

def test_joinable_queue(self):
self.test_queue("JoinableQueue")

@classmethod
def _test_event(cls, obj):
assert obj.is_set()
Expand Down Expand Up @@ -5109,6 +5068,31 @@ def test_barrier(self):
o = self.manager.Barrier(5)
self.run_worker(self._test_barrier, o)


class TestSyncManagerTypes(_MixinTestCommonManagerTypes, unittest.TestCase):
"""Test all the types which can be shared between a parent and a
child process by using a manager which acts as an intermediary
between them.

In the following unit-tests the base type is created in the parent
process, the @classmethod represents the worker process and the
shared object is readable and editable between the two.

# The child.
@classmethod
def _test_list(cls, obj):
assert obj[0] == 5
assert obj.append(6)

# The parent.
def test_list(self):
o = self.manager.list()
o.append(5)
self.run_worker(self._test_list, o)
assert o[1] == 6
"""
manager_class = multiprocessing.managers.SyncManager

@classmethod
def _test_pool(cls, obj):
# TODO: fix https://bugs.python.org/issue35919
Expand All @@ -5119,6 +5103,27 @@ def test_pool(self):
o = self.manager.Pool(processes=4)
self.run_worker(self._test_pool, o)

@classmethod
def _test_queue(cls, obj):
assert obj.qsize() == 2
assert obj.full()
assert not obj.empty()
assert obj.get() == 5
assert not obj.empty()
assert obj.get() == 6
assert obj.empty()

def test_queue(self, qname="Queue"):
o = getattr(self.manager, qname)(2)
o.put(5)
o.put(6)
self.run_worker(self._test_queue, o)
assert o.empty()
assert not o.full()

def test_joinable_queue(self):
self.test_queue("JoinableQueue")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand why you need to move these 2 methods from the previous tests. The logic appears split between the mixing class and this one for no apparent reason (but I may be missing it). If you need a different logic for certain TestSharedMemoryManagerTypes methods/tests you can simply override them (e.g. if you want to skip them).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both Queue and JoinableQueue are implemented as distributed shared memory (a queue.Queue object existing in one process and an AutoProxy object existing in the local process). For the same reasons mentioned in my b.p.o. comment, I believe it is much clearer to not include these on SharedMemoryManager by default and to instead document how they may be added on-demand with a single line of code (by calling register()).

There is also the potential for a POSIX shared memory message queue (not currently implemented in shared_memory.py). This is also something worth considering for the future.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question: if we expose them they are not subject to the speedup, correct? Would it be the same as using them via the SyncManager?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct, exposing Queue or JoinableQueue here would not make them faster; they would be the same as if using them via SyncManager.

We could simply encourage users to employ a SyncManager anytime they want to use a Queue or Lock but this forces the creation of two managers with a server behind each and therefore two processes instead of only one. I think it will be easier to explain, "create one manager to oversee work and provide cleanup at the end."

Although my use of empty() in this example is admittedly fragile, this example shows augmenting SyncManager with Queue is really only one line of code:

from multiprocessing import shared_memory
import queue
shared_memory.SharedMemoryManager.register('Queue', queue.Queue)
with shared_memory.SharedMemoryManager() as smm:
    sl = smm.ShareableList("feb")  # fast, zero-copy shared memory
    q = smm.Queue()                # slow, copy+message distributed shared memory
    for letter in sl:
        q.put(letter.upper())
    while not q.empty():
        print(q.get(timeout=1))

Copy link
Copy Markdown
Contributor

@giampaolo giampaolo Feb 16, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also think we should get rid of Queue, Lock, etc. Just for confirmation: ShareableList and AsyncManager.Lock can be safely mixed/used together, correct?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed -- I have now gotten rid of Queue, Lock, etc. in commit 395709b.

Yes, confirmed: ShareableList and SyncManager.Lock can be safely mixed/used together.

I was sure you meant SyncManager when your wrote AsyncManager but I still went searching just to make sure there was not some AsyncManager hiding inside asyncio somewhere that I was not thinking of. =)

@classmethod
def _test_list(cls, obj):
assert obj[0] == 5
Expand Down Expand Up @@ -5197,10 +5202,12 @@ def test_namespace(self):
import multiprocessing.shared_memory
except ImportError:
@unittest.skip("SharedMemoryManager not available on this platform")
class TestSharedMemoryManagerTypes(TestSyncManagerTypes):
class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
unittest.TestCase):
pass
else:
class TestSharedMemoryManagerTypes(TestSyncManagerTypes):
class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
unittest.TestCase):
"""Same as above but by using SharedMemoryManager."""
manager_class = multiprocessing.shared_memory.SharedMemoryManager

Expand Down