Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
720a8ea
Added tests for shared_memory submodule.
applio Feb 2, 2019
29a7f80
Added tests for ShareableList.
applio Feb 3, 2019
c56e29c
Fix bug in allocationn size during creation of empty ShareableList il…
applio Feb 3, 2019
c36de70
Initial set of docs for shared_memory module.
applio Feb 7, 2019
3c89c7c
Added docs for ShareableList, added doctree entry for shared_memory s…
applio Feb 8, 2019
5f4ba8f
Added examples to SharedMemoryManager docs, for ease of documentation…
applio Feb 9, 2019
f9aaa11
Wording tweaks to docs.
applio Feb 9, 2019
2377cfd
Fix test failures on Windows.
applio Feb 9, 2019
6bfa560
Added tests around SharedMemoryManager.
applio Feb 9, 2019
eaf7888
Documentation tweaks.
applio Feb 9, 2019
e166ed9
Fix inappropriate test on Windows.
applio Feb 9, 2019
0f18511
Further documentation tweaks.
applio Feb 11, 2019
a097dbb
Fix bare exception.
applio Feb 11, 2019
7c65017
Removed __copyright__.
applio Feb 11, 2019
da7731d
Fixed typo in doc, removed comment.
applio Feb 11, 2019
242a5e9
Merge remote-tracking branch 'upstream/master' into enh-tests-shmem
applio Feb 11, 2019
7bdfbbb
Updated SharedMemoryManager preliminary tests to reflect change of no…
applio Feb 11, 2019
eec4bb1
Added Sphinx doctest run controls.
applio Feb 11, 2019
1076567
CloseHandle should be in a finally block in case MapViewOfFile fails.
applio Feb 12, 2019
0be0531
Missed opportunity to use with statement.
applio Feb 12, 2019
1e5341e
Switch to self.addCleanup to spare long try/finally blocks and save o…
applio Feb 12, 2019
a5800a9
Simplify the posixshmem extension module.
nascheme Feb 13, 2019
34f1e9a
Added to doc around size parameter of SharedMemory.
applio Feb 16, 2019
9846290
Changed PosixSharedMemory.size to use os.fstat.
applio Feb 16, 2019
1f9bbf2
Change SharedMemory.buf to a read-only property as well as NamedShare…
applio Feb 17, 2019
69dd8a9
Marked as provisional per PEP411 in docstring.
applio Feb 17, 2019
8cf9ba3
Merge branch 'enh-tests-neilsimplify-shmem' into enh-tests-shmem
applio Feb 17, 2019
594140a
Changed SharedMemoryTracker to be private.
applio Feb 17, 2019
395709b
Removed registered Proxy Objects from SharedMemoryManager.
applio Feb 17, 2019
aa4a887
Removed shareable_wrap().
applio Feb 17, 2019
885592b
Removed shareable_wrap() and dangling references to it.
applio Feb 17, 2019
9001b76
Merge remote and local branches regarding elimination of
applio Feb 17, 2019
5848ec4
For consistency added __reduce__ to key classes.
applio Feb 17, 2019
6ff8eed
Fix for potential race condition on Windows for O_CREX.
applio Feb 18, 2019
06620e2
Remove unused imports.
applio Feb 18, 2019
868b83d
Update access to kernel32 on Windows per feedback from eryksun.
applio Feb 19, 2019
9d83b06
Moved kernel32 calls to _winapi.
applio Feb 20, 2019
715ded9
Removed ShareableList.copy as redundant.
applio Feb 20, 2019
6878533
Changes to _winapi use from eryksun feedback.
applio Feb 20, 2019
0d3d06f
Adopt simpler SharedMemory API, collapsing PosixSharedMemory and Wind…
applio Feb 21, 2019
05e26dd
Fix missing docstring on class, add test for ignoring size when attac…
applio Feb 21, 2019
7a3c7e5
Moved SharedMemoryManager to managers module, tweak to fragile test.
applio Feb 21, 2019
caf0a5d
Tweak to exception in OpenFileMapping suggested by eryksun.
applio Feb 21, 2019
12c097d
Mark a few dangling bits as private as suggested by Giampaolo.
applio Feb 22, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Removed shareable_wrap() and dangling references to it.
  • Loading branch information
applio committed Feb 17, 2019
commit 885592b56f555c03e6444c7de1839c38c4792fb6
162 changes: 1 addition & 161 deletions Lib/multiprocessing/shared_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@


__all__ = [ 'SharedMemory', 'PosixSharedMemory', 'WindowsNamedSharedMemory',
'ShareableList', 'shareable_wrap',
'ShareableList',
'SharedMemoryServer', 'SharedMemoryManager' ]


Expand Down Expand Up @@ -263,161 +263,6 @@ def __new__(cls, *args, **kwargs):
return cls(*args, **kwargs)


def shareable_wrap(
existing_obj=None,
shmem_name=None,
cls=None,
shape=(0,),
strides=None,
dtype=None,
format=None,
**kwargs
):
"""Provides a fast, convenient way to encapsulate objects that support
the buffer protocol as both producer and consumer, duplicating the
original object's data in shared memory and returning a new wrapped
object that when serialized via pickle does not serialize its data.

The function has been written in a general way to potentially work with
any object supporting the buffer protocol as producer and consumer. It
is known to work well with NumPy ndarrays. Among the Python core data
types and standard library, there are a number of objects supporting
the buffer protocol as a producer but not as a consumer.

Without an example of a producer+consumer of the buffer protocol in
the Python core to demonstrate the use of this function, this function
should likely be removed from this module and potentially be made
available instead via a pip-installable package."""

augmented_kwargs = dict(kwargs)
extras = dict(shape=shape, strides=strides, dtype=dtype, format=format)
for key, value in extras.items():
if value is not None:
augmented_kwargs[key] = value

if existing_obj is not None:
existing_type = getattr(
existing_obj,
"_proxied_type",
type(existing_obj)
)

#agg = existing_obj.itemsize
#size = [ agg := i * agg for i in existing_obj.shape ][-1]
# TODO: replace use of reduce below with above 2 lines once available
size = reduce(
lambda x, y: x * y,
existing_obj.shape,
existing_obj.itemsize
)

else:
assert shmem_name is not None
existing_type = cls
size = 1

shm = SharedMemory(shmem_name, size=size)

class CustomShareableProxy(existing_type):

def __init__(self, *args, buffer=None, **kwargs):
# If copy method called, prevent recursion from replacing _shm.
if not hasattr(self, "_shm"):
self._shm = shm
self._proxied_type = existing_type
else:
# _proxied_type only used in pickling.
assert hasattr(self, "_proxied_type")
try:
existing_type.__init__(self, *args, **kwargs)
except Exception:
pass

def __repr__(self):
if not hasattr(self, "_shm"):
return existing_type.__repr__(self)
formatted_pairs = (
"%s=%r" % kv for kv in self._build_state(self).items()
)
return f"{self.__class__.__name__}({', '.join(formatted_pairs)})"

#def __getstate__(self):
# if not hasattr(self, "_shm"):
# return existing_type.__getstate__(self)
# state = self._build_state(self)
# return state

#def __setstate__(self, state):
# self.__init__(**state)

def __reduce__(self):
return (
shareable_wrap,
(
None,
self._shm.name,
self._proxied_type,
self.shape,
self.strides,
self.dtype.str if hasattr(self, "dtype") else None,
getattr(self, "format", None),
),
)

def copy(self):
dupe = existing_type.copy(self)
if not hasattr(dupe, "_shm"):
dupe = shareable_wrap(dupe)
return dupe

@staticmethod
def _build_state(existing_obj, generics_only=False):
state = {
"shape": existing_obj.shape,
"strides": existing_obj.strides,
}
try:
state["dtype"] = existing_obj.dtype
except AttributeError:
try:
state["format"] = existing_obj.format
except AttributeError:
pass
if not generics_only:
try:
state["shmem_name"] = existing_obj._shm.name
state["cls"] = existing_type
except AttributeError:
pass
return state

proxy_type = type(
f"{existing_type.__name__}Shareable",
CustomShareableProxy.__bases__,
dict(CustomShareableProxy.__dict__),
)

if existing_obj is not None:
try:
proxy_obj = proxy_type(
buffer=shm.buf,
**proxy_type._build_state(existing_obj)
)
except Exception:
proxy_obj = proxy_type(
buffer=shm.buf,
**proxy_type._build_state(existing_obj, True)
)

mveo = memoryview(existing_obj)
proxy_obj._shm.buf[:mveo.nbytes] = mveo.tobytes()

else:
proxy_obj = proxy_type(buffer=shm.buf, **augmented_kwargs)

return proxy_obj


encoding = "utf8"
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is only used by ShareableList. Why not make it a class attribute of that class instead? Also not sure if there are use-cases where one may want to change this (if not it's probably better to set it private).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Though it did not get included, the plan was to also supply a ShareableDict and encoding was to be used across both. Anticipating that a ShareableDict will still land in the future, let's keep it outside the class but I agree it makes sense to make this private.

Changed to private in commit 12c097d.


class ShareableList:
Expand Down Expand Up @@ -730,11 +575,6 @@ def __getstate__(self):
def __setstate__(self, state):
self.__init__(*state)

def wrap(self, obj_exposing_buffer_protocol):
wrapped_obj = shareable_wrap(obj_exposing_buffer_protocol)
self.register_segment(wrapped_obj._shm.name)
return wrapped_obj


class SharedMemoryServer(Server):

Expand Down