From 720a8ea4d0b208180176c312ad65e7222f2c3b94 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 2 Feb 2019 15:34:08 -0600
Subject: [PATCH 01/41] Added tests for shared_memory submodule.

---
 Lib/multiprocessing/shared_memory.py |  22 +++--
 Lib/test/_test_multiprocessing.py    | 128 +++++++++++++++++++++++++++
 2 files changed, 144 insertions(+), 6 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 11eac4bf0e3994..1a1fa6a873cbfb 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -15,7 +15,9 @@
 import struct
 import sys
 try:
-    from _posixshmem import _PosixSharedMemory, Error, ExistentialError, O_CREX
+    from _posixshmem import _PosixSharedMemory, \
+                            Error, ExistentialError, PermissionsError, \
+                            O_CREAT, O_EXCL, O_CREX, O_TRUNC
 except ImportError as ie:
     if os.name != "nt":
         # On Windows, posixshmem is not required to be available.
@@ -24,12 +26,12 @@
         _PosixSharedMemory = object
         class ExistentialError(BaseException): pass
         class Error(BaseException): pass
-        O_CREX = -1
+        O_CREAT, O_EXCL, O_CREX, O_TRUNC = -1, -1, -1, -1
 
 
 class WindowsNamedSharedMemory:
 
-    def __init__(self, name, flags=None, mode=None, size=None, read_only=False):
+    def __init__(self, name, flags=None, mode=384, size=None, read_only=False):
         if name is None:
             name = f'wnsm_{os.getpid()}_{random.randrange(100000)}'
 
@@ -53,13 +55,21 @@ def unlink(self):
 
 class PosixSharedMemory(_PosixSharedMemory):
 
-    def __init__(self, name, flags=None, mode=None, size=None, read_only=False):
+    def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if name and (flags is None):
-            _PosixSharedMemory.__init__(self, name)
+            _PosixSharedMemory.__init__(self, name, mode=mode)
         else:
             if name is None:
                 name = f'psm_{os.getpid()}_{random.randrange(100000)}'
-            _PosixSharedMemory.__init__(self, name, flags=O_CREX, size=size)
+            flags = O_CREX if flags is None else flags
+            _PosixSharedMemory.__init__(
+                self,
+                name,
+                flags=flags,
+                mode=mode,
+                size=size,
+                read_only=read_only
+            )
 
         self._mmap = mmap.mmap(self.fd, self.size)
         self.buf = memoryview(self._mmap)
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 7341131231a4f0..c0a4f61fbae95f 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -53,6 +53,12 @@
 except ImportError:
     HAS_SHAREDCTYPES = False
 
+try:
+    from multiprocessing import shared_memory
+    HAS_SHMEM = True
+except ImportError:
+    HAS_SHMEM = False
+
 try:
     import msvcrt
 except ImportError:
@@ -3606,6 +3612,128 @@ def test_copy(self):
         self.assertAlmostEqual(bar.y, 5.0)
         self.assertEqual(bar.z, 2 ** 33)
 
+
+class _TestSharedMemory(BaseTestCase):
+
+    ALLOWED_TYPES = ('processes',)
+
+    def setUp(self):
+        if not HAS_SHMEM:
+            self.skipTest("requires multiprocessing.shared_memory")
+
+    @staticmethod
+    def _attach_existing_shmem_then_write(shmem_name, binary_data):
+        local_sms = shared_memory.SharedMemory(shmem_name)
+        local_sms.buf[:len(binary_data)] = binary_data
+        local_sms.close()
+
+    def test_shared_memory_basics(self):
+        sms = shared_memory.SharedMemory(
+            'test01_tsmb',
+            flags=shared_memory.O_CREX,
+            size=512
+        )
+        try:
+            # Verify attributes are readable.
+            self.assertEqual(sms.name, 'test01_tsmb')
+            self.assertGreaterEqual(sms.size, 512)
+            self.assertGreaterEqual(len(sms.buf), sms.size)
+            self.assertEqual(sms.mode, 0o600)
+
+            # Modify contents of shared memory segment through memoryview.
+            sms.buf[0] = 42
+            self.assertEqual(sms.buf[0], 42)
+
+            # Attach to existing shared memory segment.
+            also_sms = shared_memory.SharedMemory('test01_tsmb')
+            self.assertEqual(also_sms.buf[0], 42)
+            also_sms.close()
+
+            if isinstance(sms, shared_memory.PosixSharedMemory):
+                # Posix Shared Memory can only be unlinked once.  Here we
+                # test an implementation detail that is not observed across
+                # all supported platforms (since WindowsNamedSharedMemory
+                # manages unlinking on its own and unlink() does nothing).
+                # True release of shared memory segment does not necessarily
+                # happen until process exits, depending on the OS platform.
+                with self.assertRaises(shared_memory.ExistentialError):
+                    sms_uno = shared_memory.SharedMemory(
+                        'test01_dblunlink',
+                        flags=shared_memory.O_CREX,
+                        size=5000
+                    )
+
+                    try:
+                        self.assertGreaterEqual(sms_uno.size, 5000)
+
+                        sms_duo = shared_memory.SharedMemory('test01_dblunlink')
+                        sms_duo.unlink()  # First shm_unlink() call.
+                        sms_duo.close()
+                        sms_uno.close()
+
+                    finally:
+                        sms_uno.unlink()  # A second shm_unlink() call is bad.
+
+            # Enforcement of `mode` and `read_only` is OS platform dependent
+            # and as such will not be tested here.
+
+            with self.assertRaises(shared_memory.ExistentialError):
+                # Attempting to create a new shared memory segment with a
+                # name that is already in use triggers an exception.
+                there_can_only_be_one_sms = shared_memory.SharedMemory(
+                    'test01_tsmb',
+                    flags=shared_memory.O_CREX,
+                    size=512
+                )
+
+            # Requesting creation of a shared memory segment with the option
+            # to attach to an existing segment, if that name is currently in
+            # use, should not trigger an exception.
+            # Note:  Using a smaller size could possibly cause truncation of
+            # the existing segment but is OS platform dependent.  In the
+            # case of MacOS/darwin, requesting a smaller size is disallowed.
+            ok_if_exists_sms = shared_memory.SharedMemory(
+                'test01_tsmb',
+                flags=shared_memory.O_CREAT,
+                size=sms.size if sys.platform != 'darwin' else 0
+            )
+            self.assertEqual(ok_if_exists_sms.size, sms.size)
+            ok_if_exists_sms.close()
+
+            # Attempting to attach to an existing shared memory segment when
+            # no segment exists with the supplied name triggers an exception.
+            with self.assertRaises(shared_memory.ExistentialError):
+                nonexisting_sms = shared_memory.SharedMemory('test01_notthere')
+                nonexisting_sms.unlink()  # Error should occur on prior line.
+
+            sms.close()
+
+        finally:
+            # Prevent test failures from leading to a dangling segment.
+            sms.unlink()
+
+    def test_shared_memory_across_processes(self):
+        sms = shared_memory.SharedMemory(
+            'test02_tsmap',
+            flags=shared_memory.O_CREX,
+            size=512
+        )
+
+        try:
+            p = self.Process(
+                target=self._attach_existing_shmem_then_write,
+                args=(sms.name, b'howdy')
+            )
+            p.daemon = True
+            p.start()
+            p.join()
+            self.assertEqual(bytes(sms.buf[:5]), b'howdy')
+
+            sms.close()
+
+        finally:
+            sms.unlink()
+
 #
 #
 #

From 29a7f804530d2a90eb48d065748ab33da5fad674 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 3 Feb 2019 10:22:40 -0600
Subject: [PATCH 02/41] Added tests for ShareableList.

---
 Lib/multiprocessing/shared_memory.py | 12 +++--
 Lib/test/_test_multiprocessing.py    | 81 ++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 5 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 1a1fa6a873cbfb..014d99b9da4697 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -275,7 +275,7 @@ def _extract_recreation_code(value):
         else:
             return 3  # NoneType
 
-    def __init__(self, iterable=None, name=None):
+    def __init__(self, iterable=None, *, name=None):
         if iterable is not None:
             _formats = [
                 self.types_mapping[type(item)]
@@ -301,7 +301,7 @@ def __init__(self, iterable=None, name=None):
         else:
             requested_size = 1  # Some platforms require > 0.
 
-        self.shm = SharedMemory(name, size=requested_size)
+        self.shm = SharedMemory(name, flags=O_CREX, size=requested_size)
 
         if iterable is not None:
             _enc = encoding
@@ -474,11 +474,13 @@ def _offset_packing_formats(self):
     def _offset_back_transform_codes(self):
         return self._offset_packing_formats + self._list_len * 8
 
-    @classmethod
-    def copy(cls, self):
+    def copy(self, *, name=None):
         "L.copy() -> ShareableList -- a shallow copy of L."
 
-        return cls(self)
+        if name is None:
+            return self.__class__(self)
+        else:
+            return self.__class__(self, name=name)
 
     def count(self, value):
         "L.count(value) -> integer -- return number of occurrences of value."
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c0a4f61fbae95f..d2e30ccfa7b688 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3734,6 +3734,87 @@ def test_shared_memory_across_processes(self):
         finally:
             sms.unlink()
 
+    def test_shared_memory_ShareableList_basics(self):
+        sl = shared_memory.ShareableList(
+            ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
+        )
+
+        try:
+            # Verify attributes are readable.
+            self.assertEqual(sl.alignment, 8)
+            self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
+
+            # Exercise len().
+            self.assertEqual(len(sl), 7)
+
+            # Exercise index().
+            with warnings.catch_warnings():
+                # Suppress BytesWarning when comparing against b'HoWdY'.
+                warnings.simplefilter('ignore')
+                with self.assertRaises(ValueError):
+                    sl.index('100')
+                self.assertEqual(sl.index(100), 3)
+
+            # Exercise retrieving individual values.
+            self.assertEqual(sl[0], 'howdy')
+            self.assertEqual(sl[-2], True)
+
+            # Exercise iterability.
+            self.assertEqual(
+                tuple(sl),
+                ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
+            )
+
+            # Exercise modifying individual values.
+            sl[3] = 42
+            self.assertEqual(sl[3], 42)
+            sl[4] = 'some'  # Change type at a given position.
+            self.assertEqual(sl[4], 'some')
+            self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+            with self.assertRaises(ValueError):
+                sl[4] = 'far too many'  # Exceeds available storage.
+            self.assertEqual(sl[4], 'some')
+
+            # Exercise count().
+            with warnings.catch_warnings():
+                # Suppress BytesWarning when comparing against b'HoWdY'.
+                warnings.simplefilter('ignore')
+                self.assertEqual(sl.count(42), 2)
+                self.assertEqual(sl.count(b'HoWdY'), 1)
+                self.assertEqual(sl.count(b'adios'), 0)
+
+            # Exercise creating a duplicate.
+            sl_copy = sl.copy(name='test03_duplicate')
+            try:
+                self.assertNotEqual(sl.shm.name, sl_copy.shm.name)
+                self.assertEqual('test03_duplicate', sl_copy.shm.name)
+                self.assertEqual(list(sl), list(sl_copy))
+                self.assertEqual(sl.format, sl_copy.format)
+                sl_copy[-1] = 77
+                self.assertEqual(sl_copy[-1], 77)
+                self.assertNotEqual(sl[-1], 77)
+                sl_copy.shm.close()
+            finally:
+                sl_copy.shm.unlink()
+
+        finally:
+            # Prevent test failures from leading to a dangling segment.
+            sl.shm.unlink()
+            sl.shm.close()
+
+        # Exercise creating an empty ShareableList.
+        empty_sl = shared_memory.ShareableList()
+        try:
+            self.assertEqual(len(empty_sl), 0)
+            self.assertEqual(empty_sl.format, '')
+            self.assertEqual(empty_sl.alignment, 8)
+            self.assertEqual(empty_sl.count('any'), 0)
+            with self.assertRaises(ValueError):
+                empty_sl.index(None)
+            empty_sl.shm.close()
+        finally:
+            empty_sl.shm.unlink()
+
 #
 #
 #

From c56e29c63afa815e7c1128e980b515778d2158d6 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 3 Feb 2019 11:55:08 -0600
Subject: [PATCH 03/41] Fix bug in allocationn size during creation of empty
 ShareableList illuminated by existing test run on Linux.

---
 Lib/multiprocessing/shared_memory.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 014d99b9da4697..43a64112027f24 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -295,11 +295,14 @@ def __init__(self, iterable=None, *, name=None):
                 self._extract_recreation_code(item) for item in iterable
             ]
             requested_size = struct.calcsize(
-                "q" + self._format_size_metainfo + "".join(_formats)
+                "q" + self._format_size_metainfo +
+                "".join(_formats) +
+                self._format_packing_metainfo +
+                self._format_back_transform_codes
             )
 
         else:
-            requested_size = 1  # Some platforms require > 0.
+            requested_size = 8  # Some platforms require > 0.
 
         self.shm = SharedMemory(name, flags=O_CREX, size=requested_size)
 

From c36de7004c7fb1d2edae2e79c1d68d2d9482f44c Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 6 Feb 2019 21:36:11 -0600
Subject: [PATCH 04/41] Initial set of docs for shared_memory module.

---
 Doc/library/multiprocessing.shared_memory.rst | 187 ++++++++++++++++++
 1 file changed, 187 insertions(+)
 create mode 100644 Doc/library/multiprocessing.shared_memory.rst

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
new file mode 100644
index 00000000000000..b02f30e625bf08
--- /dev/null
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -0,0 +1,187 @@
+:mod:`multiprocessing.shared_memory` ---  Provides shared memory for direct access across processes
+===================================================================================================
+
+.. module:: multiprocessing.shared_memory
+   :synopsis: Provides shared memory for direct access across processes.
+
+**Source code:** :source:`Lib/multiprocessing/shared_memory.py`
+
+.. versionadded:: 3.8
+
+.. index::
+   single: Shared Memory
+   single: POSIX Shared Memory
+   single: Named Shared Memory
+
+--------------
+
+This module provides a class, :class:`SharedMemory`, for the allocation
+and management of shared memory to be accessed by one or more processes
+on a multicore or SMP machine.  To assist with the life-cycle management
+of shared memory across distinct processes, a
+:func:`multiprocessing.Manager` class, :class:`SharedMemoryManager`, is
+also provided.
+
+In this module, shared memory refers to "System V style" shared memory blocks
+(though is not necessarily implemented explicitly as such).  This style
+of memory, when created, permits distinct processes to potentially read and
+write to a common (or shared) region of volatile memory.  Because processes
+conventionally only have access to their own process memory space, this
+construct of shared memory permits the sharing of data between processes,
+avoiding the need to instead send messages between processes containing
+that data.  Sharing data directly via memory can provide significant
+performance benefits compared to sharing data via disk or socket or other
+communications requiring the serialization/de-serialization and copying of
+data.
+
+
+.. class:: SharedMemory(name, flags=None, mode=384, size=0, read_only=False)
+
+   This class creates and returns an instance of either a
+   :class:`PosixSharedMemory` or :class:`NamedSharedMemory` class depending
+   upon their availability on the local system.
+
+   *name* is the unique name for the requested shared memory, specified as
+   a string.  If ``None`` is supplied for the name, a new shared memory
+   block with a novel name will be created without needing to also specify
+   ``flags``.
+
+   *flags* is set to ``None`` when attempting to attach to an existing shared
+   memory block by its unique name but if no existing block has that name, an
+   exception will be raised.  To request the creation of a new shared
+   memory block, set to ``O_CREX``.  To request the optional creation of a
+   new shared memory block or attach to an existing one by the same name,
+   set to ``O_CREAT``.
+
+   *mode* controls user/group/all-based read/write permissions on the
+   shared memory block.  Its specification is not enforceable on all platforms.
+
+   *size* specifies the number of bytes requested for a shared memory block.
+   Because some platforms choose to allocate chunks of memory based upon
+   that platform's memory page size, the exact size of the shared memory
+   block may be larger or equal to the size requested.  When attaching to an
+   existing shared memory block, set to ``0`` (which is the default).
+
+   *read_only* controls whether a shared memory block is to be available
+   for only reading or for both reading and writing.  Its specification is
+   not enforceable on all platforms.
+
+   .. method:: close()
+
+      Closes access to the shared memory from this instance.  In order to
+      ensure proper cleanup of resources, all instances should call
+      ``close()`` once the instance is no longer needed.  Note that calling
+      ``close()`` does not cause the shared memory block itself to be
+      destroyed.
+
+   .. method:: unlink()
+
+      Requests that the underlying shared memory block be destroyed.  In
+      order to ensure proper cleanup of resources, ``unlink()`` should be
+      called once (and only once) across all processes which have need
+      for the shared memory block.  After requesting its destruction, a
+      shared memory block may or may not be immediately destroyed and
+      this behavior may differ across platforms.  Attempts to access data
+      inside the shared memory block after ``unlink()`` has been called may
+      result in memory access errors.  Note: the last process relinquishing
+      its hold on a shared memory block may call ``unlink()`` and
+      ``close()`` in either order.
+
+   .. attribute:: buf
+
+      A memoryview of contents of the shared memory block.
+
+   .. attribute:: name
+
+      Read-only access to the unique name of the shared memory block.
+
+   .. attribute:: mode
+
+      Read-only access to access permissions mode of the shared memory block.
+
+   .. attribute:: size
+
+      Read-only access to size in bytes of the shared memory block.
+
+
+The following example demonstrates low-level use of :class:`SharedMemory`
+instances::
+
+   >>> from multiprocessing import shared_memory
+   >>> shm_a = shared_memory.SharedMemory(None, size=10)
+   >>> type(shm_b.buf)
+   <class 'memoryview'>
+   >>> buffer = shm_a.buf
+   >>> len(buffer)
+   10
+   >>> buffer[:4] = bytearray([22, 33, 44, 55])  # Modify multiple at once
+   >>> buffer[4] = 100  # Modify a single byte at a time
+   >>> # Attach to an existing shared memory block
+   >>> shm_b = shared_memory.SharedMemory(shm_a.name)
+   >>> import array
+   >>> array.array('b', shm_b.buf[:5])  # Copy the data into a new array.array
+   array('b', [22, 33, 44, 55, 100])
+   >>> shm_b.buf[:5] = b'howdy'  # Modify via shm_b using bytes
+   >>> bytes(shm_a.buf[:5])  # Access via shm_a
+   b'howdy'
+   >>> shm_b.close()  # Close each SharedMemory instance
+   >>> shm_a.close()
+   >>> shm_a.unlink()  # Call unlink only once to release the shared memory
+
+
+
+The following example demonstrates a practical use of the :class:`SharedMemory`
+class with ``numpy`` arrays, accessing the same ``numpy.ndarray`` from
+two distinct Python shells::
+
+   >>> # In the first Python interactive shell
+   >>> import numpy as np
+   >>> a = np.array([1, 1, 2, 3, 5, 8])  # Start with an existing NumPy array
+   >>> from multiprocessing import shared_memory
+   >>> shm = shared_memory.SharedMemory(None, size=a.nbytes)
+   >>> # Now create a NumPy array backed by shared memory
+   >>> b = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf)
+   >>> b[:] = a[:]  # Copy the original data into shared memory
+   >>> b
+   array([1, 1, 2, 3, 5, 8])
+   >>> type(b)
+   <class 'numpy.ndarray'>
+   >>> type(a)
+   <class 'numpy.ndarray'>
+   >>> shm.name  # We did not specify a name so one was chosen for us
+   'psm_21467_46075'
+
+   >>> # In either the same shell or a new Python shell on the same machine
+   >>> # Attach to the existing shared memory block
+   >>> existing_shm = shared_memory.SharedMemory('psm_21467_46075')
+   >>> # Note that a.shape is (6,) and a.dtype is np.int64 in this example
+   >>> c = np.ndarray((6,), dtype=np.int64, buffer=existing_shm.buf)
+   >>> c
+   array([1, 1, 2, 3, 5, 8])
+   >>> c[-1] = 888
+   >>> c
+   array([  1,   1,   2,   3,   5, 888])
+
+   >>> # Back in the first Python interactive shell, b reflects this change
+   >>> b
+   array([  1,   1,   2,   3,   5, 888])
+
+   >>> # Clean up from within the second Python shell
+   >>> del c  # Unnecessary; merely emphasizing the array is no longer used
+   >>> existing_shm.close()
+
+   >>> # Clean up from within the first Python shell
+   >>> del b  # Unnecessary; merely emphasizing the array is no longer used
+   >>> shm.close()
+   >>> shm.unlink()  # Free and release the shared memory block at the very end
+
+
+.. class:: SharedMemoryManager
+
+   A subclass of :class:`multiprocessing.managers.SyncManager` which can be
+   used for the management of shared memory blocks across processes.
+
+   It provides methods for creating and returning a :class:`SharedMemory`
+   instance and for creating a list-like object (:class:`ShareableList`)
+   backed by shared memory.
+

From 3c89c7c73e786112a551615f14310cabe4a35dd9 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Fri, 8 Feb 2019 15:38:29 -0600
Subject: [PATCH 05/41] Added docs for ShareableList, added doctree entry for
 shared_memory submodule, name refactoring for greater clarity.

---
 Doc/library/concurrency.rst                   |  1 +
 Doc/library/multiprocessing.shared_memory.rst | 97 +++++++++++++++++++
 Lib/multiprocessing/shared_memory.py          | 51 +++++-----
 Lib/test/_test_multiprocessing.py             |  9 +-
 4 files changed, 134 insertions(+), 24 deletions(-)

diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst
index 826bf86d081793..39cd9ff4826597 100644
--- a/Doc/library/concurrency.rst
+++ b/Doc/library/concurrency.rst
@@ -15,6 +15,7 @@ multitasking). Here's an overview:
 
    threading.rst
    multiprocessing.rst
+   multiprocessing.shared_memory.rst
    concurrent.rst
    concurrent.futures.rst
    subprocess.rst
diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index b02f30e625bf08..ac0c31b4246c74 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -152,6 +152,8 @@ two distinct Python shells::
    'psm_21467_46075'
 
    >>> # In either the same shell or a new Python shell on the same machine
+   >>> import numpy as np
+   >>> from multiprocessing import shared_memory
    >>> # Attach to the existing shared memory block
    >>> existing_shm = shared_memory.SharedMemory('psm_21467_46075')
    >>> # Note that a.shape is (6,) and a.dtype is np.int64 in this example
@@ -185,3 +187,98 @@ two distinct Python shells::
    instance and for creating a list-like object (:class:`ShareableList`)
    backed by shared memory.
 
+
+.. class:: ShareableList(sequence=None, *, name=None)
+
+   Provides a mutable list-like object where all values stored within are
+   stored in a shared memory block.  This constrains storable values to
+   only the ``int``, ``float``, ``bool``, ``str`` (less than 10M bytes each),
+   ``bytes`` (less than 10M bytes each), and ``None`` built-in data types.
+   It also notably differs from the built-in ``list`` type in that these
+   lists can not change their overall length (i.e. no append, insert, etc.)
+   and do not support the dynamic creation of new `ShareableList` instances
+   via slicing.
+
+   *sequence* is used in populating a new ``ShareableList`` full of values.
+   Set to ``None`` to instead attach to an already existing
+   ``ShareableList`` in shared memory by its name.
+
+   *name* is the unique name for the requested shared memory, as described
+   in the definition for :class:`SharedMemory`.  When attaching to an
+   existing ``ShareableList``, specify its shared memory block's unique
+   name while leaving ``sequence`` set to ``None``.
+
+   .. method:: copy()
+
+      Returns a shallow copy as a new instance backed by a new and distinct
+      shared memory block.
+
+   .. method:: count(value)
+
+      Returns the number of occurrences of ``value``.
+
+   .. method:: index(value)
+
+      Returns first index position of ``value``.  Raises ValueError if
+      ``value`` is not present.
+
+   .. attribute:: format
+
+      Read-only attribute containing the struct packing format used by all
+      currently stored values.
+
+   .. attribute:: shm
+
+      The :class:`SharedMemory` instance where the values are stored.
+
+
+The following example demonstrates basic use of a :class:`ShareableList`
+instance:
+
+   >>> from multiprocessing import shared_memory
+   >>> a = shared_memory.ShareableList(['howdy', b'HoWdY', -273.154, 100, None, True, 42])
+   >>> [ type(entry) for entry in a ]
+   [<class 'str'>, <class 'bytes'>, <class 'float'>, <class 'int'>, <class 'NoneType'>, <class 'bool'>, <class 'int'>]
+   >>> a[2]
+   -273.154
+   >>> a[2] = -78.5
+   >>> a[2]
+   -78.5
+   >>> a[2] = 'dry ice'  # Changing data types is supported as well.
+   >>> a[2]
+   'dry ice'
+   >>> a[2] = 'larger than previously allocated storage space'
+   Traceback (most recent call last):
+     File "<stdin>", line 1, in <module>
+     File "/usr/local/lib/python3.8/multiprocessing/shared_memory.py", line 429, in __setitem__
+       raise ValueError("exceeds available storage for existing str")
+   ValueError: exceeds available storage for existing str
+   >>> a[2]
+   'dry ice'
+   >>> len(a)
+   7
+   >>> a.index(42)
+   6
+   >>> a.count(b'howdy')
+   0
+   >>> a.count(b'HoWdY')
+   1
+   >>> a.shm.close()
+   >>> a.shm.unlink()
+   >>> del a  # Use of a ShareableList after call to unlink() is unsupported
+
+The following example depicts how one, two, or many processes may access the
+same :class:`ShareableList` by supplying the name of the shared memory block
+behind it:
+
+   >>> b = shared_memory.ShareableList(range(5))         # In a first process
+   >>> c = shared_memory.ShareableList(name=b.shm.name)  # In a second process
+   >>> c
+   ShareableList([0, 1, 2, 3, 4], name='psm_25144_23776')
+   >>> c[-1] = -999
+   >>> b[-1]
+   -999
+   >>> b.shm.close()
+   >>> c.shm.close()
+   >>> c.shm.unlink()
+
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 43a64112027f24..194756d4a4a61f 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -244,8 +244,7 @@ class ShareableList:
     packing format for any storable value must require no more than 8
     characters to describe its format."""
 
-    # TODO: Adjust for discovered word size of machine.
-    types_mapping = {
+    _types_mapping = {
         int: "q",
         float: "d",
         bool: "xxxxxxx?",
@@ -253,8 +252,8 @@ class ShareableList:
         bytes: "%ds",
         None.__class__: "xxxxxx?x",
     }
-    alignment = 8
-    back_transform_codes = {
+    _alignment = 8
+    _back_transforms_mapping = {
         0: lambda value: value,                   # int, float, bool
         1: lambda value: value.rstrip(b'\x00').decode(encoding),  # str
         2: lambda value: value.rstrip(b'\x00'),   # bytes
@@ -263,7 +262,7 @@ class ShareableList:
 
     @staticmethod
     def _extract_recreation_code(value):
-        """Used in concert with back_transform_codes to convert values
+        """Used in concert with _back_transforms_mapping to convert values
         into the appropriate Python objects when retrieving them from
         the list as well as when storing them."""
         if not isinstance(value, (str, bytes, None.__class__)):
@@ -275,24 +274,24 @@ def _extract_recreation_code(value):
         else:
             return 3  # NoneType
 
-    def __init__(self, iterable=None, *, name=None):
-        if iterable is not None:
+    def __init__(self, sequence=None, *, name=None):
+        if sequence is not None:
             _formats = [
-                self.types_mapping[type(item)]
+                self._types_mapping[type(item)]
                     if not isinstance(item, (str, bytes))
-                    else self.types_mapping[type(item)] % (
-                        self.alignment * (len(item) // self.alignment + 1),
+                    else self._types_mapping[type(item)] % (
+                        self._alignment * (len(item) // self._alignment + 1),
                     )
-                for item in iterable
+                for item in sequence
             ]
             self._list_len = len(_formats)
             assert sum(len(fmt) <= 8 for fmt in _formats) == self._list_len
             self._allocated_bytes = tuple(
-                    self.alignment if fmt[-1] != "s" else int(fmt[:-1])
+                    self._alignment if fmt[-1] != "s" else int(fmt[:-1])
                     for fmt in _formats
             )
-            _back_transform_codes = [
-                self._extract_recreation_code(item) for item in iterable
+            _recreation_codes = [
+                self._extract_recreation_code(item) for item in sequence
             ]
             requested_size = struct.calcsize(
                 "q" + self._format_size_metainfo +
@@ -304,9 +303,12 @@ def __init__(self, iterable=None, *, name=None):
         else:
             requested_size = 8  # Some platforms require > 0.
 
-        self.shm = SharedMemory(name, flags=O_CREX, size=requested_size)
+        if name is not None and sequence is None:
+            self.shm = SharedMemory(name)
+        else:
+            self.shm = SharedMemory(name, flags=O_CREX, size=requested_size)
 
-        if iterable is not None:
+        if sequence is not None:
             _enc = encoding
             struct.pack_into(
                 "q" + self._format_size_metainfo,
@@ -319,7 +321,7 @@ def __init__(self, iterable=None, *, name=None):
                 "".join(_formats),
                 self.shm.buf,
                 self._offset_data_start,
-                *(v.encode(_enc) if isinstance(v, str) else v for v in iterable)
+                *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
             )
             struct.pack_into(
                 self._format_packing_metainfo,
@@ -331,7 +333,7 @@ def __init__(self, iterable=None, *, name=None):
                 self._format_back_transform_codes,
                 self.shm.buf,
                 self._offset_back_transform_codes,
-                *(_back_transform_codes)
+                *(_recreation_codes)
             )
 
         else:
@@ -370,7 +372,7 @@ def _get_back_transform(self, position):
             self.shm.buf,
             self._offset_back_transform_codes + position
         )[0]
-        transform_function = self.back_transform_codes[transform_code]
+        transform_function = self._back_transforms_mapping[transform_code]
 
         return transform_function
 
@@ -423,14 +425,14 @@ def __setitem__(self, position, value):
             raise IndexError("assignment index out of range")
 
         if not isinstance(value, (str, bytes)):
-            new_format = self.types_mapping[type(value)]
+            new_format = self._types_mapping[type(value)]
         else:
             if len(value) > self._allocated_bytes[position]:
                 raise ValueError("exceeds available storage for existing str")
             if current_format[-1] == "s":
                 new_format = current_format
             else:
-                new_format = self.types_mapping[str] % (
+                new_format = self._types_mapping[str] % (
                     self._allocated_bytes[position],
                 )
 
@@ -445,10 +447,15 @@ def __setitem__(self, position, value):
     def __len__(self):
         return struct.unpack_from("q", self.shm.buf, 0)[0]
 
+    def __repr__(self):
+        return f'{self.__class__.__name__}({list(self)}, name={self.shm.name!r})'
+
     @property
     def format(self):
         "The struct packing format used by all currently stored values."
-        return "".join(self._get_packing_format(i) for i in range(self._list_len))
+        return "".join(
+            self._get_packing_format(i) for i in range(self._list_len)
+        )
 
     @property
     def _format_size_metainfo(self):
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index d2e30ccfa7b688..dc455038ae9072 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3741,7 +3741,6 @@ def test_shared_memory_ShareableList_basics(self):
 
         try:
             # Verify attributes are readable.
-            self.assertEqual(sl.alignment, 8)
             self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
 
             # Exercise len().
@@ -3797,6 +3796,13 @@ def test_shared_memory_ShareableList_basics(self):
             finally:
                 sl_copy.shm.unlink()
 
+            # Obtain a second handle on the same ShareableList.
+            sl_tethered = shared_memory.ShareableList(name=sl.shm.name)
+            self.assertEqual(sl.shm.name, sl_tethered.shm.name)
+            sl_tethered[-1] = 880
+            self.assertEqual(sl[-1], 880)
+            sl_tethered.shm.close()
+
         finally:
             # Prevent test failures from leading to a dangling segment.
             sl.shm.unlink()
@@ -3807,7 +3813,6 @@ def test_shared_memory_ShareableList_basics(self):
         try:
             self.assertEqual(len(empty_sl), 0)
             self.assertEqual(empty_sl.format, '')
-            self.assertEqual(empty_sl.alignment, 8)
             self.assertEqual(empty_sl.count('any'), 0)
             with self.assertRaises(ValueError):
                 empty_sl.index(None)

From 5f4ba8f4d5b12b4ab08bb58557a09eb6344cd8b8 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Fri, 8 Feb 2019 23:03:15 -0600
Subject: [PATCH 06/41] Added examples to SharedMemoryManager docs, for ease of
 documentation switched away from exclusively registered functions to some
 explicit methods on SharedMemoryManager.

---
 Doc/library/multiprocessing.shared_memory.rst | 145 +++++++++++++++---
 Lib/multiprocessing/shared_memory.py          |  88 +++++++++--
 2 files changed, 195 insertions(+), 38 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index ac0c31b4246c74..a4c956ac5a7bc6 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -18,24 +18,24 @@
 This module provides a class, :class:`SharedMemory`, for the allocation
 and management of shared memory to be accessed by one or more processes
 on a multicore or SMP machine.  To assist with the life-cycle management
-of shared memory across distinct processes, a
-:func:`multiprocessing.Manager` class, :class:`SharedMemoryManager`, is
-also provided.
+of shared memory especially across distinct processes, a
+:class:`multiprocessing.managers.BaseManager` subclass,
+:class:`SharedMemoryManager`, is also provided.
 
 In this module, shared memory refers to "System V style" shared memory blocks
-(though is not necessarily implemented explicitly as such).  This style
-of memory, when created, permits distinct processes to potentially read and
-write to a common (or shared) region of volatile memory.  Because processes
-conventionally only have access to their own process memory space, this
-construct of shared memory permits the sharing of data between processes,
-avoiding the need to instead send messages between processes containing
-that data.  Sharing data directly via memory can provide significant
-performance benefits compared to sharing data via disk or socket or other
-communications requiring the serialization/de-serialization and copying of
-data.
+(though is not necessarily implemented explicitly as such) and does not refer
+to "distributed shared memory".  This style of shared memory permits distinct
+processes to potentially read and write to a common (or shared) region of
+volatile memory.  Because processes conventionally only have access to their
+own process memory space, this construct of shared memory permits the sharing
+of data between processes, avoiding the need to instead send messages between
+processes containing that data.  Sharing data directly via memory can provide
+significant performance benefits compared to sharing data via disk or socket
+or other communications requiring the serialization/de-serialization and
+copying of data.
 
 
-.. class:: SharedMemory(name, flags=None, mode=384, size=0, read_only=False)
+.. class:: SharedMemory(name, flags=None, mode=0o600, size=0, read_only=False)
 
    This class creates and returns an instance of either a
    :class:`PosixSharedMemory` or :class:`NamedSharedMemory` class depending
@@ -115,16 +115,16 @@ instances::
    >>> len(buffer)
    10
    >>> buffer[:4] = bytearray([22, 33, 44, 55])  # Modify multiple at once
-   >>> buffer[4] = 100  # Modify a single byte at a time
+   >>> buffer[4] = 100                           # Modify single byte at a time
    >>> # Attach to an existing shared memory block
    >>> shm_b = shared_memory.SharedMemory(shm_a.name)
    >>> import array
    >>> array.array('b', shm_b.buf[:5])  # Copy the data into a new array.array
    array('b', [22, 33, 44, 55, 100])
    >>> shm_b.buf[:5] = b'howdy'  # Modify via shm_b using bytes
-   >>> bytes(shm_a.buf[:5])  # Access via shm_a
+   >>> bytes(shm_a.buf[:5])      # Access via shm_a
    b'howdy'
-   >>> shm_b.close()  # Close each SharedMemory instance
+   >>> shm_b.close()   # Close each SharedMemory instance
    >>> shm_a.close()
    >>> shm_a.unlink()  # Call unlink only once to release the shared memory
 
@@ -178,14 +178,109 @@ two distinct Python shells::
    >>> shm.unlink()  # Free and release the shared memory block at the very end
 
 
-.. class:: SharedMemoryManager
+.. class:: SharedMemoryManager([address[, authkey]])
 
-   A subclass of :class:`multiprocessing.managers.SyncManager` which can be
+   A subclass of :class:`multiprocessing.managers.BaseManager` which can be
    used for the management of shared memory blocks across processes.
 
-   It provides methods for creating and returning a :class:`SharedMemory`
-   instance and for creating a list-like object (:class:`ShareableList`)
-   backed by shared memory.
+   Instantiation of a :class:`SharedMemoryManager` causes a new process to
+   be started.  This new process's sole purpose is to manage the life cycle
+   of all shared memory blocks created through it.  To trigger the release
+   of all shared memory blocks managed by that process, call
+   :func:`multiprocessing.managers.BaseManager.shutdown()` on the instance.
+   This triggers a :func:`SharedMemory.unlink()` call on all of the
+   :class:`SharedMemory` instances managed by that process and then
+   stops the process itself.  By creating ``SharedMemory`` instances
+   through a ``SharedMemoryManager``, we avoid the need to manually track
+   and trigger the freeing of shared memory resources.
+
+   This class provides methods for creating and returning :class:`SharedMemory`
+   instances and for creating a list-like object (:class:`ShareableList`)
+   backed by shared memory.  It also provides methods that create and
+   return :ref:`multiprocessing-proxy_objects` that support synchronization
+   across processes (i.e. multi-process-safe locks and semaphores).
+
+   Refer to :class:`multiprocessing.managers.BaseManager` for a description
+   of the inherited *address* and *authkey* optional input arguments and how
+   they may be used to connect to an existing ``SharedMemoryManager`` service
+   from other processes.
+
+   .. method:: SharedMemory(size)
+
+      Create and return a new :class:`SharedMemory` object with the
+      specified ``size`` in bytes.
+
+   .. method:: ShareableList(sequence)
+
+      Create and return a new :class:`ShareableList` object, initialized
+      by the values from the input ``sequence``.
+
+   .. method:: Barrier(parties[, action[, timeout]])
+
+      Create a shared :class:`threading.Barrier` object and return a
+      proxy for it.
+
+   .. method:: BoundedSemaphore([value])
+
+      Create a shared :class:`threading.BoundedSemaphore` object and return
+      a proxy for it.
+
+   .. method:: Condition([lock])
+
+      Create a shared :class:`threading.Condition` object and return a proxy
+      for it.  The optional input *lock* supports a proxy for a
+      :class:`threading.Lock` or :class:`threading.RLock` object.
+
+   .. method:: Event()
+
+      Create a shared :class:`threading.Event` object and return a proxy for it.
+
+   .. method:: Lock()
+
+      Create a shared :class:`threading.Lock` object and return a proxy for it.
+
+   .. method:: RLock()
+
+      Create a shared :class:`threading.RLock` object and return a proxy for it.
+
+   .. method:: Semaphore([value])
+
+      Create a shared :class:`threading.Semaphore` object and return a proxy
+      for it.
+
+The following example demonstrates the basic mechanisms of a
+:class:`SharedMemoryManager`:
+
+   >>> from multiprocessing import shared_memory
+   >>> smm = shared_memory.SharedMemoryManager()
+   >>> smm.start()  # Start the process that manages the shared memory blocks
+   >>> sl = smm.ShareableList(range(4))
+   >>> sl
+   ShareableList([0, 1, 2, 3], name='psm_6572_7512')
+   >>> raw_shm = smm.SharedMemory(size=128)
+   >>> another_sl = smm.ShareableList('alpha')
+   >>> another_sl
+   ShareableList(['a', 'l', 'p', 'h', 'a'], name='psm_6572_12221')
+   >>> smm.shutdown()  # Calls unlink() on sl, raw_shm, and another_sl
+
+The following example depicts a potentially more convenient pattern for using
+:class:`SharedMemoryManager` objects in a :keyword:`with` statement to ensure
+that all shared memory blocks are released after they are no longer needed:
+
+   >>> with shared_memory.SharedMemoryManager() as smm:
+   ...     sl = smm.ShareableList(range(2000))
+   ...     # Divide the work among two processes, storing partial results in sl
+   ...     p1 = Process(target=do_work, args=(sl.shm.name, 0, 1000))
+   ...     p2 = Process(target=do_work, args=(sl.shm.name, 1000, 2000))
+   ...     p1.start()
+   ...     p2.start()  # A multiprocessing.Pool might be more efficient
+   ...     p1.join()
+   ...     p2.join()   # Wait for all work to complete in both processes
+   ...     total_result = sum(sl)  # Consolidate the partial results now in sl
+
+When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the
+shared memory blocks created using that manager are all released when the
+:keyword:`with` statement's code block finishes execution.
 
 
 .. class:: ShareableList(sequence=None, *, name=None)
@@ -196,12 +291,12 @@ two distinct Python shells::
    ``bytes`` (less than 10M bytes each), and ``None`` built-in data types.
    It also notably differs from the built-in ``list`` type in that these
    lists can not change their overall length (i.e. no append, insert, etc.)
-   and do not support the dynamic creation of new `ShareableList` instances
-   via slicing.
+   and do not support the dynamic creation of new :class:`ShareableList`
+   instances via slicing.
 
    *sequence* is used in populating a new ``ShareableList`` full of values.
    Set to ``None`` to instead attach to an already existing
-   ``ShareableList`` in shared memory by its name.
+   ``ShareableList`` by its unique shared memory name.
 
    *name* is the unique name for the requested shared memory, as described
    in the definition for :class:`SharedMemory`.  When attaching to an
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 194756d4a4a61f..f8ae6db7779777 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -8,12 +8,14 @@
 
 from functools import reduce
 import mmap
-from .managers import DictProxy, SyncManager, Server
+from .managers import dispatch, BaseManager, Server, State, ProcessError, \
+                      BarrierProxy, AcquirerProxy, ConditionProxy, EventProxy
 from . import util
 import os
 import random
 import struct
 import sys
+import threading
 try:
     from _posixshmem import _PosixSharedMemory, \
                             Error, ExistentialError, PermissionsError, \
@@ -515,14 +517,14 @@ def __init__(self, name, segment_names=[]):
         self.shared_memory_context_name = name
         self.segment_names = segment_names
 
-    def register_segment(self, segment):
-        util.debug(f"Registering segment {segment.name!r} in pid {os.getpid()}")
-        self.segment_names.append(segment.name)
+    def register_segment(self, segment_name):
+        util.debug(f"Registering segment {segment_name!r} in pid {os.getpid()}")
+        self.segment_names.append(segment_name)
 
     def destroy_segment(self, segment_name):
         util.debug(f"Destroying segment {segment_name!r} in pid {os.getpid()}")
         self.segment_names.remove(segment_name)
-        segment = SharedMemory(segment_name, size=1)
+        segment = SharedMemory(segment_name)
         segment.close()
         segment.unlink()
 
@@ -542,11 +544,15 @@ def __setstate__(self, state):
 
     def wrap(self, obj_exposing_buffer_protocol):
         wrapped_obj = shareable_wrap(obj_exposing_buffer_protocol)
-        self.register_segment(wrapped_obj._shm)
+        self.register_segment(wrapped_obj._shm.name)
         return wrapped_obj
 
 
 class SharedMemoryServer(Server):
+
+    public = Server.public + \
+             ['track_segment', 'release_segment', 'list_segments']
+
     def __init__(self, *args, **kwargs):
         Server.__init__(self, *args, **kwargs)
         self.shared_memory_context = \
@@ -565,16 +571,30 @@ def shutdown(self, c):
         self.shared_memory_context.unlink()
         return Server.shutdown(self, c)
 
+    def track_segment(self, c, segment_name):
+        self.shared_memory_context.register_segment(segment_name)
+
+    def release_segment(self, c, segment_name):
+        self.shared_memory_context.destroy_segment(segment_name)
+
+    def list_segments(self, c):
+        return self.shared_memory_context.segment_names
+
 
-class SharedMemoryManager(SyncManager):
+class SharedMemoryManager(BaseManager):
     """Like SyncManager but uses SharedMemoryServer instead of Server.
 
-    TODO: Consider relocate/merge into managers submodule."""
+    It provides methods for creating and returning SharedMemory instances
+    and for creating a list-like object (ShareableList) backed by shared
+    memory.  It also provides methods that create and return Proxy Objects
+    that support synchronization across processes (i.e. multi-process-safe
+    locks and semaphores).
+    """
 
     _Server = SharedMemoryServer
 
     def __init__(self, *args, **kwargs):
-        SyncManager.__init__(self, *args, **kwargs)
+        BaseManager.__init__(self, *args, **kwargs)
         util.debug(f"{self.__class__.__name__} created by pid {os.getpid()}")
 
     def __del__(self):
@@ -585,11 +605,53 @@ def get_server(self):
         'Better than monkeypatching for now; merge into Server ultimately'
         if self._state.value != State.INITIAL:
             if self._state.value == State.STARTED:
-                raise ProcessError("Already started server")
+                raise ProcessError("Already started SharedMemoryServer")
             elif self._state.value == State.SHUTDOWN:
-                raise ProcessError("Manager has shut down")
+                raise ProcessError("SharedMemoryManager has shut down")
             else:
                 raise ProcessError(
                     "Unknown state {!r}".format(self._state.value))
-        return _Server(self._registry, self._address,
-                       self._authkey, self._serializer)
+        return self._Server(self._registry, self._address,
+                            self._authkey, self._serializer)
+
+    def SharedMemory(self, size):
+        """Returns a new SharedMemory instance with the specified size in
+        bytes, to be tracked by the manager."""
+        conn = self._Client(self._address, authkey=self._authkey)
+        try:
+            sms = SharedMemory(None, flags=O_CREX, size=size)
+            try:
+                dispatch(conn, None, 'track_segment', (sms.name,))
+            except BaseException as e:
+                sms.unlink()
+                raise e
+        finally:
+            conn.close()
+        return sms
+
+    def ShareableList(self, sequence):
+        """Returns a new ShareableList instance populated with the values
+        from the input sequence, to be tracked by the manager."""
+        conn = self._Client(self._address, authkey=self._authkey)
+        try:
+            sl = ShareableList(sequence)
+            try:
+                dispatch(conn, None, 'track_segment', (sl.shm.name,))
+            except BaseException as e:
+                sl.shm.unlink()
+                raise e
+        finally:
+            conn.close()
+        return sl
+
+SharedMemoryManager.register('Barrier', threading.Barrier, BarrierProxy)
+SharedMemoryManager.register(
+    'BoundedSemaphore',
+    threading.BoundedSemaphore,
+    AcquirerProxy
+)
+SharedMemoryManager.register('Condition', threading.Condition, ConditionProxy)
+SharedMemoryManager.register('Event', threading.Event, EventProxy)
+SharedMemoryManager.register('Lock', threading.Lock, AcquirerProxy)
+SharedMemoryManager.register('RLock', threading.RLock, AcquirerProxy)
+SharedMemoryManager.register('Semaphore', threading.Semaphore, AcquirerProxy)

From f9aaa119f5067620bf69999d06da4256e89c57a9 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Fri, 8 Feb 2019 23:17:42 -0600
Subject: [PATCH 07/41] Wording tweaks to docs.

---
 Doc/library/multiprocessing.shared_memory.rst | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index a4c956ac5a7bc6..3e241be31333a4 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -26,8 +26,8 @@ In this module, shared memory refers to "System V style" shared memory blocks
 (though is not necessarily implemented explicitly as such) and does not refer
 to "distributed shared memory".  This style of shared memory permits distinct
 processes to potentially read and write to a common (or shared) region of
-volatile memory.  Because processes conventionally only have access to their
-own process memory space, this construct of shared memory permits the sharing
+volatile memory.  Processes are conventionally limited to only have access to
+their own process memory space but shared memory permits the sharing
 of data between processes, avoiding the need to instead send messages between
 processes containing that data.  Sharing data directly via memory can provide
 significant performance benefits compared to sharing data via disk or socket
@@ -264,8 +264,9 @@ The following example demonstrates the basic mechanisms of a
    >>> smm.shutdown()  # Calls unlink() on sl, raw_shm, and another_sl
 
 The following example depicts a potentially more convenient pattern for using
-:class:`SharedMemoryManager` objects in a :keyword:`with` statement to ensure
-that all shared memory blocks are released after they are no longer needed:
+:class:`SharedMemoryManager` objects via the :keyword:`with` statement to
+ensure that all shared memory blocks are released after they are no longer
+needed:
 
    >>> with shared_memory.SharedMemoryManager() as smm:
    ...     sl = smm.ShareableList(range(2000))

From 2377cfdbb2faba6c970692acd296bca6c49def2c Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 9 Feb 2019 12:14:39 -0600
Subject: [PATCH 08/41] Fix test failures on Windows.

---
 Lib/multiprocessing/shared_memory.py | 90 ++++++++++++++++++++++++++--
 1 file changed, 86 insertions(+), 4 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index f8ae6db7779777..9c99360c7e95c2 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -26,20 +26,102 @@
         raise ie
     else:
         _PosixSharedMemory = object
-        class ExistentialError(BaseException): pass
-        class Error(BaseException): pass
-        O_CREAT, O_EXCL, O_CREX, O_TRUNC = -1, -1, -1, -1
+        class Error(Exception): pass
+        class ExistentialError(Error): pass
+        O_CREAT, O_EXCL, O_CREX, O_TRUNC = 64, 128, 192, 512
+
+if os.name == "nt":
+    import ctypes
+    from ctypes import wintypes
+
+    #kernel32 = ctypes.wintypes.WinDLL("kernel32", use_last_error=True)
+    kernel32 = ctypes.windll.kernel32
+
+    class MEMORY_BASIC_INFORMATION(ctypes.Structure):
+        _fields_ = (
+            ('BaseAddress',       ctypes.c_void_p),
+            ('AllocationBase',    ctypes.c_void_p),
+            ('AllocationProtect', wintypes.DWORD),
+            ('RegionSize',        ctypes.c_size_t),
+            ('State',             wintypes.DWORD),
+            ('Protect',           wintypes.DWORD),
+            ('Type',              wintypes.DWORD)
+        )
+
+    PMEMORY_BASIC_INFORMATION = ctypes.POINTER(MEMORY_BASIC_INFORMATION)
+    FILE_MAP_READ = 0x0004
+
+    def _errcheck_bool(result, func, args):
+        if not result:
+            raise ctypes.WinError(ctypes.get_last_error())
+        return args
+
+    kernel32.VirtualQuery.errcheck = _errcheck_bool
+    kernel32.VirtualQuery.restype = ctypes.c_size_t
+    kernel32.VirtualQuery.argtypes = (
+        wintypes.LPCVOID,
+        PMEMORY_BASIC_INFORMATION,
+        ctypes.c_size_t
+    )
+
+    kernel32.OpenFileMappingW.errcheck = _errcheck_bool
+    kernel32.OpenFileMappingW.restype = wintypes.HANDLE
+    kernel32.OpenFileMappingW.argtypes = (
+        wintypes.DWORD,
+        wintypes.BOOL,
+        wintypes.LPCWSTR
+    )
+
+    kernel32.MapViewOfFile.errcheck = _errcheck_bool
+    kernel32.MapViewOfFile.restype = wintypes.LPVOID
+    kernel32.MapViewOfFile.argtypes = (
+        wintypes.HANDLE,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        ctypes.c_size_t
+    )
+
+    kernel32.CloseHandle.errcheck = _errcheck_bool
+    kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
 
 
 class WindowsNamedSharedMemory:
 
-    def __init__(self, name, flags=None, mode=384, size=None, read_only=False):
+    def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if name is None:
             name = f'wnsm_{os.getpid()}_{random.randrange(100000)}'
 
+        if size == 0:
+            # Attempt to dynamically determine the existing named shared
+            # memory block's size which is likely a multiple of mmap.PAGESIZE.
+            try:
+                h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
+            except OSError as ose:
+                raise ExistentialError(*ose.args)
+            p_buf = kernel32.MapViewOfFile(h_map, FILE_MAP_READ, 0, 0, 0)
+            kernel32.CloseHandle(h_map)
+            mbi = MEMORY_BASIC_INFORMATION()
+            kernel32.VirtualQuery(p_buf, ctypes.byref(mbi), mmap.PAGESIZE)
+            size = mbi.RegionSize
+
+        if flags == O_CREX:
+            # Verify no named shared memory block already exists by this name.
+            try:
+                h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
+                kernel32.CloseHandle(h_map)
+                name_collision = True
+            except OSError as ose:
+                name_collision = False
+            if name_collision:
+                raise ExistentialError(
+                    f"Shared memory already exists with name={name}"
+                )
+
         self._mmap = mmap.mmap(-1, size, tagname=name)
         self.buf = memoryview(self._mmap)
         self.name = name
+        self.mode = mode
         self.size = size
 
     def __repr__(self):

From 6bfa56098dd8c1506bfd8c03ee2b67ab9220ecea Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 9 Feb 2019 13:47:33 -0600
Subject: [PATCH 09/41] Added tests around SharedMemoryManager.

---
 Lib/test/_test_multiprocessing.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index dc455038ae9072..c4f5127bbfd063 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3734,6 +3734,34 @@ def test_shared_memory_across_processes(self):
         finally:
             sms.unlink()
 
+    def test_shared_memory_SharedMemoryManager_basics(self):
+        smm1 = shared_memory.SharedMemoryManager()
+        with self.assertRaises(ValueError):
+            smm1.SharedMemory(size=9)  # Fails if SharedMemoryServer not started
+        smm1.start()
+        lol = [ smm1.ShareableList(range(i)) for i in range(5, 10) ]
+        lom = [ smm1.SharedMemory(size=j) for j in range(32, 128, 16) ]
+        doppleganger_list0 = shared_memory.ShareableList(name=lol[0].shm.name)
+        self.assertEqual(len(doppleganger_list0), 5)
+        doppleganger_shm0 = shared_memory.SharedMemory(name=lom[0].name)
+        self.assertGreaterEqual(len(doppleganger_shm0.buf), 32)
+        held_name = lom[0].name
+        smm1.shutdown()
+        with self.assertRaises(shared_memory.ExistentialError):
+            # No longer there to be attached to again.
+            absent_shm = shared_memory.SharedMemory(name=held_name)
+
+        with shared_memory.SharedMemoryManager() as smm2:
+            sl = smm2.ShareableList("howdy")
+            unnecessary_lock = smm2.Lock()
+            with unnecessary_lock:
+                shm = smm2.SharedMemory(size=128)
+            held_name = sl.shm.name
+        with self.assertRaises(shared_memory.ExistentialError):
+            # No longer there to be attached to again.
+            absent_sl = shared_memory.ShareableList(name=held_name)
+
+
     def test_shared_memory_ShareableList_basics(self):
         sl = shared_memory.ShareableList(
             ['howdy', b'HoWdY', -273.154, 100, None, True, 42]

From eaf788854b8179cb1e740d7f1278f61b8184e3e5 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 9 Feb 2019 15:30:15 -0600
Subject: [PATCH 10/41] Documentation tweaks.

---
 Doc/library/multiprocessing.shared_memory.rst | 25 ++++++++++---------
 Lib/multiprocessing/shared_memory.py          | 16 ++++++++++++
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 3e241be31333a4..86f453368f547d 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -19,7 +19,7 @@ This module provides a class, :class:`SharedMemory`, for the allocation
 and management of shared memory to be accessed by one or more processes
 on a multicore or SMP machine.  To assist with the life-cycle management
 of shared memory especially across distinct processes, a
-:class:`multiprocessing.managers.BaseManager` subclass,
+:class:`~multiprocessing.managers.BaseManager` subclass,
 :class:`SharedMemoryManager`, is also provided.
 
 In this module, shared memory refers to "System V style" shared memory blocks
@@ -85,7 +85,7 @@ copying of data.
       inside the shared memory block after ``unlink()`` has been called may
       result in memory access errors.  Note: the last process relinquishing
       its hold on a shared memory block may call ``unlink()`` and
-      ``close()`` in either order.
+      :meth:`close()` in either order.
 
    .. attribute:: buf
 
@@ -180,16 +180,17 @@ two distinct Python shells::
 
 .. class:: SharedMemoryManager([address[, authkey]])
 
-   A subclass of :class:`multiprocessing.managers.BaseManager` which can be
+   A subclass of :class:`~multiprocessing.managers.BaseManager` which can be
    used for the management of shared memory blocks across processes.
 
-   Instantiation of a :class:`SharedMemoryManager` causes a new process to
-   be started.  This new process's sole purpose is to manage the life cycle
+   A call to :meth:`~multiprocessing.managers.BaseManager.start` on a
+   :class:`SharedMemoryManager` instance causes a new process to be started.
+   This new process's sole purpose is to manage the life cycle
    of all shared memory blocks created through it.  To trigger the release
    of all shared memory blocks managed by that process, call
-   :func:`multiprocessing.managers.BaseManager.shutdown()` on the instance.
-   This triggers a :func:`SharedMemory.unlink()` call on all of the
-   :class:`SharedMemory` instances managed by that process and then
+   :meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance.
+   This triggers a :meth:`SharedMemory.unlink()` call on all of the
+   :class:`SharedMemory` objects managed by that process and then
    stops the process itself.  By creating ``SharedMemory`` instances
    through a ``SharedMemoryManager``, we avoid the need to manually track
    and trigger the freeing of shared memory resources.
@@ -315,13 +316,13 @@ shared memory blocks created using that manager are all released when the
 
    .. method:: index(value)
 
-      Returns first index position of ``value``.  Raises ValueError if
+      Returns first index position of ``value``.  Raises :exc:`ValueError` if
       ``value`` is not present.
 
    .. attribute:: format
 
-      Read-only attribute containing the struct packing format used by all
-      currently stored values.
+      Read-only attribute containing the :mod:`struct` packing format used by
+      all currently stored values.
 
    .. attribute:: shm
 
@@ -340,7 +341,7 @@ instance:
    >>> a[2] = -78.5
    >>> a[2]
    -78.5
-   >>> a[2] = 'dry ice'  # Changing data types is supported as well.
+   >>> a[2] = 'dry ice'  # Changing data types is supported as well
    >>> a[2]
    'dry ice'
    >>> a[2] = 'larger than previously allocated storage space'
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 9c99360c7e95c2..0d4a935ba3156a 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -187,6 +187,22 @@ def shareable_wrap(
     format=None,
     **kwargs
 ):
+    """Provides a fast, convenient way to encapsulate objects that support
+    the buffer protocol as both producer and consumer, duplicating the
+    original object's data in shared memory and returning a new wrapped
+    object that when serialized via pickle does not serialize its data.
+
+    The function has been written in a general way to potentially work with
+    any object supporting the buffer protocol as producer and consumer.  It
+    is known to work well with NumPy ndarrays.  Among the Python core data
+    types and standard library, there are a number of objects supporting
+    the buffer protocol as a producer but not as a consumer.
+
+    Without an example of a producer+consumer of the buffer protocol in
+    the Python core to demonstrate the use of this function, this function
+    should likely be removed from this module and potentially be made
+    available instead via a pip-installable package."""
+
     augmented_kwargs = dict(kwargs)
     extras = dict(shape=shape, strides=strides, dtype=dtype, format=format)
     for key, value in extras.items():

From e166ed9acde340f797109582df2961cd3263018f Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 9 Feb 2019 16:12:22 -0600
Subject: [PATCH 11/41] Fix inappropriate test on Windows.

---
 Lib/test/_test_multiprocessing.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c4f5127bbfd063..03c2e9a69bb25f 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3747,9 +3747,12 @@ def test_shared_memory_SharedMemoryManager_basics(self):
         self.assertGreaterEqual(len(doppleganger_shm0.buf), 32)
         held_name = lom[0].name
         smm1.shutdown()
-        with self.assertRaises(shared_memory.ExistentialError):
-            # No longer there to be attached to again.
-            absent_shm = shared_memory.SharedMemory(name=held_name)
+        if sys.platform != "win32":
+            # Calls to unlink() have no effect on Windows platform; shared
+            # memory will only be released once final process exits.
+            with self.assertRaises(shared_memory.ExistentialError):
+                # No longer there to be attached to again.
+                absent_shm = shared_memory.SharedMemory(name=held_name)
 
         with shared_memory.SharedMemoryManager() as smm2:
             sl = smm2.ShareableList("howdy")
@@ -3757,9 +3760,10 @@ def test_shared_memory_SharedMemoryManager_basics(self):
             with unnecessary_lock:
                 shm = smm2.SharedMemory(size=128)
             held_name = sl.shm.name
-        with self.assertRaises(shared_memory.ExistentialError):
-            # No longer there to be attached to again.
-            absent_sl = shared_memory.ShareableList(name=held_name)
+        if sys.platform != "win32":
+            with self.assertRaises(shared_memory.ExistentialError):
+                # No longer there to be attached to again.
+                absent_sl = shared_memory.ShareableList(name=held_name)
 
 
     def test_shared_memory_ShareableList_basics(self):

From 0f18511ab2a3699787756969ae13aaa687c6a1c9 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 10 Feb 2019 22:14:40 -0600
Subject: [PATCH 12/41] Further documentation tweaks.

---
 Doc/library/multiprocessing.shared_memory.rst | 12 ++++++------
 Lib/multiprocessing/shared_memory.py          | 12 ++++++++++++
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 86f453368f547d..44718d05cc8fb7 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -17,9 +17,9 @@
 
 This module provides a class, :class:`SharedMemory`, for the allocation
 and management of shared memory to be accessed by one or more processes
-on a multicore or SMP machine.  To assist with the life-cycle management
-of shared memory especially across distinct processes, a
-:class:`~multiprocessing.managers.BaseManager` subclass,
+on a multicore or symmetric multiprocessor (SMP) machine.  To assist with
+the life-cycle management of shared memory especially across distinct
+processes, a :class:`~multiprocessing.managers.BaseManager` subclass,
 :class:`SharedMemoryManager`, is also provided.
 
 In this module, shared memory refers to "System V style" shared memory blocks
@@ -31,7 +31,7 @@ their own process memory space but shared memory permits the sharing
 of data between processes, avoiding the need to instead send messages between
 processes containing that data.  Sharing data directly via memory can provide
 significant performance benefits compared to sharing data via disk or socket
-or other communications requiring the serialization/de-serialization and
+or other communications requiring the serialization/deserialization and
 copying of data.
 
 
@@ -131,8 +131,8 @@ instances::
 
 
 The following example demonstrates a practical use of the :class:`SharedMemory`
-class with ``numpy`` arrays, accessing the same ``numpy.ndarray`` from
-two distinct Python shells::
+class with `NumPy arrays <https://www.numpy.org/>`_, accessing the
+same ``numpy.ndarray`` from two distinct Python shells::
 
    >>> # In the first Python interactive shell
    >>> import numpy as np
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 0d4a935ba3156a..b9738041b0e72c 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -616,10 +616,13 @@ def __init__(self, name, segment_names=[]):
         self.segment_names = segment_names
 
     def register_segment(self, segment_name):
+        "Adds the supplied shared memory block name to tracker."
         util.debug(f"Registering segment {segment_name!r} in pid {os.getpid()}")
         self.segment_names.append(segment_name)
 
     def destroy_segment(self, segment_name):
+        """Calls unlink() on the shared memory block with the supplied name
+        and removes it from the list of blocks being tracked."""
         util.debug(f"Destroying segment {segment_name!r} in pid {os.getpid()}")
         self.segment_names.remove(segment_name)
         segment = SharedMemory(segment_name)
@@ -627,6 +630,7 @@ def destroy_segment(self, segment_name):
         segment.unlink()
 
     def unlink(self):
+        "Calls destroy_segment() on all currently tracked shared memory blocks."
         for segment_name in self.segment_names[:]:
             self.destroy_segment(segment_name)
 
@@ -658,6 +662,8 @@ def __init__(self, *args, **kwargs):
         util.debug(f"SharedMemoryServer started by pid {os.getpid()}")
 
     def create(self, c, typeid, *args, **kwargs):
+        """Create a new distributed-shared object (not backed by a shared
+        memory block) and return its id to be used in a Proxy Object."""
         # Unless set up as a shared proxy, don't make shared_memory_context
         # a standard part of kwargs.  This makes things easier for supplying
         # simple functions.
@@ -666,16 +672,22 @@ def create(self, c, typeid, *args, **kwargs):
         return Server.create(self, c, typeid, *args, **kwargs)
 
     def shutdown(self, c):
+        "Call unlink() on all tracked shared memory then terminate the Server."
         self.shared_memory_context.unlink()
         return Server.shutdown(self, c)
 
     def track_segment(self, c, segment_name):
+        "Adds the supplied shared memory block name to Server's tracker."
         self.shared_memory_context.register_segment(segment_name)
 
     def release_segment(self, c, segment_name):
+        """Calls unlink() on the shared memory block with the supplied name
+        and removes it from the tracker instance inside the Server."""
         self.shared_memory_context.destroy_segment(segment_name)
 
     def list_segments(self, c):
+        """Returns a list of names of shared memory blocks that the Server
+        is currently tracking."""
         return self.shared_memory_context.segment_names
 
 

From a097dbb6fd2fe46be2715818338b874ba350824e Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 10 Feb 2019 22:30:54 -0600
Subject: [PATCH 13/41] Fix bare exception.

---
 Lib/multiprocessing/shared_memory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index b9738041b0e72c..6f60cbca7715bc 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -244,7 +244,7 @@ def __init__(self, *args, buffer=None, **kwargs):
                 assert hasattr(self, "_proxied_type")
             try:
                 existing_type.__init__(self, *args, **kwargs)
-            except:
+            except Exception:
                 pass
 
         def __repr__(self):

From 7c6501781744c4960eaa4991aba2cd73f3c7dae0 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 12:58:10 -0600
Subject: [PATCH 14/41] Removed __copyright__.

---
 Modules/_multiprocessing/posixshmem.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c
index 7dd29f405e4101..b4337d2fb07850 100644
--- a/Modules/_multiprocessing/posixshmem.c
+++ b/Modules/_multiprocessing/posixshmem.c
@@ -690,9 +690,6 @@ PyInit__posixshmem(void) {
     Py_INCREF(&SharedMemoryType);
     PyModule_AddObject(module, "_PosixSharedMemory", (PyObject *)&SharedMemoryType);
 
-
-    PyModule_AddStringConstant(module, "__copyright__", "Copyright 2012 Philip Semanchuk, 2018-2019 Davin Potts");
-
     PyModule_AddIntConstant(module, "O_CREAT", O_CREAT);
     PyModule_AddIntConstant(module, "O_EXCL", O_EXCL);
     PyModule_AddIntConstant(module, "O_CREX", O_CREAT | O_EXCL);

From da7731db412a94d9f06a99500b31c9147e601e3e Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 13:43:00 -0600
Subject: [PATCH 15/41] Fixed typo in doc, removed comment.

---
 Doc/library/multiprocessing.shared_memory.rst | 2 +-
 Lib/multiprocessing/shared_memory.py          | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 44718d05cc8fb7..2753f76927d54e 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -109,7 +109,7 @@ instances::
 
    >>> from multiprocessing import shared_memory
    >>> shm_a = shared_memory.SharedMemory(None, size=10)
-   >>> type(shm_b.buf)
+   >>> type(shm_a.buf)
    <class 'memoryview'>
    >>> buffer = shm_a.buf
    >>> len(buffer)
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 6f60cbca7715bc..8d2fe2d05d4dbe 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -34,7 +34,6 @@ class ExistentialError(Error): pass
     import ctypes
     from ctypes import wintypes
 
-    #kernel32 = ctypes.wintypes.WinDLL("kernel32", use_last_error=True)
     kernel32 = ctypes.windll.kernel32
 
     class MEMORY_BASIC_INFORMATION(ctypes.Structure):

From 7bdfbbb71908aa7af236058453a0618e64ff1580 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 15:21:30 -0600
Subject: [PATCH 16/41] Updated SharedMemoryManager preliminary tests to
 reflect change of not supporting all registered functions on SyncManager.

---
 Lib/test/_test_multiprocessing.py | 97 +++++++++++++++++--------------
 1 file changed, 52 insertions(+), 45 deletions(-)

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index f1809e1dabbf74..4ce2ef0a2a1f5b 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -4956,29 +4956,9 @@ def is_alive(self):
             any(process.is_alive() for process in forked_processes))
 
 
-class TestSyncManagerTypes(unittest.TestCase):
-    """Test all the types which can be shared between a parent and a
-    child process by using a manager which acts as an intermediary
-    between them.
-
-    In the following unit-tests the base type is created in the parent
-    process, the @classmethod represents the worker process and the
-    shared object is readable and editable between the two.
-
-    # The child.
-    @classmethod
-    def _test_list(cls, obj):
-        assert obj[0] == 5
-        assert obj.append(6)
+class _MixinTestCommonManagerTypes(object):
 
-    # The parent.
-    def test_list(self):
-        o = self.manager.list()
-        o.append(5)
-        self.run_worker(self._test_list, o)
-        assert o[1] == 6
-    """
-    manager_class = multiprocessing.managers.SyncManager
+    manager_class = None
 
     def setUp(self):
         self.manager = self.manager_class()
@@ -5025,27 +5005,6 @@ def run_worker(self, worker, obj):
         self.wait_proc_exit()
         self.assertEqual(self.proc.exitcode, 0)
 
-    @classmethod
-    def _test_queue(cls, obj):
-        assert obj.qsize() == 2
-        assert obj.full()
-        assert not obj.empty()
-        assert obj.get() == 5
-        assert not obj.empty()
-        assert obj.get() == 6
-        assert obj.empty()
-
-    def test_queue(self, qname="Queue"):
-        o = getattr(self.manager, qname)(2)
-        o.put(5)
-        o.put(6)
-        self.run_worker(self._test_queue, o)
-        assert o.empty()
-        assert not o.full()
-
-    def test_joinable_queue(self):
-        self.test_queue("JoinableQueue")
-
     @classmethod
     def _test_event(cls, obj):
         assert obj.is_set()
@@ -5109,6 +5068,31 @@ def test_barrier(self):
         o = self.manager.Barrier(5)
         self.run_worker(self._test_barrier, o)
 
+
+class TestSyncManagerTypes(_MixinTestCommonManagerTypes, unittest.TestCase):
+    """Test all the types which can be shared between a parent and a
+    child process by using a manager which acts as an intermediary
+    between them.
+
+    In the following unit-tests the base type is created in the parent
+    process, the @classmethod represents the worker process and the
+    shared object is readable and editable between the two.
+
+    # The child.
+    @classmethod
+    def _test_list(cls, obj):
+        assert obj[0] == 5
+        assert obj.append(6)
+
+    # The parent.
+    def test_list(self):
+        o = self.manager.list()
+        o.append(5)
+        self.run_worker(self._test_list, o)
+        assert o[1] == 6
+    """
+    manager_class = multiprocessing.managers.SyncManager
+
     @classmethod
     def _test_pool(cls, obj):
         # TODO: fix https://bugs.python.org/issue35919
@@ -5119,6 +5103,27 @@ def test_pool(self):
         o = self.manager.Pool(processes=4)
         self.run_worker(self._test_pool, o)
 
+    @classmethod
+    def _test_queue(cls, obj):
+        assert obj.qsize() == 2
+        assert obj.full()
+        assert not obj.empty()
+        assert obj.get() == 5
+        assert not obj.empty()
+        assert obj.get() == 6
+        assert obj.empty()
+
+    def test_queue(self, qname="Queue"):
+        o = getattr(self.manager, qname)(2)
+        o.put(5)
+        o.put(6)
+        self.run_worker(self._test_queue, o)
+        assert o.empty()
+        assert not o.full()
+
+    def test_joinable_queue(self):
+        self.test_queue("JoinableQueue")
+
     @classmethod
     def _test_list(cls, obj):
         assert obj[0] == 5
@@ -5197,10 +5202,12 @@ def test_namespace(self):
     import multiprocessing.shared_memory
 except ImportError:
     @unittest.skip("SharedMemoryManager not available on this platform")
-    class TestSharedMemoryManagerTypes(TestSyncManagerTypes):
+    class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
+                                       unittest.TestCase):
         pass
 else:
-    class TestSharedMemoryManagerTypes(TestSyncManagerTypes):
+    class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
+                                       unittest.TestCase):
         """Same as above but by using SharedMemoryManager."""
         manager_class = multiprocessing.shared_memory.SharedMemoryManager
 

From eec4bb13e8ebe6b2f3a0025062e97115f1f30d22 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 16:36:42 -0600
Subject: [PATCH 17/41] Added Sphinx doctest run controls.

---
 Doc/library/multiprocessing.shared_memory.rst | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 2753f76927d54e..fd217e29bf81d5 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -132,7 +132,10 @@ instances::
 
 The following example demonstrates a practical use of the :class:`SharedMemory`
 class with `NumPy arrays <https://www.numpy.org/>`_, accessing the
-same ``numpy.ndarray`` from two distinct Python shells::
+same ``numpy.ndarray`` from two distinct Python shells:
+
+.. doctest::
+   :options: +SKIP
 
    >>> # In the first Python interactive shell
    >>> import numpy as np
@@ -252,6 +255,9 @@ same ``numpy.ndarray`` from two distinct Python shells::
 The following example demonstrates the basic mechanisms of a
 :class:`SharedMemoryManager`:
 
+.. doctest::
+   :options: +SKIP
+
    >>> from multiprocessing import shared_memory
    >>> smm = shared_memory.SharedMemoryManager()
    >>> smm.start()  # Start the process that manages the shared memory blocks
@@ -269,6 +275,9 @@ The following example depicts a potentially more convenient pattern for using
 ensure that all shared memory blocks are released after they are no longer
 needed:
 
+.. doctest::
+   :options: +SKIP
+
    >>> with shared_memory.SharedMemoryManager() as smm:
    ...     sl = smm.ShareableList(range(2000))
    ...     # Divide the work among two processes, storing partial results in sl
@@ -346,9 +355,7 @@ instance:
    'dry ice'
    >>> a[2] = 'larger than previously allocated storage space'
    Traceback (most recent call last):
-     File "<stdin>", line 1, in <module>
-     File "/usr/local/lib/python3.8/multiprocessing/shared_memory.py", line 429, in __setitem__
-       raise ValueError("exceeds available storage for existing str")
+     ...
    ValueError: exceeds available storage for existing str
    >>> a[2]
    'dry ice'
@@ -371,7 +378,7 @@ behind it:
    >>> b = shared_memory.ShareableList(range(5))         # In a first process
    >>> c = shared_memory.ShareableList(name=b.shm.name)  # In a second process
    >>> c
-   ShareableList([0, 1, 2, 3, 4], name='psm_25144_23776')
+   ShareableList([0, 1, 2, 3, 4], name='...')
    >>> c[-1] = -999
    >>> b[-1]
    -999

From 107656716917bd68bded9f746c6b031a01ecaa39 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 22:15:42 -0600
Subject: [PATCH 18/41] CloseHandle should be in a finally block in case
 MapViewOfFile fails.

---
 Lib/multiprocessing/shared_memory.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 8d2fe2d05d4dbe..de9e640fcfe49c 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -98,8 +98,10 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
                 h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
             except OSError as ose:
                 raise ExistentialError(*ose.args)
-            p_buf = kernel32.MapViewOfFile(h_map, FILE_MAP_READ, 0, 0, 0)
-            kernel32.CloseHandle(h_map)
+            try:
+                p_buf = kernel32.MapViewOfFile(h_map, FILE_MAP_READ, 0, 0, 0)
+            finally:
+                kernel32.CloseHandle(h_map)
             mbi = MEMORY_BASIC_INFORMATION()
             kernel32.VirtualQuery(p_buf, ctypes.byref(mbi), mmap.PAGESIZE)
             size = mbi.RegionSize

From 0be05318219ca1872f07af45404aca5619c3f0d5 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Mon, 11 Feb 2019 22:50:25 -0600
Subject: [PATCH 19/41] Missed opportunity to use with statement.

---
 Lib/multiprocessing/shared_memory.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index de9e640fcfe49c..a93a67a73d3d33 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -728,31 +728,25 @@ def get_server(self):
     def SharedMemory(self, size):
         """Returns a new SharedMemory instance with the specified size in
         bytes, to be tracked by the manager."""
-        conn = self._Client(self._address, authkey=self._authkey)
-        try:
+        with self._Client(self._address, authkey=self._authkey) as conn:
             sms = SharedMemory(None, flags=O_CREX, size=size)
             try:
                 dispatch(conn, None, 'track_segment', (sms.name,))
             except BaseException as e:
                 sms.unlink()
                 raise e
-        finally:
-            conn.close()
         return sms
 
     def ShareableList(self, sequence):
         """Returns a new ShareableList instance populated with the values
         from the input sequence, to be tracked by the manager."""
-        conn = self._Client(self._address, authkey=self._authkey)
-        try:
+        with self._Client(self._address, authkey=self._authkey) as conn:
             sl = ShareableList(sequence)
             try:
                 dispatch(conn, None, 'track_segment', (sl.shm.name,))
             except BaseException as e:
                 sl.shm.unlink()
                 raise e
-        finally:
-            conn.close()
         return sl
 
 SharedMemoryManager.register('Barrier', threading.Barrier, BarrierProxy)

From 1e5341eb171d7d2b988880020cfcc0a64021326d Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Tue, 12 Feb 2019 13:33:02 -0600
Subject: [PATCH 20/41] Switch to self.addCleanup to spare long try/finally
 blocks and save one indentation, change to use decorator to skip test
 instead.

---
 Lib/test/_test_multiprocessing.py | 284 ++++++++++++++----------------
 1 file changed, 136 insertions(+), 148 deletions(-)

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 4ce2ef0a2a1f5b..6db77fd859eb80 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3617,14 +3617,11 @@ def test_copy(self):
         self.assertEqual(bar.z, 2 ** 33)
 
 
+@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory")
 class _TestSharedMemory(BaseTestCase):
 
     ALLOWED_TYPES = ('processes',)
 
-    def setUp(self):
-        if not HAS_SHMEM:
-            self.skipTest("requires multiprocessing.shared_memory")
-
     @staticmethod
     def _attach_existing_shmem_then_write(shmem_name, binary_data):
         local_sms = shared_memory.SharedMemory(shmem_name)
@@ -3637,84 +3634,81 @@ def test_shared_memory_basics(self):
             flags=shared_memory.O_CREX,
             size=512
         )
-        try:
-            # Verify attributes are readable.
-            self.assertEqual(sms.name, 'test01_tsmb')
-            self.assertGreaterEqual(sms.size, 512)
-            self.assertGreaterEqual(len(sms.buf), sms.size)
-            self.assertEqual(sms.mode, 0o600)
-
-            # Modify contents of shared memory segment through memoryview.
-            sms.buf[0] = 42
-            self.assertEqual(sms.buf[0], 42)
-
-            # Attach to existing shared memory segment.
-            also_sms = shared_memory.SharedMemory('test01_tsmb')
-            self.assertEqual(also_sms.buf[0], 42)
-            also_sms.close()
-
-            if isinstance(sms, shared_memory.PosixSharedMemory):
-                # Posix Shared Memory can only be unlinked once.  Here we
-                # test an implementation detail that is not observed across
-                # all supported platforms (since WindowsNamedSharedMemory
-                # manages unlinking on its own and unlink() does nothing).
-                # True release of shared memory segment does not necessarily
-                # happen until process exits, depending on the OS platform.
-                with self.assertRaises(shared_memory.ExistentialError):
-                    sms_uno = shared_memory.SharedMemory(
-                        'test01_dblunlink',
-                        flags=shared_memory.O_CREX,
-                        size=5000
-                    )
-
-                    try:
-                        self.assertGreaterEqual(sms_uno.size, 5000)
+        self.addCleanup(sms.unlink)
+
+        # Verify attributes are readable.
+        self.assertEqual(sms.name, 'test01_tsmb')
+        self.assertGreaterEqual(sms.size, 512)
+        self.assertGreaterEqual(len(sms.buf), sms.size)
+        self.assertEqual(sms.mode, 0o600)
+
+        # Modify contents of shared memory segment through memoryview.
+        sms.buf[0] = 42
+        self.assertEqual(sms.buf[0], 42)
+
+        # Attach to existing shared memory segment.
+        also_sms = shared_memory.SharedMemory('test01_tsmb')
+        self.assertEqual(also_sms.buf[0], 42)
+        also_sms.close()
+
+        if isinstance(sms, shared_memory.PosixSharedMemory):
+            # Posix Shared Memory can only be unlinked once.  Here we
+            # test an implementation detail that is not observed across
+            # all supported platforms (since WindowsNamedSharedMemory
+            # manages unlinking on its own and unlink() does nothing).
+            # True release of shared memory segment does not necessarily
+            # happen until process exits, depending on the OS platform.
+            with self.assertRaises(shared_memory.ExistentialError):
+                sms_uno = shared_memory.SharedMemory(
+                    'test01_dblunlink',
+                    flags=shared_memory.O_CREX,
+                    size=5000
+                )
 
-                        sms_duo = shared_memory.SharedMemory('test01_dblunlink')
-                        sms_duo.unlink()  # First shm_unlink() call.
-                        sms_duo.close()
-                        sms_uno.close()
+                try:
+                    self.assertGreaterEqual(sms_uno.size, 5000)
 
-                    finally:
-                        sms_uno.unlink()  # A second shm_unlink() call is bad.
+                    sms_duo = shared_memory.SharedMemory('test01_dblunlink')
+                    sms_duo.unlink()  # First shm_unlink() call.
+                    sms_duo.close()
+                    sms_uno.close()
 
-            # Enforcement of `mode` and `read_only` is OS platform dependent
-            # and as such will not be tested here.
+                finally:
+                    sms_uno.unlink()  # A second shm_unlink() call is bad.
 
-            with self.assertRaises(shared_memory.ExistentialError):
-                # Attempting to create a new shared memory segment with a
-                # name that is already in use triggers an exception.
-                there_can_only_be_one_sms = shared_memory.SharedMemory(
-                    'test01_tsmb',
-                    flags=shared_memory.O_CREX,
-                    size=512
-                )
+        # Enforcement of `mode` and `read_only` is OS platform dependent
+        # and as such will not be tested here.
 
-            # Requesting creation of a shared memory segment with the option
-            # to attach to an existing segment, if that name is currently in
-            # use, should not trigger an exception.
-            # Note:  Using a smaller size could possibly cause truncation of
-            # the existing segment but is OS platform dependent.  In the
-            # case of MacOS/darwin, requesting a smaller size is disallowed.
-            ok_if_exists_sms = shared_memory.SharedMemory(
+        with self.assertRaises(shared_memory.ExistentialError):
+            # Attempting to create a new shared memory segment with a
+            # name that is already in use triggers an exception.
+            there_can_only_be_one_sms = shared_memory.SharedMemory(
                 'test01_tsmb',
-                flags=shared_memory.O_CREAT,
-                size=sms.size if sys.platform != 'darwin' else 0
+                flags=shared_memory.O_CREX,
+                size=512
             )
-            self.assertEqual(ok_if_exists_sms.size, sms.size)
-            ok_if_exists_sms.close()
 
-            # Attempting to attach to an existing shared memory segment when
-            # no segment exists with the supplied name triggers an exception.
-            with self.assertRaises(shared_memory.ExistentialError):
-                nonexisting_sms = shared_memory.SharedMemory('test01_notthere')
-                nonexisting_sms.unlink()  # Error should occur on prior line.
+        # Requesting creation of a shared memory segment with the option
+        # to attach to an existing segment, if that name is currently in
+        # use, should not trigger an exception.
+        # Note:  Using a smaller size could possibly cause truncation of
+        # the existing segment but is OS platform dependent.  In the
+        # case of MacOS/darwin, requesting a smaller size is disallowed.
+        ok_if_exists_sms = shared_memory.SharedMemory(
+            'test01_tsmb',
+            flags=shared_memory.O_CREAT,
+            size=sms.size if sys.platform != 'darwin' else 0
+        )
+        self.assertEqual(ok_if_exists_sms.size, sms.size)
+        ok_if_exists_sms.close()
 
-            sms.close()
+        # Attempting to attach to an existing shared memory segment when
+        # no segment exists with the supplied name triggers an exception.
+        with self.assertRaises(shared_memory.ExistentialError):
+            nonexisting_sms = shared_memory.SharedMemory('test01_notthere')
+            nonexisting_sms.unlink()  # Error should occur on prior line.
 
-        finally:
-            # Prevent test failures from leading to a dangling segment.
-            sms.unlink()
+        sms.close()
 
     def test_shared_memory_across_processes(self):
         sms = shared_memory.SharedMemory(
@@ -3722,21 +3716,18 @@ def test_shared_memory_across_processes(self):
             flags=shared_memory.O_CREX,
             size=512
         )
+        self.addCleanup(sms.unlink)
 
-        try:
-            p = self.Process(
-                target=self._attach_existing_shmem_then_write,
-                args=(sms.name, b'howdy')
-            )
-            p.daemon = True
-            p.start()
-            p.join()
-            self.assertEqual(bytes(sms.buf[:5]), b'howdy')
-
-            sms.close()
+        p = self.Process(
+            target=self._attach_existing_shmem_then_write,
+            args=(sms.name, b'howdy')
+        )
+        p.daemon = True
+        p.start()
+        p.join()
+        self.assertEqual(bytes(sms.buf[:5]), b'howdy')
 
-        finally:
-            sms.unlink()
+        sms.close()
 
     def test_shared_memory_SharedMemoryManager_basics(self):
         smm1 = shared_memory.SharedMemoryManager()
@@ -3774,75 +3765,72 @@ def test_shared_memory_ShareableList_basics(self):
         sl = shared_memory.ShareableList(
             ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
         )
+        self.addCleanup(sl.shm.unlink)
 
-        try:
-            # Verify attributes are readable.
-            self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
-
-            # Exercise len().
-            self.assertEqual(len(sl), 7)
-
-            # Exercise index().
-            with warnings.catch_warnings():
-                # Suppress BytesWarning when comparing against b'HoWdY'.
-                warnings.simplefilter('ignore')
-                with self.assertRaises(ValueError):
-                    sl.index('100')
-                self.assertEqual(sl.index(100), 3)
-
-            # Exercise retrieving individual values.
-            self.assertEqual(sl[0], 'howdy')
-            self.assertEqual(sl[-2], True)
-
-            # Exercise iterability.
-            self.assertEqual(
-                tuple(sl),
-                ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
-            )
+        # Verify attributes are readable.
+        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
 
-            # Exercise modifying individual values.
-            sl[3] = 42
-            self.assertEqual(sl[3], 42)
-            sl[4] = 'some'  # Change type at a given position.
-            self.assertEqual(sl[4], 'some')
-            self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+        # Exercise len().
+        self.assertEqual(len(sl), 7)
+
+        # Exercise index().
+        with warnings.catch_warnings():
+            # Suppress BytesWarning when comparing against b'HoWdY'.
+            warnings.simplefilter('ignore')
             with self.assertRaises(ValueError):
-                sl[4] = 'far too many'  # Exceeds available storage.
-            self.assertEqual(sl[4], 'some')
-
-            # Exercise count().
-            with warnings.catch_warnings():
-                # Suppress BytesWarning when comparing against b'HoWdY'.
-                warnings.simplefilter('ignore')
-                self.assertEqual(sl.count(42), 2)
-                self.assertEqual(sl.count(b'HoWdY'), 1)
-                self.assertEqual(sl.count(b'adios'), 0)
-
-            # Exercise creating a duplicate.
-            sl_copy = sl.copy(name='test03_duplicate')
-            try:
-                self.assertNotEqual(sl.shm.name, sl_copy.shm.name)
-                self.assertEqual('test03_duplicate', sl_copy.shm.name)
-                self.assertEqual(list(sl), list(sl_copy))
-                self.assertEqual(sl.format, sl_copy.format)
-                sl_copy[-1] = 77
-                self.assertEqual(sl_copy[-1], 77)
-                self.assertNotEqual(sl[-1], 77)
-                sl_copy.shm.close()
-            finally:
-                sl_copy.shm.unlink()
+                sl.index('100')
+            self.assertEqual(sl.index(100), 3)
+
+        # Exercise retrieving individual values.
+        self.assertEqual(sl[0], 'howdy')
+        self.assertEqual(sl[-2], True)
 
-            # Obtain a second handle on the same ShareableList.
-            sl_tethered = shared_memory.ShareableList(name=sl.shm.name)
-            self.assertEqual(sl.shm.name, sl_tethered.shm.name)
-            sl_tethered[-1] = 880
-            self.assertEqual(sl[-1], 880)
-            sl_tethered.shm.close()
+        # Exercise iterability.
+        self.assertEqual(
+            tuple(sl),
+            ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
+        )
 
+        # Exercise modifying individual values.
+        sl[3] = 42
+        self.assertEqual(sl[3], 42)
+        sl[4] = 'some'  # Change type at a given position.
+        self.assertEqual(sl[4], 'some')
+        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+        with self.assertRaises(ValueError):
+            sl[4] = 'far too many'  # Exceeds available storage.
+        self.assertEqual(sl[4], 'some')
+
+        # Exercise count().
+        with warnings.catch_warnings():
+            # Suppress BytesWarning when comparing against b'HoWdY'.
+            warnings.simplefilter('ignore')
+            self.assertEqual(sl.count(42), 2)
+            self.assertEqual(sl.count(b'HoWdY'), 1)
+            self.assertEqual(sl.count(b'adios'), 0)
+
+        # Exercise creating a duplicate.
+        sl_copy = sl.copy(name='test03_duplicate')
+        try:
+            self.assertNotEqual(sl.shm.name, sl_copy.shm.name)
+            self.assertEqual('test03_duplicate', sl_copy.shm.name)
+            self.assertEqual(list(sl), list(sl_copy))
+            self.assertEqual(sl.format, sl_copy.format)
+            sl_copy[-1] = 77
+            self.assertEqual(sl_copy[-1], 77)
+            self.assertNotEqual(sl[-1], 77)
+            sl_copy.shm.close()
         finally:
-            # Prevent test failures from leading to a dangling segment.
-            sl.shm.unlink()
-            sl.shm.close()
+            sl_copy.shm.unlink()
+
+        # Obtain a second handle on the same ShareableList.
+        sl_tethered = shared_memory.ShareableList(name=sl.shm.name)
+        self.assertEqual(sl.shm.name, sl_tethered.shm.name)
+        sl_tethered[-1] = 880
+        self.assertEqual(sl[-1], 880)
+        sl_tethered.shm.close()
+
+        sl.shm.close()
 
         # Exercise creating an empty ShareableList.
         empty_sl = shared_memory.ShareableList()

From a5800a910fd96d113f029754d7b1cb3aee4e22b9 Mon Sep 17 00:00:00 2001
From: Neil Schemenauer <nas@arctrix.com>
Date: Wed, 13 Feb 2019 12:08:26 -0800
Subject: [PATCH 21/41] Simplify the posixshmem extension module.

Provide shm_open() and shm_unlink() functions.  Move other
functionality into the shared_memory.py module.
---
 Lib/multiprocessing/shared_memory.py          | 125 ++-
 Lib/test/_test_multiprocessing.py             |  10 +-
 .../_multiprocessing/clinic/posixshmem.c.h    |  92 +++
 Modules/_multiprocessing/posixshmem.c         | 740 ++----------------
 4 files changed, 263 insertions(+), 704 deletions(-)
 create mode 100644 Modules/_multiprocessing/clinic/posixshmem.c.h

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index a93a67a73d3d33..c76d16888f7a7b 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -12,23 +12,17 @@
                       BarrierProxy, AcquirerProxy, ConditionProxy, EventProxy
 from . import util
 import os
-import random
 import struct
 import sys
 import threading
+import secrets
 try:
-    from _posixshmem import _PosixSharedMemory, \
-                            Error, ExistentialError, PermissionsError, \
-                            O_CREAT, O_EXCL, O_CREX, O_TRUNC
+    import  _posixshmem
+    from os import O_CREAT, O_EXCL, O_TRUNC
 except ImportError as ie:
-    if os.name != "nt":
-        # On Windows, posixshmem is not required to be available.
-        raise ie
-    else:
-        _PosixSharedMemory = object
-        class Error(Exception): pass
-        class ExistentialError(Error): pass
-        O_CREAT, O_EXCL, O_CREX, O_TRUNC = 64, 128, 192, 512
+    O_CREAT, O_EXCL, O_TRUNC = 64, 128, 512
+
+O_CREX = O_CREAT | O_EXCL
 
 if os.name == "nt":
     import ctypes
@@ -89,15 +83,15 @@ class WindowsNamedSharedMemory:
 
     def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if name is None:
-            name = f'wnsm_{os.getpid()}_{random.randrange(100000)}'
+            name = _make_filename()
 
         if size == 0:
             # Attempt to dynamically determine the existing named shared
             # memory block's size which is likely a multiple of mmap.PAGESIZE.
             try:
                 h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
-            except OSError as ose:
-                raise ExistentialError(*ose.args)
+            except OSError:
+                raise FileNotFoundError(name)
             try:
                 p_buf = kernel32.MapViewOfFile(h_map, FILE_MAP_READ, 0, 0, 0)
             finally:
@@ -115,9 +109,7 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             except OSError as ose:
                 name_collision = False
             if name_collision:
-                raise ExistentialError(
-                    f"Shared memory already exists with name={name}"
-                )
+                raise FileExistsError(name)
 
         self._mmap = mmap.mmap(-1, size, tagname=name)
         self.buf = memoryview(self._mmap)
@@ -138,34 +130,99 @@ def unlink(self):
         pass
 
 
-class PosixSharedMemory(_PosixSharedMemory):
+# FreeBSD (and perhaps other BSDs) limit names to 14 characters.
+_SHM_SAFE_NAME_LENGTH = 14
+
+# shared object name prefix
+if os.name == "nt":
+    _SHM_NAME_PREFIX = 'wnsm_'
+else:
+    _SHM_NAME_PREFIX = '/psm_'
+
+
+def _make_filename():
+    """Create a random filename for the shared memory object.
+    """
+    # number of random bytes to use for name
+    nbytes = (_SHM_SAFE_NAME_LENGTH - len(_SHM_NAME_PREFIX)) // 2
+    assert nbytes >= 2, '_SHM_NAME_PREFIX too long'
+    name = _SHM_NAME_PREFIX + secrets.token_hex(nbytes)
+    assert len(name) <= _SHM_SAFE_NAME_LENGTH
+    return name
+
+
+class PosixSharedMemory:
+
+    # defaults so close() and unlink() can run without errors
+    fd = -1
+    name = None
+    _mmap = None
+    buf = None
 
     def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if name and (flags is None):
-            _PosixSharedMemory.__init__(self, name, mode=mode)
+            flags = 0
         else:
-            if name is None:
-                name = f'psm_{os.getpid()}_{random.randrange(100000)}'
             flags = O_CREX if flags is None else flags
-            _PosixSharedMemory.__init__(
-                self,
-                name,
-                flags=flags,
-                mode=mode,
-                size=size,
-                read_only=read_only
-            )
-
+        if flags & O_EXCL and not flags & O_CREAT:
+            raise ValueError("O_EXCL must be combined with O_CREAT")
+        if name is None and not flags & O_EXCL:
+            raise ValueError("'name' can only be None if O_EXCL is set")
+        flags |= os.O_RDONLY if read_only else os.O_RDWR
+        self.flags = flags
+        self.mode = mode
+        if not size >= 0:
+            raise ValueError("'size' must be a positive integer")
+        self.size = size
+        if name is None:
+            self._open_retry()
+        else:
+            self.name = name
+            self.fd = _posixshmem.shm_open(name, flags, mode=mode)
+        if self.size:
+            try:
+                os.ftruncate(self.fd, self.size)
+            except OSError:
+                self.unlink()
+                raise
         self._mmap = mmap.mmap(self.fd, self.size)
         self.buf = memoryview(self._mmap)
 
+    def _open_retry(self):
+        # generate a random name, open, retry if it exists
+        while True:
+            name = _make_filename()
+            try:
+                self.fd = _posixshmem.shm_open(name, self.flags,
+                                               mode=self.mode)
+            except FileExistsError:
+                continue
+            self.name = name
+            break
+
     def __repr__(self):
         return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
 
+    def unlink(self):
+        if self.name:
+            _posixshmem.shm_unlink(self.name)
+
     def close(self):
-        self.buf.release()
-        self._mmap.close()
-        self.close_fd()
+        if self.buf is not None:
+            self.buf.release()
+            self.buf = None
+        if self._mmap is not None:
+            self._mmap.close()
+            self._mmap = None
+        if self.fd >= 0:
+            os.close(self.fd)
+            self.fd = -1
+
+    def __del__(self):
+        try:
+            self.close()
+        except OSError:
+            pass
 
 
 class SharedMemory:
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 6db77fd859eb80..0daf1c673c1610 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3658,7 +3658,7 @@ def test_shared_memory_basics(self):
             # manages unlinking on its own and unlink() does nothing).
             # True release of shared memory segment does not necessarily
             # happen until process exits, depending on the OS platform.
-            with self.assertRaises(shared_memory.ExistentialError):
+            with self.assertRaises(FileNotFoundError):
                 sms_uno = shared_memory.SharedMemory(
                     'test01_dblunlink',
                     flags=shared_memory.O_CREX,
@@ -3679,7 +3679,7 @@ def test_shared_memory_basics(self):
         # Enforcement of `mode` and `read_only` is OS platform dependent
         # and as such will not be tested here.
 
-        with self.assertRaises(shared_memory.ExistentialError):
+        with self.assertRaises(FileExistsError):
             # Attempting to create a new shared memory segment with a
             # name that is already in use triggers an exception.
             there_can_only_be_one_sms = shared_memory.SharedMemory(
@@ -3704,7 +3704,7 @@ def test_shared_memory_basics(self):
 
         # Attempting to attach to an existing shared memory segment when
         # no segment exists with the supplied name triggers an exception.
-        with self.assertRaises(shared_memory.ExistentialError):
+        with self.assertRaises(FileNotFoundError):
             nonexisting_sms = shared_memory.SharedMemory('test01_notthere')
             nonexisting_sms.unlink()  # Error should occur on prior line.
 
@@ -3745,7 +3745,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
         if sys.platform != "win32":
             # Calls to unlink() have no effect on Windows platform; shared
             # memory will only be released once final process exits.
-            with self.assertRaises(shared_memory.ExistentialError):
+            with self.assertRaises(FileNotFoundError):
                 # No longer there to be attached to again.
                 absent_shm = shared_memory.SharedMemory(name=held_name)
 
@@ -3756,7 +3756,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
                 shm = smm2.SharedMemory(size=128)
             held_name = sl.shm.name
         if sys.platform != "win32":
-            with self.assertRaises(shared_memory.ExistentialError):
+            with self.assertRaises(FileNotFoundError):
                 # No longer there to be attached to again.
                 absent_sl = shared_memory.ShareableList(name=held_name)
 
diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h
new file mode 100644
index 00000000000000..20abddc0a2e975
--- /dev/null
+++ b/Modules/_multiprocessing/clinic/posixshmem.c.h
@@ -0,0 +1,92 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+#if defined(HAVE_SHM_OPEN)
+
+PyDoc_STRVAR(_posixshmem_shm_open__doc__,
+"shm_open($module, /, path, flags, mode=511)\n"
+"--\n"
+"\n"
+"Open a shared memory object.  Returns a file descriptor (integer).");
+
+#define _POSIXSHMEM_SHM_OPEN_METHODDEF    \
+    {"shm_open", (PyCFunction)(void(*)(void))_posixshmem_shm_open, METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_open__doc__},
+
+static int
+_posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags,
+                          int mode);
+
+static PyObject *
+_posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"path", "flags", "mode", NULL};
+    static _PyArg_Parser _parser = {"Ui|i:shm_open", _keywords, 0};
+    PyObject *path;
+    int flags;
+    int mode = 511;
+    int _return_value;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &path, &flags, &mode)) {
+        goto exit;
+    }
+    _return_value = _posixshmem_shm_open_impl(module, path, flags, mode);
+    if ((_return_value == -1) && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = PyLong_FromLong((long)_return_value);
+
+exit:
+    return return_value;
+}
+
+#endif /* defined(HAVE_SHM_OPEN) */
+
+#if defined(HAVE_SHM_UNLINK)
+
+PyDoc_STRVAR(_posixshmem_shm_unlink__doc__,
+"shm_unlink($module, /, path)\n"
+"--\n"
+"\n"
+"Remove a shared memory object (similar to unlink()).\n"
+"\n"
+"Remove a shared memory object name, and, once all processes  have  unmapped\n"
+"the object, de-allocates and destroys the contents of the associated memory\n"
+"region.");
+
+#define _POSIXSHMEM_SHM_UNLINK_METHODDEF    \
+    {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_unlink__doc__},
+
+static PyObject *
+_posixshmem_shm_unlink_impl(PyObject *module, PyObject *path);
+
+static PyObject *
+_posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"path", NULL};
+    static _PyArg_Parser _parser = {"U:shm_unlink", _keywords, 0};
+    PyObject *path;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &path)) {
+        goto exit;
+    }
+    return_value = _posixshmem_shm_unlink_impl(module, path);
+
+exit:
+    return return_value;
+}
+
+#endif /* defined(HAVE_SHM_UNLINK) */
+
+#ifndef _POSIXSHMEM_SHM_OPEN_METHODDEF
+    #define _POSIXSHMEM_SHM_OPEN_METHODDEF
+#endif /* !defined(_POSIXSHMEM_SHM_OPEN_METHODDEF) */
+
+#ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF
+    #define _POSIXSHMEM_SHM_UNLINK_METHODDEF
+#endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */
+/*[clinic end generated code: output=ff9cf0bc9b8baddf input=a9049054013a1b77]*/
diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c
index b4337d2fb07850..2049dbbc6fa83b 100644
--- a/Modules/_multiprocessing/posixshmem.c
+++ b/Modules/_multiprocessing/posixshmem.c
@@ -1,31 +1,5 @@
 /*
-posixshmem - A Python module for accessing POSIX 1003.1b-1993 shared memory.
-
-Copyright (c) 2012, Philip Semanchuk
-Copyright (c) 2018, 2019, Davin Potts
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of posixshmem nor the names of its contributors may
-      be used to endorse or promote products derived from this software
-      without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY ITS CONTRIBUTORS ''AS IS'' AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL Philip Semanchuk BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+posixshmem - A Python extension that provides shm_open() and shm_unlink()
 */
 
 #define PY_SSIZE_T_CLEAN
@@ -33,627 +7,106 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <Python.h>
 #include "structmember.h"
 
-#include <time.h>
-#include <sys/time.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <stdio.h>
-
-// For shared memory stuff
-#include <sys/stat.h>
+// for shm_open() and shm_unlink()
+#ifdef HAVE_SYS_MMAN_H
 #include <sys/mman.h>
-
-/* SEM_FAILED is defined as an int in Apple's headers, and this makes the
-compiler complain when I compare it to a pointer. Python faced the same
-problem (issue 9586) and I copied their solution here.
-ref: http://bugs.python.org/issue9586
-
-Note that in /Developer/SDKs/MacOSX10.4u.sdk/usr/include/sys/semaphore.h,
-SEM_FAILED is #defined as -1 and that's apparently the definition used by
-Python when building. In /usr/include/sys/semaphore.h, it's defined
-as ((sem_t *)-1).
-*/
-#ifdef __APPLE__
-    #undef  SEM_FAILED
-    #define SEM_FAILED ((sem_t *)-1)
 #endif
 
-/* POSIX says that a mode_t "shall be an integer type". To avoid the need
-for a specific get_mode function for each type, I'll just stuff the mode into
-a long and mention it in the Xxx_members list for each type.
-ref: http://www.opengroup.org/onlinepubs/000095399/basedefs/sys/types.h.html
-*/
-
-typedef struct {
-    PyObject_HEAD
-    char *name;
-    long mode;
-    int fd;
-} SharedMemory;
-
-
-// FreeBSD (and perhaps other BSDs) limit names to 14 characters. In the
-// code below, strings of this length are allocated on the stack, so
-// increase this gently or change that code to use malloc().
-#define MAX_SAFE_NAME_LENGTH  14
-
-
-/* Struct to contain an IPC object name which can be None */
-typedef struct {
-    int is_none;
-    char *name;
-} NoneableName;
-
+/*[clinic input]
+module _posixshmem
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a416734e49164bf8]*/
 
 /*
-      Exceptions for this module
-*/
-
-static PyObject *pBaseException;
-static PyObject *pPermissionsException;
-static PyObject *pExistentialException;
-
-
-#ifdef POSIX_IPC_DEBUG
-#define DPRINTF(fmt, args...) fprintf(stderr, "+++ " fmt, ## args)
-#else
-#define DPRINTF(fmt, args...)
-#endif
-
-static char *
-bytes_to_c_string(PyObject* o, int lock) {
-/* Convert a bytes object to a char *. Optionally lock the buffer if it is a
-   bytes array.
-   This code swiped directly from Python 3.1's posixmodule.c by Philip S.
-   The name there is bytes2str().
-*/
-    if (PyBytes_Check(o))
-        return PyBytes_AsString(o);
-    else if (PyByteArray_Check(o)) {
-        if (lock && PyObject_GetBuffer(o, NULL, 0) < 0)
-            /* On a bytearray, this should not fail. */
-            PyErr_BadInternalCall();
-        return PyByteArray_AsString(o);
-    } else {
-        /* The FS converter should have verified that this
-           is either bytes or bytearray. */
-        Py_FatalError("bad object passed to bytes2str");
-        /* not reached. */
-        return "";
-    }
-}
-
-static void
-release_bytes(PyObject* o)
-    /* Release the lock, decref the object.
-   This code swiped directly from Python 3.1's posixmodule.c by Philip S.
-   */
-{
-    if (PyByteArray_Check(o))
-        o->ob_type->tp_as_buffer->bf_releasebuffer(NULL, 0);
-    Py_DECREF(o);
-}
-
-
-static int
-random_in_range(int min, int max) {
-    // returns a random int N such that min <= N <= max
-    int diff = (max - min) + 1;
-
-    // ref: http://www.c-faq.com/lib/randrange.html
-    return ((int)((double)rand() / ((double)RAND_MAX + 1) * diff)) + min;
-}
-
-
-static
-int create_random_name(char *name) {
-    // The random name is always lowercase so that this code will work
-    // on case-insensitive file systems. It always starts with a forward
-    // slash.
-    int length;
-    char *alphabet = "abcdefghijklmnopqrstuvwxyz";
-    int i;
+ *
+ * Module-level functions & meta stuff
+ *
+ */
 
-    // Generate a random length for the name. I subtract 1 from the
-    // MAX_SAFE_NAME_LENGTH in order to allow for the name's leading "/".
-    length = random_in_range(6, MAX_SAFE_NAME_LENGTH - 1);
+#ifdef HAVE_SHM_OPEN
+/*[clinic input]
+_posixshmem.shm_open -> int
+    path: unicode
+    flags: int
+    mode: int = 0o777
 
-    name[0] = '/';
-    name[length] = '\0';
-    i = length;
-    while (--i)
-        name[i] = alphabet[random_in_range(0, 25)];
+# "shm_open(path, flags, mode=0o777)\n\n\
 
-    return length;
-}
+Open a shared memory object.  Returns a file descriptor (integer).
 
+[clinic start generated code]*/
 
 static int
-convert_name_param(PyObject *py_name_param, void *checked_name) {
-    /* Verifies that the py_name_param is either None or a string.
-    If it's a string, checked_name->name points to a PyMalloc-ed buffer
-    holding a NULL-terminated C version of the string when this function
-    concludes. The caller is responsible for releasing the buffer.
-    */
-    int rc = 0;
-    NoneableName *p_name = (NoneableName *)checked_name;
-    PyObject *py_name_as_bytes = NULL;
-    char *p_name_as_c_string = NULL;
-
-    DPRINTF("inside convert_name_param\n");
-    DPRINTF("PyBytes_Check() = %d \n", PyBytes_Check(py_name_param));
-    DPRINTF("PyString_Check() = %d \n", PyString_Check(py_name_param));
-    DPRINTF("PyUnicode_Check() = %d \n", PyUnicode_Check(py_name_param));
-
-    p_name->is_none = 0;
-
-    // The name can be None or a Python string
-    if (py_name_param == Py_None) {
-        DPRINTF("name is None\n");
-        rc = 1;
-        p_name->is_none = 1;
+_posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags,
+                          int mode)
+/*[clinic end generated code: output=8d110171a4fa20df input=e83b58fa802fac25]*/
+{
+    int fd;
+    int async_err = 0;
+    const char *name = PyUnicode_AsUTF8(path);
+    if (name == NULL) {
+        return -1;
     }
-    else if (PyUnicode_Check(py_name_param) || PyBytes_Check(py_name_param)) {
-        DPRINTF("name is Unicode or bytes\n");
-        // The caller passed me a Unicode string or a byte array; I need a
-        // char *. Getting from one to the other takes a couple steps.
-
-        if (PyUnicode_Check(py_name_param)) {
-            DPRINTF("name is Unicode\n");
-            // PyUnicode_FSConverter() converts the Unicode object into a
-            // bytes or a bytearray object. (Why can't it be one or the other?)
-            PyUnicode_FSConverter(py_name_param, &py_name_as_bytes);
-        }
-        else {
-            DPRINTF("name is bytes\n");
-            // Make a copy of the name param.
-            py_name_as_bytes = PyBytes_FromObject(py_name_param);
-        }
-
-        // bytes_to_c_string() returns a pointer to the buffer.
-        p_name_as_c_string = bytes_to_c_string(py_name_as_bytes, 0);
-
-        // PyMalloc memory and copy the user-supplied name to it.
-        p_name->name = (char *)PyMem_Malloc(strlen(p_name_as_c_string) + 1);
-        if (p_name->name) {
-            rc = 1;
-            strcpy(p_name->name, p_name_as_c_string);
-        }
-        else
-            PyErr_SetString(PyExc_MemoryError, "Out of memory");
-
-        // The bytes version of the name isn't useful to me, and per the
-        // documentation for PyUnicode_FSConverter(), I am responsible for
-        // releasing it when I'm done.
-        release_bytes(py_name_as_bytes);
+    do {
+        Py_BEGIN_ALLOW_THREADS
+        fd = shm_open(name, flags, mode);
+        Py_END_ALLOW_THREADS
+    } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
+
+    if (fd < 0) {
+        if (!async_err)
+            PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path);
+        return -1;
     }
-    else
-        PyErr_SetString(PyExc_TypeError, "Name must be None or a string");
-
-    return rc;
-}
-
-
-
-/*   =====  Begin Shared Memory implementation functions ===== */
-
-static PyObject *
-shm_str(SharedMemory *self) {
-    return PyUnicode_FromString(self->name ? self->name : "(no name)");
-}
 
-static PyObject *
-shm_repr(SharedMemory *self) {
-    char mode[32];
-
-    sprintf(mode, "0%o", (int)(self->mode));
-
-    return PyUnicode_FromFormat("_posixshmem.SharedMemory(\"%s\", mode=%s)",
-                                self->name, mode);
+    return fd;
 }
+#endif /* HAVE_SHM_OPEN */
 
-static PyObject *
-my_shm_unlink(const char *name) {
-    DPRINTF("unlinking shm name %s\n", name);
-    if (-1 == shm_unlink(name)) {
-        switch (errno) {
-            case EACCES:
-                PyErr_SetString(pPermissionsException, "Permission denied");
-            break;
-
-            case ENOENT:
-                PyErr_SetString(pExistentialException,
-                    "No shared memory exists with the specified name");
-            break;
-
-            case ENAMETOOLONG:
-                PyErr_SetString(PyExc_ValueError, "The name is too long");
-            break;
+#ifdef HAVE_SHM_UNLINK
+/*[clinic input]
+_posixshmem.shm_unlink
+    path: unicode
 
-            default:
-                PyErr_SetFromErrno(PyExc_OSError);
-            break;
-        }
+Remove a shared memory object (similar to unlink()).
 
-        goto error_return;
-    }
-
-    Py_RETURN_NONE;
-
-    error_return:
-    return NULL;
-}
+Remove a shared memory object name, and, once all processes  have  unmapped
+the object, de-allocates and destroys the contents of the associated memory
+region.
 
+[clinic start generated code]*/
 
 static PyObject *
-SharedMemory_new(PyTypeObject *type, PyObject *args, PyObject *kwlist) {
-    SharedMemory *self;
-
-    self = (SharedMemory *)type->tp_alloc(type, 0);
-
-    return (PyObject *)self;
-}
-
-
-static int
-SharedMemory_init(SharedMemory *self, PyObject *args, PyObject *keywords) {
-    NoneableName name;
-    char temp_name[MAX_SAFE_NAME_LENGTH + 1];
-    unsigned int flags = 0;
-    unsigned long size = 0;
-    int read_only = 0;
-    static char *keyword_list[ ] = {"name", "flags", "mode", "size", "read_only", NULL};
-
-    // First things first -- initialize the self struct.
-    self->name = NULL;
-    self->fd = 0;
-    self->mode = 0600;
-
-    if (!PyArg_ParseTupleAndKeywords(args, keywords, "O&|Iiki", keyword_list,
-                                    &convert_name_param, &name, &flags,
-                                    &(self->mode), &size, &read_only))
-        goto error_return;
-
-    if ( !(flags & O_CREAT) && (flags & O_EXCL) ) {
-        PyErr_SetString(PyExc_ValueError,
-                "O_EXCL must be combined with O_CREAT");
-        goto error_return;
-    }
-
-    if (name.is_none && ((flags & O_EXCL) != O_EXCL)) {
-        PyErr_SetString(PyExc_ValueError,
-                "Name can only be None if O_EXCL is set");
-        goto error_return;
-    }
-
-    flags |= (read_only ? O_RDONLY : O_RDWR);
-
-    if (name.is_none) {
-        // (name == None) ==> generate a name for the caller
-        do {
-            errno = 0;
-            create_random_name(temp_name);
-
-            DPRINTF("calling shm_open, name=%s, flags=0x%x, mode=0%o\n",
-                        temp_name, flags, (int)self->mode);
-            self->fd = shm_open(temp_name, flags, (mode_t)self->mode);
-
-        } while ( (-1 == self->fd) && (EEXIST == errno) );
-
-        // PyMalloc memory and copy the randomly-generated name to it.
-        self->name = (char *)PyMem_Malloc(strlen(temp_name) + 1);
-        if (self->name)
-            strcpy(self->name, temp_name);
-        else {
-            PyErr_SetString(PyExc_MemoryError, "Out of memory");
-            goto error_return;
-        }
-    }
-    else {
-        // (name != None) ==> use name supplied by the caller. It was
-        // already converted to C by convert_name_param().
-        self->name = name.name;
-
-        DPRINTF("calling shm_open, name=%s, flags=0x%x, mode=0%o\n",
-                    self->name, flags, (int)self->mode);
-        self->fd = shm_open(self->name, flags, (mode_t)self->mode);
-    }
-
-    DPRINTF("shm fd = %d\n", self->fd);
-
-    if (-1 == self->fd) {
-        self->fd = 0;
-        switch (errno) {
-            case EACCES:
-                PyErr_Format(pPermissionsException,
-                                "No permission to %s this segment",
-                                (flags & O_TRUNC) ? "truncate" : "access"
-                                );
-            break;
-
-            case EEXIST:
-                PyErr_SetString(pExistentialException,
-                                "Shared memory with the specified name already exists");
-            break;
-
-            case ENOENT:
-                PyErr_SetString(pExistentialException,
-                                "No shared memory exists with the specified name");
-            break;
-
-            case EINVAL:
-                PyErr_SetString(PyExc_ValueError, "Invalid parameter(s)");
-            break;
-
-            case EMFILE:
-                PyErr_SetString(PyExc_OSError,
-                                 "This process already has the maximum number of files open");
-            break;
-
-            case ENFILE:
-                PyErr_SetString(PyExc_OSError,
-                                 "The system limit on the total number of open files has been reached");
-            break;
-
-            case ENAMETOOLONG:
-                PyErr_SetString(PyExc_ValueError,
-                                 "The name is too long");
-            break;
-
-            default:
-                PyErr_SetFromErrno(PyExc_OSError);
-            break;
-        }
-
-        goto error_return;
-    }
-    else {
-        if (size) {
-            DPRINTF("calling ftruncate, fd = %d, size = %ld\n", self->fd, size);
-            if (-1 == ftruncate(self->fd, (off_t)size)) {
-                // The code below will raise a Python error. Since that error
-                // is raised during __init__(), it will look to the caller
-                // as if object creation failed entirely. Here I clean up
-                // the system object I just created.
-                close(self->fd);
-                shm_unlink(self->name);
-
-                // ftruncate can return a ton of different errors, but most
-                // are not relevant or are extremely unlikely.
-                switch (errno) {
-                    case EINVAL:
-                        PyErr_SetString(PyExc_ValueError,
-                                        "The size is invalid or the memory is read-only");
-                    break;
-
-                    case EFBIG:
-                        PyErr_SetString(PyExc_ValueError,
-                                        "The size is too large");
-                    break;
-
-                    case EROFS:
-                    case EACCES:
-                        PyErr_SetString(pPermissionsException,
-                                        "The memory is read-only");
-                    break;
-
-                    default:
-                        PyErr_SetFromErrno(PyExc_OSError);
-                    break;
-                }
-
-                goto error_return;
-            }
-        }
-    }
-
-    return 0;
-
-    error_return:
-    return -1;
-}
-
-
-static void SharedMemory_dealloc(SharedMemory *self) {
-    DPRINTF("dealloc\n");
-    PyMem_Free(self->name);
-    self->name = NULL;
-
-    Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-
-PyObject *
-SharedMemory_getsize(SharedMemory *self, void *closure) {
-    struct stat fileinfo;
-    off_t size = -1;
-
-    if (0 == fstat(self->fd, &fileinfo))
-        size = fileinfo.st_size;
-    else {
-        switch (errno) {
-            case EBADF:
-            case EINVAL:
-                PyErr_SetString(pExistentialException,
-                                "The segment does not exist");
-            break;
-
-            default:
-                PyErr_SetFromErrno(PyExc_OSError);
-            break;
-        }
-
-        goto error_return;
+_posixshmem_shm_unlink_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/
+{
+    int rv;
+    int async_err = 0;
+    const char *name = PyUnicode_AsUTF8(path);
+    if (name == NULL) {
+        return NULL;
     }
-
-    return Py_BuildValue("k", (unsigned long)size);
-
-    error_return:
-    return NULL;
-}
-
-
-PyObject *
-SharedMemory_close_fd(SharedMemory *self) {
-    if (self->fd) {
-        if (-1 == close(self->fd)) {
-            switch (errno) {
-                case EBADF:
-                    PyErr_SetString(PyExc_ValueError,
-                                    "The file descriptor is invalid");
-                break;
-
-                default:
-                    PyErr_SetFromErrno(PyExc_OSError);
-                break;
-            }
-
-            goto error_return;
-        }
+    do {
+        Py_BEGIN_ALLOW_THREADS
+        rv = shm_unlink(name);
+        Py_END_ALLOW_THREADS
+    } while (rv < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
+
+    if (rv < 0) {
+        if (!async_err)
+            PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path);
+        return NULL;
     }
 
     Py_RETURN_NONE;
-
-    error_return:
-    return NULL;
-}
-
-
-PyObject *
-SharedMemory_unlink(SharedMemory *self) {
-    return my_shm_unlink(self->name);
-}
-
-
-/*   =====  End Shared Memory functions =====           */
-
-
-/*
- *
- * Shared memory meta stuff for describing myself to Python
- *
- */
-
-
-static PyMemberDef SharedMemory_members[] = {
-    {   "name",
-        T_STRING,
-        offsetof(SharedMemory, name),
-        READONLY,
-        "The name specified in the constructor"
-    },
-    {   "fd",
-        T_INT,
-        offsetof(SharedMemory, fd),
-        READONLY,
-        "Shared memory segment file descriptor"
-    },
-    {   "mode",
-        T_LONG,
-        offsetof(SharedMemory, mode),
-        READONLY,
-        "The mode specified in the constructor"
-    },
-    {NULL} /* Sentinel */
-};
-
-
-static PyMethodDef SharedMemory_methods[] = {
-    {   "close_fd",
-        (PyCFunction)SharedMemory_close_fd,
-        METH_NOARGS,
-        "Closes the file descriptor associated with the shared memory."
-    },
-    {   "unlink",
-        (PyCFunction)SharedMemory_unlink,
-        METH_NOARGS,
-        "Unlink (remove) the shared memory."
-    },
-    {NULL, NULL, 0, NULL} /* Sentinel */
-};
-
-
-static PyGetSetDef SharedMemory_getseters[] = {
-    // size is read-only
-    {   "size",
-        (getter)SharedMemory_getsize,
-        (setter)NULL,
-        "size",
-        NULL
-    },
-    {NULL} /* Sentinel */
-};
-
-
-static PyTypeObject SharedMemoryType = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    "_posixshmem._PosixSharedMemory",   // tp_name
-    sizeof(SharedMemory),               // tp_basicsize
-    0,                                  // tp_itemsize
-    (destructor) SharedMemory_dealloc,  // tp_dealloc
-    0,                                  // tp_print
-    0,                                  // tp_getattr
-    0,                                  // tp_setattr
-    0,                                  // tp_compare
-    (reprfunc) shm_repr,                // tp_repr
-    0,                                  // tp_as_number
-    0,                                  // tp_as_sequence
-    0,                                  // tp_as_mapping
-    0,                                  // tp_hash
-    0,                                  // tp_call
-    (reprfunc) shm_str,                 // tp_str
-    0,                                  // tp_getattro
-    0,                                  // tp_setattro
-    0,                                  // tp_as_buffer
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-                                        // tp_flags
-    "POSIX shared memory object",       // tp_doc
-    0,                                  // tp_traverse
-    0,                                  // tp_clear
-    0,                                  // tp_richcompare
-    0,                                  // tp_weaklistoffset
-    0,                                  // tp_iter
-    0,                                  // tp_iternext
-    SharedMemory_methods,               // tp_methods
-    SharedMemory_members,               // tp_members
-    SharedMemory_getseters,             // tp_getset
-    0,                                  // tp_base
-    0,                                  // tp_dict
-    0,                                  // tp_descr_get
-    0,                                  // tp_descr_set
-    0,                                  // tp_dictoffset
-    (initproc) SharedMemory_init,       // tp_init
-    0,                                  // tp_alloc
-    (newfunc) SharedMemory_new,         // tp_new
-    0,                                  // tp_free
-    0,                                  // tp_is_gc
-    0                                   // tp_bases
-};
-
-
-/*
- *
- * Module-level functions & meta stuff
- *
- */
-
-static PyObject *
-posixshmem_unlink_shared_memory(PyObject *self, PyObject *args) {
-    const char *name;
-
-    if (!PyArg_ParseTuple(args, "s", &name))
-        return NULL;
-    else
-        return my_shm_unlink(name);
 }
+#endif /* HAVE_SHM_UNLINK */
 
+#include "clinic/posixshmem.c.h"
 
 static PyMethodDef module_methods[ ] = {
-    {   "unlink_shared_memory",
-        (PyCFunction)posixshmem_unlink_shared_memory,
-        METH_VARARGS,
-        "Unlink shared memory"
-    },
+    _POSIXSHMEM_SHM_OPEN_METHODDEF
+    _POSIXSHMEM_SHM_UNLINK_METHODDEF
     {NULL} /* Sentinel */
 };
 
@@ -664,58 +117,15 @@ static struct PyModuleDef this_module = {
     "POSIX shared memory module",     // m_doc
     -1,                     // m_size (space allocated for module globals)
     module_methods,         // m_methods
-    NULL,                   // m_reload
-    NULL,                   // m_traverse
-    NULL,                   // m_clear
-    NULL                    // m_free
 };
 
 /* Module init function */
 PyMODINIT_FUNC
 PyInit__posixshmem(void) {
     PyObject *module;
-    PyObject *module_dict;
-
-    // I call this in case I'm asked to create any random names.
-    srand((unsigned int)time(NULL));
-
     module = PyModule_Create(&this_module);
-
-    if (!module)
-        goto error_return;
-
-    if (PyType_Ready(&SharedMemoryType) < 0)
-        goto error_return;
-
-    Py_INCREF(&SharedMemoryType);
-    PyModule_AddObject(module, "_PosixSharedMemory", (PyObject *)&SharedMemoryType);
-
-    PyModule_AddIntConstant(module, "O_CREAT", O_CREAT);
-    PyModule_AddIntConstant(module, "O_EXCL", O_EXCL);
-    PyModule_AddIntConstant(module, "O_CREX", O_CREAT | O_EXCL);
-    PyModule_AddIntConstant(module, "O_TRUNC", O_TRUNC);
-
-    if (!(module_dict = PyModule_GetDict(module)))
-        goto error_return;
-
-    // Exceptions
-    if (!(pBaseException = PyErr_NewException("_posixshmem.Error", NULL, NULL)))
-        goto error_return;
-    else
-        PyDict_SetItemString(module_dict, "Error", pBaseException);
-
-    if (!(pPermissionsException = PyErr_NewException("_posixshmem.PermissionsError", pBaseException, NULL)))
-        goto error_return;
-    else
-        PyDict_SetItemString(module_dict, "PermissionsError", pPermissionsException);
-
-    if (!(pExistentialException = PyErr_NewException("_posixshmem.ExistentialError", pBaseException, NULL)))
-        goto error_return;
-    else
-        PyDict_SetItemString(module_dict, "ExistentialError", pExistentialException);
-
+    if (!module) {
+        return NULL;
+    }
     return module;
-
-    error_return:
-    return NULL;
 }

From 34f1e9a7514553f1ee017137aacda8ee4246ee01 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 14:53:28 -0600
Subject: [PATCH 22/41] Added to doc around size parameter of SharedMemory.

---
 Doc/library/multiprocessing.shared_memory.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index fd217e29bf81d5..cdafdd7cacc4ff 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -61,6 +61,11 @@ copying of data.
    that platform's memory page size, the exact size of the shared memory
    block may be larger or equal to the size requested.  When attaching to an
    existing shared memory block, set to ``0`` (which is the default).
+   Requesting a size greater than the original when attaching to an existing
+   shared memory block will attempt a resize of the shared memory block
+   which may or may not be successful.  Requesting a size smaller than the
+   original will attempt to attach to the first N bytes of the existing
+   shared memory block but may still give access to the full allocated size.
 
    *read_only* controls whether a shared memory block is to be available
    for only reading or for both reading and writing.  Its specification is

From 98462905186f6d0a25600ec9e309de3efdc2084a Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 17:16:26 -0600
Subject: [PATCH 23/41] Changed PosixSharedMemory.size to use os.fstat.

---
 Lib/multiprocessing/shared_memory.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index c76d16888f7a7b..b9183403ae07af 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -173,21 +173,28 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         self.mode = mode
         if not size >= 0:
             raise ValueError("'size' must be a positive integer")
-        self.size = size
         if name is None:
             self._open_retry()
         else:
             self.name = name
             self.fd = _posixshmem.shm_open(name, flags, mode=mode)
-        if self.size:
+        if size:
             try:
-                os.ftruncate(self.fd, self.size)
+                os.ftruncate(self.fd, size)
             except OSError:
                 self.unlink()
                 raise
         self._mmap = mmap.mmap(self.fd, self.size)
         self.buf = memoryview(self._mmap)
 
+    @property
+    def size(self):
+        "Size in bytes."
+        if self.fd >= 0:
+            return os.fstat(self.fd).st_size
+        else:
+            return 0
+
     def _open_retry(self):
         # generate a random name, open, retry if it exists
         while True:

From 1f9bbf2fb8e482a625e0fc35af4c75faafa53f09 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 18:29:51 -0600
Subject: [PATCH 24/41] Change SharedMemory.buf to a read-only property as well
 as NamedSharedMemory.size.

---
 Lib/multiprocessing/shared_memory.py | 37 +++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index b9183403ae07af..b4a39bd7087419 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -112,17 +112,31 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
                 raise FileExistsError(name)
 
         self._mmap = mmap.mmap(-1, size, tagname=name)
-        self.buf = memoryview(self._mmap)
+        self._buf = memoryview(self._mmap)
         self.name = name
         self.mode = mode
-        self.size = size
+        self._size = size
+
+    @property
+    def size(self):
+        "Size in bytes."
+        return self._size
+
+    @property
+    def buf(self):
+        "A memoryview of contents of the shared memory block."
+        return self._buf
 
     def __repr__(self):
         return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
 
     def close(self):
-        self.buf.release()
-        self._mmap.close()
+        if self._buf is not None:
+            self._buf.release()
+            self._buf = None
+        if self._mmap is not None:
+            self._mmap.close()
+            self._mmap = None
 
     def unlink(self):
         """Windows ensures that destruction of the last reference to this
@@ -157,7 +171,7 @@ class PosixSharedMemory:
     fd = -1
     name = None
     _mmap = None
-    buf = None
+    _buf = None
 
     def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if name and (flags is None):
@@ -185,7 +199,7 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
                 self.unlink()
                 raise
         self._mmap = mmap.mmap(self.fd, self.size)
-        self.buf = memoryview(self._mmap)
+        self._buf = memoryview(self._mmap)
 
     @property
     def size(self):
@@ -195,6 +209,11 @@ def size(self):
         else:
             return 0
 
+    @property
+    def buf(self):
+        "A memoryview of contents of the shared memory block."
+        return self._buf
+
     def _open_retry(self):
         # generate a random name, open, retry if it exists
         while True:
@@ -215,9 +234,9 @@ def unlink(self):
             _posixshmem.shm_unlink(self.name)
 
     def close(self):
-        if self.buf is not None:
-            self.buf.release()
-            self.buf = None
+        if self._buf is not None:
+            self._buf.release()
+            self._buf = None
         if self._mmap is not None:
             self._mmap.close()
             self._mmap = None

From 69dd8a9248fc08a449ce16b903c9a2f94e7914df Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 18:41:42 -0600
Subject: [PATCH 25/41] Marked as provisional per PEP411 in docstring.

---
 Lib/multiprocessing/shared_memory.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index b4a39bd7087419..58e81533bc7570 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -1,4 +1,8 @@
-"Provides shared memory for direct access across processes."
+"""Provides shared memory for direct access across processes.
+
+The API of this package is currently provisional. Refer to the
+documentation for details.
+"""
 
 
 __all__ = [ 'SharedMemory', 'PosixSharedMemory', 'WindowsNamedSharedMemory',

From 594140a65489adde4f60b14925e295bb1639501e Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 19:45:23 -0600
Subject: [PATCH 26/41] Changed SharedMemoryTracker to be private.

---
 Lib/multiprocessing/shared_memory.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 58e81533bc7570..ab2621ad151015 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -7,7 +7,7 @@
 
 __all__ = [ 'SharedMemory', 'PosixSharedMemory', 'WindowsNamedSharedMemory',
             'ShareableList', 'shareable_wrap',
-            'SharedMemoryServer', 'SharedMemoryManager', 'SharedMemoryTracker' ]
+            'SharedMemoryServer', 'SharedMemoryManager' ]
 
 
 from functools import reduce
@@ -696,7 +696,7 @@ def index(self, value):
             raise ValueError(f"{value!r} not in this container")
 
 
-class SharedMemoryTracker:
+class _SharedMemoryTracker:
     "Manages one or more shared memory segments."
 
     def __init__(self, name, segment_names=[]):
@@ -746,7 +746,7 @@ class SharedMemoryServer(Server):
     def __init__(self, *args, **kwargs):
         Server.__init__(self, *args, **kwargs)
         self.shared_memory_context = \
-            SharedMemoryTracker(f"shmm_{self.address}_{os.getpid()}")
+            _SharedMemoryTracker(f"shmm_{self.address}_{os.getpid()}")
         util.debug(f"SharedMemoryServer started by pid {os.getpid()}")
 
     def create(self, c, typeid, *args, **kwargs):

From 395709bbfa564c528d6d8aeba780d988bb774ede Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 22:27:02 -0600
Subject: [PATCH 27/41] Removed registered Proxy Objects from
 SharedMemoryManager.

---
 Doc/library/multiprocessing.shared_memory.rst | 36 +---------
 Lib/multiprocessing/shared_memory.py          | 16 +----
 Lib/test/_test_multiprocessing.py             | 67 +++++++------------
 3 files changed, 25 insertions(+), 94 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index cdafdd7cacc4ff..5bcea8ab3e16b5 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -205,9 +205,7 @@ same ``numpy.ndarray`` from two distinct Python shells:
 
    This class provides methods for creating and returning :class:`SharedMemory`
    instances and for creating a list-like object (:class:`ShareableList`)
-   backed by shared memory.  It also provides methods that create and
-   return :ref:`multiprocessing-proxy_objects` that support synchronization
-   across processes (i.e. multi-process-safe locks and semaphores).
+   backed by shared memory.
 
    Refer to :class:`multiprocessing.managers.BaseManager` for a description
    of the inherited *address* and *authkey* optional input arguments and how
@@ -224,38 +222,6 @@ same ``numpy.ndarray`` from two distinct Python shells:
       Create and return a new :class:`ShareableList` object, initialized
       by the values from the input ``sequence``.
 
-   .. method:: Barrier(parties[, action[, timeout]])
-
-      Create a shared :class:`threading.Barrier` object and return a
-      proxy for it.
-
-   .. method:: BoundedSemaphore([value])
-
-      Create a shared :class:`threading.BoundedSemaphore` object and return
-      a proxy for it.
-
-   .. method:: Condition([lock])
-
-      Create a shared :class:`threading.Condition` object and return a proxy
-      for it.  The optional input *lock* supports a proxy for a
-      :class:`threading.Lock` or :class:`threading.RLock` object.
-
-   .. method:: Event()
-
-      Create a shared :class:`threading.Event` object and return a proxy for it.
-
-   .. method:: Lock()
-
-      Create a shared :class:`threading.Lock` object and return a proxy for it.
-
-   .. method:: RLock()
-
-      Create a shared :class:`threading.RLock` object and return a proxy for it.
-
-   .. method:: Semaphore([value])
-
-      Create a shared :class:`threading.Semaphore` object and return a proxy
-      for it.
 
 The following example demonstrates the basic mechanisms of a
 :class:`SharedMemoryManager`:
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index ab2621ad151015..bd2a89bca38ac8 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -12,13 +12,11 @@
 
 from functools import reduce
 import mmap
-from .managers import dispatch, BaseManager, Server, State, ProcessError, \
-                      BarrierProxy, AcquirerProxy, ConditionProxy, EventProxy
+from .managers import dispatch, BaseManager, Server, State, ProcessError
 from . import util
 import os
 import struct
 import sys
-import threading
 import secrets
 try:
     import  _posixshmem
@@ -835,15 +833,3 @@ def ShareableList(self, sequence):
                 sl.shm.unlink()
                 raise e
         return sl
-
-SharedMemoryManager.register('Barrier', threading.Barrier, BarrierProxy)
-SharedMemoryManager.register(
-    'BoundedSemaphore',
-    threading.BoundedSemaphore,
-    AcquirerProxy
-)
-SharedMemoryManager.register('Condition', threading.Condition, ConditionProxy)
-SharedMemoryManager.register('Event', threading.Event, EventProxy)
-SharedMemoryManager.register('Lock', threading.Lock, AcquirerProxy)
-SharedMemoryManager.register('RLock', threading.RLock, AcquirerProxy)
-SharedMemoryManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 0daf1c673c1610..f39ba8b87853f2 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3751,9 +3751,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
 
         with shared_memory.SharedMemoryManager() as smm2:
             sl = smm2.ShareableList("howdy")
-            unnecessary_lock = smm2.Lock()
-            with unnecessary_lock:
-                shm = smm2.SharedMemory(size=128)
+            shm = smm2.SharedMemory(size=128)
             held_name = sl.shm.name
         if sys.platform != "win32":
             with self.assertRaises(FileNotFoundError):
@@ -4944,9 +4942,29 @@ def is_alive(self):
             any(process.is_alive() for process in forked_processes))
 
 
-class _MixinTestCommonManagerTypes(object):
+class TestSyncManagerTypes(unittest.TestCase):
+    """Test all the types which can be shared between a parent and a
+    child process by using a manager which acts as an intermediary
+    between them.
+
+    In the following unit-tests the base type is created in the parent
+    process, the @classmethod represents the worker process and the
+    shared object is readable and editable between the two.
+
+    # The child.
+    @classmethod
+    def _test_list(cls, obj):
+        assert obj[0] == 5
+        assert obj.append(6)
 
-    manager_class = None
+    # The parent.
+    def test_list(self):
+        o = self.manager.list()
+        o.append(5)
+        self.run_worker(self._test_list, o)
+        assert o[1] == 6
+    """
+    manager_class = multiprocessing.managers.SyncManager
 
     def setUp(self):
         self.manager = self.manager_class()
@@ -5056,31 +5074,6 @@ def test_barrier(self):
         o = self.manager.Barrier(5)
         self.run_worker(self._test_barrier, o)
 
-
-class TestSyncManagerTypes(_MixinTestCommonManagerTypes, unittest.TestCase):
-    """Test all the types which can be shared between a parent and a
-    child process by using a manager which acts as an intermediary
-    between them.
-
-    In the following unit-tests the base type is created in the parent
-    process, the @classmethod represents the worker process and the
-    shared object is readable and editable between the two.
-
-    # The child.
-    @classmethod
-    def _test_list(cls, obj):
-        assert obj[0] == 5
-        assert obj.append(6)
-
-    # The parent.
-    def test_list(self):
-        o = self.manager.list()
-        o.append(5)
-        self.run_worker(self._test_list, o)
-        assert o[1] == 6
-    """
-    manager_class = multiprocessing.managers.SyncManager
-
     @classmethod
     def _test_pool(cls, obj):
         # TODO: fix https://bugs.python.org/issue35919
@@ -5186,20 +5179,6 @@ def test_namespace(self):
         self.run_worker(self._test_namespace, o)
 
 
-try:
-    import multiprocessing.shared_memory
-except ImportError:
-    @unittest.skip("SharedMemoryManager not available on this platform")
-    class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
-                                       unittest.TestCase):
-        pass
-else:
-    class TestSharedMemoryManagerTypes(_MixinTestCommonManagerTypes,
-                                       unittest.TestCase):
-        """Same as above but by using SharedMemoryManager."""
-        manager_class = multiprocessing.shared_memory.SharedMemoryManager
-
-
 class MiscTestCase(unittest.TestCase):
     def test__all__(self):
         # Just make sure names in blacklist are excluded

From aa4a887c109418ae00b5096092bd03dbcec1d85c Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 22:31:57 -0600
Subject: [PATCH 28/41] Removed shareable_wrap().

---
 Lib/multiprocessing/shared_memory.py | 155 ---------------------------
 1 file changed, 155 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index bd2a89bca38ac8..4456db46804401 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -263,161 +263,6 @@ def __new__(cls, *args, **kwargs):
         return cls(*args, **kwargs)
 
 
-def shareable_wrap(
-    existing_obj=None,
-    shmem_name=None,
-    cls=None,
-    shape=(0,),
-    strides=None,
-    dtype=None,
-    format=None,
-    **kwargs
-):
-    """Provides a fast, convenient way to encapsulate objects that support
-    the buffer protocol as both producer and consumer, duplicating the
-    original object's data in shared memory and returning a new wrapped
-    object that when serialized via pickle does not serialize its data.
-
-    The function has been written in a general way to potentially work with
-    any object supporting the buffer protocol as producer and consumer.  It
-    is known to work well with NumPy ndarrays.  Among the Python core data
-    types and standard library, there are a number of objects supporting
-    the buffer protocol as a producer but not as a consumer.
-
-    Without an example of a producer+consumer of the buffer protocol in
-    the Python core to demonstrate the use of this function, this function
-    should likely be removed from this module and potentially be made
-    available instead via a pip-installable package."""
-
-    augmented_kwargs = dict(kwargs)
-    extras = dict(shape=shape, strides=strides, dtype=dtype, format=format)
-    for key, value in extras.items():
-        if value is not None:
-            augmented_kwargs[key] = value
-
-    if existing_obj is not None:
-        existing_type = getattr(
-            existing_obj,
-            "_proxied_type",
-            type(existing_obj)
-        )
-
-        #agg = existing_obj.itemsize
-        #size = [ agg := i * agg for i in existing_obj.shape ][-1]
-        # TODO: replace use of reduce below with above 2 lines once available
-        size = reduce(
-            lambda x, y: x * y,
-            existing_obj.shape,
-            existing_obj.itemsize
-        )
-
-    else:
-        assert shmem_name is not None
-        existing_type = cls
-        size = 1
-
-    shm = SharedMemory(shmem_name, size=size)
-
-    class CustomShareableProxy(existing_type):
-
-        def __init__(self, *args, buffer=None, **kwargs):
-            # If copy method called, prevent recursion from replacing _shm.
-            if not hasattr(self, "_shm"):
-                self._shm = shm
-                self._proxied_type = existing_type
-            else:
-                # _proxied_type only used in pickling.
-                assert hasattr(self, "_proxied_type")
-            try:
-                existing_type.__init__(self, *args, **kwargs)
-            except Exception:
-                pass
-
-        def __repr__(self):
-            if not hasattr(self, "_shm"):
-                return existing_type.__repr__(self)
-            formatted_pairs = (
-                "%s=%r" % kv for kv in self._build_state(self).items()
-            )
-            return f"{self.__class__.__name__}({', '.join(formatted_pairs)})"
-
-        #def __getstate__(self):
-        #    if not hasattr(self, "_shm"):
-        #        return existing_type.__getstate__(self)
-        #    state = self._build_state(self)
-        #    return state
-
-        #def __setstate__(self, state):
-        #    self.__init__(**state)
-
-        def __reduce__(self):
-            return (
-                shareable_wrap,
-                (
-                    None,
-                    self._shm.name,
-                    self._proxied_type,
-                    self.shape,
-                    self.strides,
-                    self.dtype.str if hasattr(self, "dtype") else None,
-                    getattr(self, "format", None),
-                ),
-            )
-
-        def copy(self):
-            dupe = existing_type.copy(self)
-            if not hasattr(dupe, "_shm"):
-                dupe = shareable_wrap(dupe)
-            return dupe
-
-        @staticmethod
-        def _build_state(existing_obj, generics_only=False):
-            state = {
-                "shape": existing_obj.shape,
-                "strides": existing_obj.strides,
-            }
-            try:
-                state["dtype"] = existing_obj.dtype
-            except AttributeError:
-                try:
-                    state["format"] = existing_obj.format
-                except AttributeError:
-                    pass
-            if not generics_only:
-                try:
-                    state["shmem_name"] = existing_obj._shm.name
-                    state["cls"] = existing_type
-                except AttributeError:
-                    pass
-            return state
-
-    proxy_type = type(
-        f"{existing_type.__name__}Shareable",
-        CustomShareableProxy.__bases__,
-        dict(CustomShareableProxy.__dict__),
-    )
-
-    if existing_obj is not None:
-        try:
-            proxy_obj = proxy_type(
-                buffer=shm.buf,
-                **proxy_type._build_state(existing_obj)
-            )
-        except Exception:
-            proxy_obj = proxy_type(
-                buffer=shm.buf,
-                **proxy_type._build_state(existing_obj, True)
-            )
-
-        mveo = memoryview(existing_obj)
-        proxy_obj._shm.buf[:mveo.nbytes] = mveo.tobytes()
-
-    else:
-        proxy_obj = proxy_type(buffer=shm.buf, **augmented_kwargs)
-
-    return proxy_obj
-
-
 encoding = "utf8"
 
 class ShareableList:

From 885592b56f555c03e6444c7de1839c38c4792fb6 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sat, 16 Feb 2019 22:39:45 -0600
Subject: [PATCH 29/41] Removed shareable_wrap() and dangling references to it.

---
 Lib/multiprocessing/shared_memory.py | 162 +--------------------------
 1 file changed, 1 insertion(+), 161 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index bd2a89bca38ac8..69c60dbadcac1c 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -6,7 +6,7 @@
 
 
 __all__ = [ 'SharedMemory', 'PosixSharedMemory', 'WindowsNamedSharedMemory',
-            'ShareableList', 'shareable_wrap',
+            'ShareableList',
             'SharedMemoryServer', 'SharedMemoryManager' ]
 
 
@@ -263,161 +263,6 @@ def __new__(cls, *args, **kwargs):
         return cls(*args, **kwargs)
 
 
-def shareable_wrap(
-    existing_obj=None,
-    shmem_name=None,
-    cls=None,
-    shape=(0,),
-    strides=None,
-    dtype=None,
-    format=None,
-    **kwargs
-):
-    """Provides a fast, convenient way to encapsulate objects that support
-    the buffer protocol as both producer and consumer, duplicating the
-    original object's data in shared memory and returning a new wrapped
-    object that when serialized via pickle does not serialize its data.
-
-    The function has been written in a general way to potentially work with
-    any object supporting the buffer protocol as producer and consumer.  It
-    is known to work well with NumPy ndarrays.  Among the Python core data
-    types and standard library, there are a number of objects supporting
-    the buffer protocol as a producer but not as a consumer.
-
-    Without an example of a producer+consumer of the buffer protocol in
-    the Python core to demonstrate the use of this function, this function
-    should likely be removed from this module and potentially be made
-    available instead via a pip-installable package."""
-
-    augmented_kwargs = dict(kwargs)
-    extras = dict(shape=shape, strides=strides, dtype=dtype, format=format)
-    for key, value in extras.items():
-        if value is not None:
-            augmented_kwargs[key] = value
-
-    if existing_obj is not None:
-        existing_type = getattr(
-            existing_obj,
-            "_proxied_type",
-            type(existing_obj)
-        )
-
-        #agg = existing_obj.itemsize
-        #size = [ agg := i * agg for i in existing_obj.shape ][-1]
-        # TODO: replace use of reduce below with above 2 lines once available
-        size = reduce(
-            lambda x, y: x * y,
-            existing_obj.shape,
-            existing_obj.itemsize
-        )
-
-    else:
-        assert shmem_name is not None
-        existing_type = cls
-        size = 1
-
-    shm = SharedMemory(shmem_name, size=size)
-
-    class CustomShareableProxy(existing_type):
-
-        def __init__(self, *args, buffer=None, **kwargs):
-            # If copy method called, prevent recursion from replacing _shm.
-            if not hasattr(self, "_shm"):
-                self._shm = shm
-                self._proxied_type = existing_type
-            else:
-                # _proxied_type only used in pickling.
-                assert hasattr(self, "_proxied_type")
-            try:
-                existing_type.__init__(self, *args, **kwargs)
-            except Exception:
-                pass
-
-        def __repr__(self):
-            if not hasattr(self, "_shm"):
-                return existing_type.__repr__(self)
-            formatted_pairs = (
-                "%s=%r" % kv for kv in self._build_state(self).items()
-            )
-            return f"{self.__class__.__name__}({', '.join(formatted_pairs)})"
-
-        #def __getstate__(self):
-        #    if not hasattr(self, "_shm"):
-        #        return existing_type.__getstate__(self)
-        #    state = self._build_state(self)
-        #    return state
-
-        #def __setstate__(self, state):
-        #    self.__init__(**state)
-
-        def __reduce__(self):
-            return (
-                shareable_wrap,
-                (
-                    None,
-                    self._shm.name,
-                    self._proxied_type,
-                    self.shape,
-                    self.strides,
-                    self.dtype.str if hasattr(self, "dtype") else None,
-                    getattr(self, "format", None),
-                ),
-            )
-
-        def copy(self):
-            dupe = existing_type.copy(self)
-            if not hasattr(dupe, "_shm"):
-                dupe = shareable_wrap(dupe)
-            return dupe
-
-        @staticmethod
-        def _build_state(existing_obj, generics_only=False):
-            state = {
-                "shape": existing_obj.shape,
-                "strides": existing_obj.strides,
-            }
-            try:
-                state["dtype"] = existing_obj.dtype
-            except AttributeError:
-                try:
-                    state["format"] = existing_obj.format
-                except AttributeError:
-                    pass
-            if not generics_only:
-                try:
-                    state["shmem_name"] = existing_obj._shm.name
-                    state["cls"] = existing_type
-                except AttributeError:
-                    pass
-            return state
-
-    proxy_type = type(
-        f"{existing_type.__name__}Shareable",
-        CustomShareableProxy.__bases__,
-        dict(CustomShareableProxy.__dict__),
-    )
-
-    if existing_obj is not None:
-        try:
-            proxy_obj = proxy_type(
-                buffer=shm.buf,
-                **proxy_type._build_state(existing_obj)
-            )
-        except Exception:
-            proxy_obj = proxy_type(
-                buffer=shm.buf,
-                **proxy_type._build_state(existing_obj, True)
-            )
-
-        mveo = memoryview(existing_obj)
-        proxy_obj._shm.buf[:mveo.nbytes] = mveo.tobytes()
-
-    else:
-        proxy_obj = proxy_type(buffer=shm.buf, **augmented_kwargs)
-
-    return proxy_obj
-
-
 encoding = "utf8"
 
 class ShareableList:
@@ -730,11 +575,6 @@ def __getstate__(self):
     def __setstate__(self, state):
         self.__init__(*state)
 
-    def wrap(self, obj_exposing_buffer_protocol):
-        wrapped_obj = shareable_wrap(obj_exposing_buffer_protocol)
-        self.register_segment(wrapped_obj._shm.name)
-        return wrapped_obj
-
 
 class SharedMemoryServer(Server):
 

From 5848ec413bee0e9f5dd068f5d7563ac7786e329c Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 17 Feb 2019 12:48:32 -0600
Subject: [PATCH 30/41] For consistency added __reduce__ to key classes.

---
 Lib/multiprocessing/shared_memory.py | 29 +++++++++++++++++-
 Lib/test/_test_multiprocessing.py    | 44 ++++++++++++++++++++++++++--
 2 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 69c60dbadcac1c..4c2bfe4ff65542 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -10,7 +10,7 @@
             'SharedMemoryServer', 'SharedMemoryManager' ]
 
 
-from functools import reduce
+from functools import partial, reduce
 import mmap
 from .managers import dispatch, BaseManager, Server, State, ProcessError
 from . import util
@@ -129,6 +129,18 @@ def buf(self):
         "A memoryview of contents of the shared memory block."
         return self._buf
 
+    def __reduce__(self):
+        return (
+            self.__class__,
+            (
+                self.name,
+                None,
+                self.mode,
+                0,
+                False,
+            ),
+        )
+
     def __repr__(self):
         return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
 
@@ -228,6 +240,18 @@ def _open_retry(self):
             self.name = name
             break
 
+    def __reduce__(self):
+        return (
+            self.__class__,
+            (
+                self.name,
+                None,
+                self.mode,
+                0,
+                False,
+            ),
+        )
+
     def __repr__(self):
         return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
 
@@ -475,6 +499,9 @@ def __setitem__(self, position, value):
         value = value.encode(encoding) if isinstance(value, str) else value
         struct.pack_into(new_format, self.shm.buf, offset, value)
 
+    def __reduce__(self):
+        return partial(self.__class__, name=self.shm.name), ()
+
     def __len__(self):
         return struct.unpack_from("q", self.shm.buf, 0)[0]
 
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index f39ba8b87853f2..985664cd43c740 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -19,6 +19,7 @@
 import logging
 import struct
 import operator
+import pickle
 import weakref
 import warnings
 import test.support
@@ -3623,8 +3624,11 @@ class _TestSharedMemory(BaseTestCase):
     ALLOWED_TYPES = ('processes',)
 
     @staticmethod
-    def _attach_existing_shmem_then_write(shmem_name, binary_data):
-        local_sms = shared_memory.SharedMemory(shmem_name)
+    def _attach_existing_shmem_then_write(shmem_name_or_obj, binary_data):
+        if isinstance(shmem_name_or_obj, str):
+            local_sms = shared_memory.SharedMemory(shmem_name_or_obj)
+        else:
+            local_sms = shmem_name_or_obj
         local_sms.buf[:len(binary_data)] = binary_data
         local_sms.close()
 
@@ -3718,6 +3722,7 @@ def test_shared_memory_across_processes(self):
         )
         self.addCleanup(sms.unlink)
 
+        # Verify remote attachment to existing block by name is working.
         p = self.Process(
             target=self._attach_existing_shmem_then_write,
             args=(sms.name, b'howdy')
@@ -3727,6 +3732,16 @@ def test_shared_memory_across_processes(self):
         p.join()
         self.assertEqual(bytes(sms.buf[:5]), b'howdy')
 
+        # Verify pickling of SharedMemory instance also works.
+        p = self.Process(
+            target=self._attach_existing_shmem_then_write,
+            args=(sms, b'HELLO')
+        )
+        p.daemon = True
+        p.start()
+        p.join()
+        self.assertEqual(bytes(sms.buf[:5]), b'HELLO')
+
         sms.close()
 
     def test_shared_memory_SharedMemoryManager_basics(self):
@@ -3842,6 +3857,31 @@ def test_shared_memory_ShareableList_basics(self):
         finally:
             empty_sl.shm.unlink()
 
+    def test_shared_memory_ShareableList_pickling(self):
+        sl = shared_memory.ShareableList(range(10))
+        self.addCleanup(sl.shm.unlink)
+
+        serialized_sl = pickle.dumps(sl)
+        deserialized_sl = pickle.loads(serialized_sl)
+        self.assertTrue(
+            isinstance(deserialized_sl, shared_memory.ShareableList)
+        )
+        self.assertTrue(deserialized_sl[-1], 9)
+        self.assertFalse(sl is deserialized_sl)
+        deserialized_sl[4] = "changed"
+        self.assertEqual(sl[4], "changed")
+
+        # Verify data is not being put into the pickled representation.
+        name = 'a' * len(sl.shm.name)
+        larger_sl = shared_memory.ShareableList(range(400))
+        self.addCleanup(larger_sl.shm.unlink)
+        serialized_larger_sl = pickle.dumps(larger_sl)
+        self.assertTrue(len(serialized_sl) == len(serialized_larger_sl))
+        larger_sl.shm.close()
+
+        deserialized_sl.shm.close()
+        sl.shm.close()
+
 #
 #
 #

From 6ff8eed30695d920c57de8d96a9ceaf199290238 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 17 Feb 2019 22:56:15 -0600
Subject: [PATCH 31/41] Fix for potential race condition on Windows for O_CREX.

---
 Lib/multiprocessing/shared_memory.py | 49 +++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 4c2bfe4ff65542..1933086f683ee7 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -44,7 +44,10 @@ class MEMORY_BASIC_INFORMATION(ctypes.Structure):
         )
 
     PMEMORY_BASIC_INFORMATION = ctypes.POINTER(MEMORY_BASIC_INFORMATION)
-    FILE_MAP_READ = 0x0004
+    PAGE_READONLY = 0x02
+    PAGE_EXECUTE_READWRITE = 0x04
+    INVALID_HANDLE_VALUE = -1
+    FILE_ALREADY_EXISTS = 183
 
     def _errcheck_bool(result, func, args):
         if not result:
@@ -67,6 +70,20 @@ def _errcheck_bool(result, func, args):
         wintypes.LPCWSTR
     )
 
+    kernel32.CreateFileMappingW.errcheck = _errcheck_bool
+    kernel32.CreateFileMappingW.restype = wintypes.HANDLE
+    kernel32.CreateFileMappingW.argtypes = (
+        wintypes.HANDLE,
+        wintypes.LPCVOID,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.LPCWSTR
+    )
+
+    kernel32.GetLastError.restype = wintypes.DWORD
+    kernel32.GetLastError.argtypes = ()
+
     kernel32.MapViewOfFile.errcheck = _errcheck_bool
     kernel32.MapViewOfFile.restype = wintypes.LPVOID
     kernel32.MapViewOfFile.argtypes = (
@@ -91,11 +108,11 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             # Attempt to dynamically determine the existing named shared
             # memory block's size which is likely a multiple of mmap.PAGESIZE.
             try:
-                h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
+                h_map = kernel32.OpenFileMappingW(PAGE_READONLY, False, name)
             except OSError:
                 raise FileNotFoundError(name)
             try:
-                p_buf = kernel32.MapViewOfFile(h_map, FILE_MAP_READ, 0, 0, 0)
+                p_buf = kernel32.MapViewOfFile(h_map, PAGE_READONLY, 0, 0, 0)
             finally:
                 kernel32.CloseHandle(h_map)
             mbi = MEMORY_BASIC_INFORMATION()
@@ -103,17 +120,27 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             size = mbi.RegionSize
 
         if flags == O_CREX:
-            # Verify no named shared memory block already exists by this name.
+            # Create and reserve shared memory block with this name until
+            # it can be attached to by mmap.
+            h_map = kernel32.CreateFileMappingW(
+                INVALID_HANDLE_VALUE,
+                None,
+                PAGE_EXECUTE_READWRITE,
+                size >> 32,
+                size & 0xFFFFFFFF,
+                name
+            )
             try:
-                h_map = kernel32.OpenFileMappingW(FILE_MAP_READ, False, name)
+                last_error_code = kernel32.GetLastError()
+                if last_error_code == FILE_ALREADY_EXISTS:
+                    raise FileExistsError(f"File exists: {name!r}")
+                self._mmap = mmap.mmap(-1, size, tagname=name)
+            finally:
                 kernel32.CloseHandle(h_map)
-                name_collision = True
-            except OSError as ose:
-                name_collision = False
-            if name_collision:
-                raise FileExistsError(name)
 
-        self._mmap = mmap.mmap(-1, size, tagname=name)
+        else:
+            self._mmap = mmap.mmap(-1, size, tagname=name)
+
         self._buf = memoryview(self._mmap)
         self.name = name
         self.mode = mode

From 06620e295a1428026798caa6454e582717b2c22d Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Sun, 17 Feb 2019 22:57:49 -0600
Subject: [PATCH 32/41] Remove unused imports.

---
 Lib/multiprocessing/shared_memory.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 1933086f683ee7..fb6d6c2c23c3ee 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -10,13 +10,12 @@
             'SharedMemoryServer', 'SharedMemoryManager' ]
 
 
-from functools import partial, reduce
+from functools import partial
 import mmap
 from .managers import dispatch, BaseManager, Server, State, ProcessError
 from . import util
 import os
 import struct
-import sys
 import secrets
 try:
     import  _posixshmem

From 868b83d681c3e11dd6de22631ac61493a5b73ded Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Tue, 19 Feb 2019 14:59:56 -0600
Subject: [PATCH 33/41] Update access to kernel32 on Windows per feedback from
 eryksun.

---
 Lib/multiprocessing/shared_memory.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index fb6d6c2c23c3ee..cb492c21893482 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -26,10 +26,11 @@
 O_CREX = O_CREAT | O_EXCL
 
 if os.name == "nt":
+    import _winapi
     import ctypes
     from ctypes import wintypes
 
-    kernel32 = ctypes.windll.kernel32
+    kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
 
     class MEMORY_BASIC_INFORMATION(ctypes.Structure):
         _fields_ = (
@@ -46,7 +47,7 @@ class MEMORY_BASIC_INFORMATION(ctypes.Structure):
     PAGE_READONLY = 0x02
     PAGE_EXECUTE_READWRITE = 0x04
     INVALID_HANDLE_VALUE = -1
-    FILE_ALREADY_EXISTS = 183
+    ERROR_ALREADY_EXISTS = 183
 
     def _errcheck_bool(result, func, args):
         if not result:
@@ -80,9 +81,6 @@ def _errcheck_bool(result, func, args):
         wintypes.LPCWSTR
     )
 
-    kernel32.GetLastError.restype = wintypes.DWORD
-    kernel32.GetLastError.argtypes = ()
-
     kernel32.MapViewOfFile.errcheck = _errcheck_bool
     kernel32.MapViewOfFile.restype = wintypes.LPVOID
     kernel32.MapViewOfFile.argtypes = (
@@ -93,9 +91,6 @@ def _errcheck_bool(result, func, args):
         ctypes.c_size_t
     )
 
-    kernel32.CloseHandle.errcheck = _errcheck_bool
-    kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
-
 
 class WindowsNamedSharedMemory:
 
@@ -113,7 +108,7 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             try:
                 p_buf = kernel32.MapViewOfFile(h_map, PAGE_READONLY, 0, 0, 0)
             finally:
-                kernel32.CloseHandle(h_map)
+                _winapi.CloseHandle(h_map)
             mbi = MEMORY_BASIC_INFORMATION()
             kernel32.VirtualQuery(p_buf, ctypes.byref(mbi), mmap.PAGESIZE)
             size = mbi.RegionSize
@@ -130,12 +125,12 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
                 name
             )
             try:
-                last_error_code = kernel32.GetLastError()
-                if last_error_code == FILE_ALREADY_EXISTS:
+                last_error_code = ctypes.get_last_error()
+                if last_error_code == ERROR_ALREADY_EXISTS:
                     raise FileExistsError(f"File exists: {name!r}")
                 self._mmap = mmap.mmap(-1, size, tagname=name)
             finally:
-                kernel32.CloseHandle(h_map)
+                _winapi.CloseHandle(h_map)
 
         else:
             self._mmap = mmap.mmap(-1, size, tagname=name)

From 9d83b060b6f0e0df82a8f2fe4833495bf1830181 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Tue, 19 Feb 2019 22:55:42 -0600
Subject: [PATCH 34/41] Moved kernel32 calls to _winapi.

---
 Lib/multiprocessing/shared_memory.py |  77 ++------------
 Modules/_winapi.c                    | 135 ++++++++++++++++++++++-
 Modules/clinic/_winapi.c.h           | 154 ++++++++++++++++++++++++++-
 3 files changed, 295 insertions(+), 71 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index cb492c21893482..b7f3ae2f80c9df 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -27,69 +27,10 @@
 
 if os.name == "nt":
     import _winapi
-    import ctypes
-    from ctypes import wintypes
-
-    kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
-
-    class MEMORY_BASIC_INFORMATION(ctypes.Structure):
-        _fields_ = (
-            ('BaseAddress',       ctypes.c_void_p),
-            ('AllocationBase',    ctypes.c_void_p),
-            ('AllocationProtect', wintypes.DWORD),
-            ('RegionSize',        ctypes.c_size_t),
-            ('State',             wintypes.DWORD),
-            ('Protect',           wintypes.DWORD),
-            ('Type',              wintypes.DWORD)
-        )
 
-    PMEMORY_BASIC_INFORMATION = ctypes.POINTER(MEMORY_BASIC_INFORMATION)
     PAGE_READONLY = 0x02
     PAGE_EXECUTE_READWRITE = 0x04
     INVALID_HANDLE_VALUE = -1
-    ERROR_ALREADY_EXISTS = 183
-
-    def _errcheck_bool(result, func, args):
-        if not result:
-            raise ctypes.WinError(ctypes.get_last_error())
-        return args
-
-    kernel32.VirtualQuery.errcheck = _errcheck_bool
-    kernel32.VirtualQuery.restype = ctypes.c_size_t
-    kernel32.VirtualQuery.argtypes = (
-        wintypes.LPCVOID,
-        PMEMORY_BASIC_INFORMATION,
-        ctypes.c_size_t
-    )
-
-    kernel32.OpenFileMappingW.errcheck = _errcheck_bool
-    kernel32.OpenFileMappingW.restype = wintypes.HANDLE
-    kernel32.OpenFileMappingW.argtypes = (
-        wintypes.DWORD,
-        wintypes.BOOL,
-        wintypes.LPCWSTR
-    )
-
-    kernel32.CreateFileMappingW.errcheck = _errcheck_bool
-    kernel32.CreateFileMappingW.restype = wintypes.HANDLE
-    kernel32.CreateFileMappingW.argtypes = (
-        wintypes.HANDLE,
-        wintypes.LPCVOID,
-        wintypes.DWORD,
-        wintypes.DWORD,
-        wintypes.DWORD,
-        wintypes.LPCWSTR
-    )
-
-    kernel32.MapViewOfFile.errcheck = _errcheck_bool
-    kernel32.MapViewOfFile.restype = wintypes.LPVOID
-    kernel32.MapViewOfFile.argtypes = (
-        wintypes.HANDLE,
-        wintypes.DWORD,
-        wintypes.DWORD,
-        wintypes.DWORD,
-        ctypes.c_size_t
-    )
 
 
 class WindowsNamedSharedMemory:
@@ -102,31 +43,29 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             # Attempt to dynamically determine the existing named shared
             # memory block's size which is likely a multiple of mmap.PAGESIZE.
             try:
-                h_map = kernel32.OpenFileMappingW(PAGE_READONLY, False, name)
+                h_map = _winapi.OpenFileMappingW(PAGE_READONLY, False, name)
             except OSError:
                 raise FileNotFoundError(name)
             try:
-                p_buf = kernel32.MapViewOfFile(h_map, PAGE_READONLY, 0, 0, 0)
+                p_buf = _winapi.MapViewOfFile(h_map, PAGE_READONLY, 0, 0, 0)
             finally:
                 _winapi.CloseHandle(h_map)
-            mbi = MEMORY_BASIC_INFORMATION()
-            kernel32.VirtualQuery(p_buf, ctypes.byref(mbi), mmap.PAGESIZE)
-            size = mbi.RegionSize
+            size = _winapi.VirtualQuerySize(p_buf)
 
         if flags == O_CREX:
             # Create and reserve shared memory block with this name until
             # it can be attached to by mmap.
-            h_map = kernel32.CreateFileMappingW(
+            h_map = _winapi.CreateFileMappingW(
                 INVALID_HANDLE_VALUE,
-                None,
+                _winapi.NULL,
                 PAGE_EXECUTE_READWRITE,
-                size >> 32,
+                (size >> 32) & 0xFFFFFFFF,
                 size & 0xFFFFFFFF,
                 name
             )
             try:
-                last_error_code = ctypes.get_last_error()
-                if last_error_code == ERROR_ALREADY_EXISTS:
+                last_error_code = _winapi.GetLastError()
+                if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
                     raise FileExistsError(f"File exists: {name!r}")
                 self._mmap = mmap.mmap(-1, size, tagname=name)
             finally:
diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index cdb45c23e78739..c949c01a2a2202 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -159,6 +159,7 @@ def create_converter(type_, format_unit):
 create_converter('HANDLE', '" F_HANDLE "')
 create_converter('HMODULE', '" F_HANDLE "')
 create_converter('LPSECURITY_ATTRIBUTES', '" F_POINTER "')
+create_converter('LPCVOID', '" F_POINTER "')
 
 create_converter('BOOL', 'i') # F_BOOL used previously (always 'i')
 create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter)
@@ -186,8 +187,17 @@ class DWORD_return_converter(CReturnConverter):
         self.err_occurred_if("_return_value == PY_DWORD_MAX", data)
         data.return_conversion.append(
             'return_value = Py_BuildValue("k", _return_value);\n')
+
+class LPVOID_return_converter(CReturnConverter):
+    type = 'LPVOID'
+
+    def render(self, function, data):
+        self.declare(data)
+        self.err_occurred_if("_return_value == NULL", data)
+        data.return_conversion.append(
+            'return_value = HANDLE_TO_PYNUM(_return_value);\n')
 [python start generated code]*/
-/*[python end generated code: output=da39a3ee5e6b4b0d input=27456f8555228b62]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=79464c61a31ae932]*/
 
 #include "clinic/_winapi.c.h"
 
@@ -464,6 +474,41 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
     return handle;
 }
 
+/*[clinic input]
+_winapi.CreateFileMappingW -> HANDLE
+
+    file_handle: HANDLE
+    security_attributes: LPSECURITY_ATTRIBUTES
+    protect: DWORD
+    max_size_high: DWORD
+    max_size_low: DWORD
+    name: LPCWSTR
+    /
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_CreateFileMappingW_impl(PyObject *module, HANDLE file_handle,
+                                LPSECURITY_ATTRIBUTES security_attributes,
+                                DWORD protect, DWORD max_size_high,
+                                DWORD max_size_low, LPCWSTR name)
+/*[clinic end generated code: output=c6b017501c929de1 input=35cadabe53b3b4da]*/
+{
+    HANDLE handle;
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = CreateFileMappingW(file_handle, security_attributes,
+                                protect, max_size_high, max_size_low,
+                                name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == NULL) {
+        PyErr_SetFromWindowsErr(0);
+        handle = INVALID_HANDLE_VALUE;
+    }
+
+    return handle;
+}
+
 /*[clinic input]
 _winapi.CreateJunction
 
@@ -1295,6 +1340,64 @@ _winapi_GetVersion_impl(PyObject *module)
 
 #pragma warning(pop)
 
+/*[clinic input]
+_winapi.MapViewOfFile -> LPVOID
+
+    file_map: HANDLE
+    desired_access: DWORD
+    file_offset_high: DWORD
+    file_offset_low: DWORD
+    number_bytes: size_t
+    /
+[clinic start generated code]*/
+
+static LPVOID
+_winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map,
+                           DWORD desired_access, DWORD file_offset_high,
+                           DWORD file_offset_low, size_t number_bytes)
+/*[clinic end generated code: output=f23b1ee4823663e3 input=177471073be1a103]*/
+{
+    LPVOID address;
+
+    Py_BEGIN_ALLOW_THREADS
+    address = MapViewOfFile(file_map, desired_access, file_offset_high,
+                            file_offset_low, number_bytes);
+    Py_END_ALLOW_THREADS
+
+    if (address == NULL)
+        PyErr_SetFromWindowsErr(0);
+
+    return address;
+}
+
+/*[clinic input]
+_winapi.OpenFileMappingW -> HANDLE
+
+    desired_access: DWORD
+    inherit_handle: BOOL
+    name: LPCWSTR
+    /
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_OpenFileMappingW_impl(PyObject *module, DWORD desired_access,
+                              BOOL inherit_handle, LPCWSTR name)
+/*[clinic end generated code: output=ad829d0e68cac379 input=68fa4e0f2d5d5c42]*/
+{
+    HANDLE handle;
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = OpenFileMappingW(desired_access, inherit_handle, name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == NULL) {
+        PyErr_SetFromWindowsErr(0);
+        handle = INVALID_HANDLE_VALUE;
+    }
+
+    return handle;
+}
+
 /*[clinic input]
 _winapi.OpenProcess -> HANDLE
 
@@ -1490,6 +1593,32 @@ _winapi_TerminateProcess_impl(PyObject *module, HANDLE handle,
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+_winapi.VirtualQuerySize -> size_t
+
+    address: LPCVOID
+    /
+[clinic start generated code]*/
+
+static size_t
+_winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address)
+/*[clinic end generated code: output=40c8e0ff5ec964df input=6b784a69755d0bb6]*/
+{
+    SIZE_T size_of_buf;
+    MEMORY_BASIC_INFORMATION mem_basic_info;
+    SIZE_T region_size;
+
+    Py_BEGIN_ALLOW_THREADS
+    size_of_buf = VirtualQuery(address, &mem_basic_info, sizeof(mem_basic_info));
+    Py_END_ALLOW_THREADS
+
+    if (size_of_buf == 0)
+        PyErr_SetFromWindowsErr(0);
+
+    region_size = mem_basic_info.RegionSize;
+    return region_size;
+}
+
 /*[clinic input]
 _winapi.WaitNamedPipe
 
@@ -1719,6 +1848,7 @@ static PyMethodDef winapi_functions[] = {
     _WINAPI_CLOSEHANDLE_METHODDEF
     _WINAPI_CONNECTNAMEDPIPE_METHODDEF
     _WINAPI_CREATEFILE_METHODDEF
+    _WINAPI_CREATEFILEMAPPINGW_METHODDEF
     _WINAPI_CREATENAMEDPIPE_METHODDEF
     _WINAPI_CREATEPIPE_METHODDEF
     _WINAPI_CREATEPROCESS_METHODDEF
@@ -1731,11 +1861,14 @@ static PyMethodDef winapi_functions[] = {
     _WINAPI_GETMODULEFILENAME_METHODDEF
     _WINAPI_GETSTDHANDLE_METHODDEF
     _WINAPI_GETVERSION_METHODDEF
+    _WINAPI_MAPVIEWOFFILE_METHODDEF
+    _WINAPI_OPENFILEMAPPINGW_METHODDEF
     _WINAPI_OPENPROCESS_METHODDEF
     _WINAPI_PEEKNAMEDPIPE_METHODDEF
     _WINAPI_READFILE_METHODDEF
     _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF
     _WINAPI_TERMINATEPROCESS_METHODDEF
+    _WINAPI_VIRTUALQUERYSIZE_METHODDEF
     _WINAPI_WAITNAMEDPIPE_METHODDEF
     _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF
     _WINAPI_WAITFORSINGLEOBJECT_METHODDEF
diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
index f1158a006210d4..1bb5ab3c45d5ca 100644
--- a/Modules/clinic/_winapi.c.h
+++ b/Modules/clinic/_winapi.c.h
@@ -168,6 +168,50 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_CreateFileMappingW__doc__,
+"CreateFileMappingW($module, file_handle, security_attributes, protect,\n"
+"                   max_size_high, max_size_low, name, /)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_CREATEFILEMAPPINGW_METHODDEF    \
+    {"CreateFileMappingW", (PyCFunction)(void(*)(void))_winapi_CreateFileMappingW, METH_FASTCALL, _winapi_CreateFileMappingW__doc__},
+
+static HANDLE
+_winapi_CreateFileMappingW_impl(PyObject *module, HANDLE file_handle,
+                                LPSECURITY_ATTRIBUTES security_attributes,
+                                DWORD protect, DWORD max_size_high,
+                                DWORD max_size_low, LPCWSTR name);
+
+static PyObject *
+_winapi_CreateFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    HANDLE file_handle;
+    LPSECURITY_ATTRIBUTES security_attributes;
+    DWORD protect;
+    DWORD max_size_high;
+    DWORD max_size_low;
+    LPCWSTR name;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkku:CreateFileMappingW",
+        &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_CreateFileMappingW_impl(module, file_handle, security_attributes, protect, max_size_high, max_size_low, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_CreateJunction__doc__,
 "CreateJunction($module, src_path, dst_path, /)\n"
 "--\n"
@@ -602,6 +646,83 @@ _winapi_GetVersion(PyObject *module, PyObject *Py_UNUSED(ignored))
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_MapViewOfFile__doc__,
+"MapViewOfFile($module, file_map, desired_access, file_offset_high,\n"
+"              file_offset_low, number_bytes, /)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_MAPVIEWOFFILE_METHODDEF    \
+    {"MapViewOfFile", (PyCFunction)(void(*)(void))_winapi_MapViewOfFile, METH_FASTCALL, _winapi_MapViewOfFile__doc__},
+
+static LPVOID
+_winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map,
+                           DWORD desired_access, DWORD file_offset_high,
+                           DWORD file_offset_low, size_t number_bytes);
+
+static PyObject *
+_winapi_MapViewOfFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    HANDLE file_map;
+    DWORD desired_access;
+    DWORD file_offset_high;
+    DWORD file_offset_low;
+    size_t number_bytes;
+    LPVOID _return_value;
+
+    if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "kkkO&:MapViewOfFile",
+        &file_map, &desired_access, &file_offset_high, &file_offset_low, _PyLong_Size_t_Converter, &number_bytes)) {
+        goto exit;
+    }
+    _return_value = _winapi_MapViewOfFile_impl(module, file_map, desired_access, file_offset_high, file_offset_low, number_bytes);
+    if ((_return_value == NULL) && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_winapi_OpenFileMappingW__doc__,
+"OpenFileMappingW($module, desired_access, inherit_handle, name, /)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_OPENFILEMAPPINGW_METHODDEF    \
+    {"OpenFileMappingW", (PyCFunction)(void(*)(void))_winapi_OpenFileMappingW, METH_FASTCALL, _winapi_OpenFileMappingW__doc__},
+
+static HANDLE
+_winapi_OpenFileMappingW_impl(PyObject *module, DWORD desired_access,
+                              BOOL inherit_handle, LPCWSTR name);
+
+static PyObject *
+_winapi_OpenFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    DWORD desired_access;
+    BOOL inherit_handle;
+    LPCWSTR name;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStack(args, nargs, "kiu:OpenFileMappingW",
+        &desired_access, &inherit_handle, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_OpenFileMappingW_impl(module, desired_access, inherit_handle, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_OpenProcess__doc__,
 "OpenProcess($module, desired_access, inherit_handle, process_id, /)\n"
 "--\n"
@@ -764,6 +885,37 @@ _winapi_TerminateProcess(PyObject *module, PyObject *const *args, Py_ssize_t nar
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_VirtualQuerySize__doc__,
+"VirtualQuerySize($module, address, /)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_VIRTUALQUERYSIZE_METHODDEF    \
+    {"VirtualQuerySize", (PyCFunction)_winapi_VirtualQuerySize, METH_O, _winapi_VirtualQuerySize__doc__},
+
+static size_t
+_winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address);
+
+static PyObject *
+_winapi_VirtualQuerySize(PyObject *module, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    LPCVOID address;
+    size_t _return_value;
+
+    if (!PyArg_Parse(arg, "" F_POINTER ":VirtualQuerySize", &address)) {
+        goto exit;
+    }
+    _return_value = _winapi_VirtualQuerySize_impl(module, address);
+    if ((_return_value == (size_t)-1) && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = PyLong_FromSize_t(_return_value);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_WaitNamedPipe__doc__,
 "WaitNamedPipe($module, name, timeout, /)\n"
 "--\n"
@@ -945,4 +1097,4 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=5063c84b2d125488 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=392caab421348fe7 input=a9049054013a1b77]*/

From 715ded90abdde274159b219172a844a3a7da8424 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 10:48:58 -0600
Subject: [PATCH 35/41] Removed ShareableList.copy as redundant.

---
 Doc/library/multiprocessing.shared_memory.rst | 5 -----
 Lib/multiprocessing/shared_memory.py          | 8 --------
 Lib/test/_test_multiprocessing.py             | 2 +-
 3 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 5bcea8ab3e16b5..fd76ea5cafe031 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -285,11 +285,6 @@ shared memory blocks created using that manager are all released when the
    existing ``ShareableList``, specify its shared memory block's unique
    name while leaving ``sequence`` set to ``None``.
 
-   .. method:: copy()
-
-      Returns a shallow copy as a new instance backed by a new and distinct
-      shared memory block.
-
    .. method:: count(value)
 
       Returns the number of occurrences of ``value``.
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index b7f3ae2f80c9df..ea3eb60fa472ff 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -502,14 +502,6 @@ def _offset_packing_formats(self):
     def _offset_back_transform_codes(self):
         return self._offset_packing_formats + self._list_len * 8
 
-    def copy(self, *, name=None):
-        "L.copy() -> ShareableList -- a shallow copy of L."
-
-        if name is None:
-            return self.__class__(self)
-        else:
-            return self.__class__(self, name=name)
-
     def count(self, value):
         "L.count(value) -> integer -- return number of occurrences of value."
 
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 985664cd43c740..c45463aca160cc 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3823,7 +3823,7 @@ def test_shared_memory_ShareableList_basics(self):
             self.assertEqual(sl.count(b'adios'), 0)
 
         # Exercise creating a duplicate.
-        sl_copy = sl.copy(name='test03_duplicate')
+        sl_copy = shared_memory.ShareableList(sl, name='test03_duplicate')
         try:
             self.assertNotEqual(sl.shm.name, sl_copy.shm.name)
             self.assertEqual('test03_duplicate', sl_copy.shm.name)

From 68785333a25d902a42252ace3a8268357efd6094 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 11:54:05 -0600
Subject: [PATCH 36/41] Changes to _winapi use from eryksun feedback.

---
 Lib/multiprocessing/shared_memory.py | 31 +++++++++-------
 Modules/_winapi.c                    | 55 +++++++++++++++++++++-------
 Modules/clinic/_winapi.c.h           | 44 +++++++++++-----------
 3 files changed, 82 insertions(+), 48 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index ea3eb60fa472ff..24c382c8e9db97 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -15,6 +15,7 @@
 from .managers import dispatch, BaseManager, Server, State, ProcessError
 from . import util
 import os
+import errno
 import struct
 import secrets
 try:
@@ -28,10 +29,6 @@
 if os.name == "nt":
     import _winapi
 
-    PAGE_READONLY = 0x02
-    PAGE_EXECUTE_READWRITE = 0x04
-    INVALID_HANDLE_VALUE = -1
-
 
 class WindowsNamedSharedMemory:
 
@@ -42,12 +39,15 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if size == 0:
             # Attempt to dynamically determine the existing named shared
             # memory block's size which is likely a multiple of mmap.PAGESIZE.
+            h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False, name)
             try:
-                h_map = _winapi.OpenFileMappingW(PAGE_READONLY, False, name)
-            except OSError:
-                raise FileNotFoundError(name)
-            try:
-                p_buf = _winapi.MapViewOfFile(h_map, PAGE_READONLY, 0, 0, 0)
+                p_buf = _winapi.MapViewOfFile(
+                    h_map,
+                    _winapi.FILE_MAP_READ,
+                    0,
+                    0,
+                    0
+                )
             finally:
                 _winapi.CloseHandle(h_map)
             size = _winapi.VirtualQuerySize(p_buf)
@@ -55,10 +55,10 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
         if flags == O_CREX:
             # Create and reserve shared memory block with this name until
             # it can be attached to by mmap.
-            h_map = _winapi.CreateFileMappingW(
-                INVALID_HANDLE_VALUE,
+            h_map = _winapi.CreateFileMapping(
+                _winapi.INVALID_HANDLE_VALUE,
                 _winapi.NULL,
-                PAGE_EXECUTE_READWRITE,
+                _winapi.PAGE_READWRITE,
                 (size >> 32) & 0xFFFFFFFF,
                 size & 0xFFFFFFFF,
                 name
@@ -66,7 +66,12 @@ def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
             try:
                 last_error_code = _winapi.GetLastError()
                 if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
-                    raise FileExistsError(f"File exists: {name!r}")
+                    raise FileExistsError(
+                        errno.EEXIST,
+                        os.strerror(errno.EEXIST),
+                        name,
+                        _winapi.ERROR_ALREADY_EXISTS
+                    )
                 self._mmap = mmap.mmap(-1, size, tagname=name)
             finally:
                 _winapi.CloseHandle(h_map)
diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index c949c01a2a2202..1996bd59c03365 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -475,7 +475,7 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
 }
 
 /*[clinic input]
-_winapi.CreateFileMappingW -> HANDLE
+_winapi.CreateFileMapping -> HANDLE
 
     file_handle: HANDLE
     security_attributes: LPSECURITY_ATTRIBUTES
@@ -487,11 +487,11 @@ _winapi.CreateFileMappingW -> HANDLE
 [clinic start generated code]*/
 
 static HANDLE
-_winapi_CreateFileMappingW_impl(PyObject *module, HANDLE file_handle,
-                                LPSECURITY_ATTRIBUTES security_attributes,
-                                DWORD protect, DWORD max_size_high,
-                                DWORD max_size_low, LPCWSTR name)
-/*[clinic end generated code: output=c6b017501c929de1 input=35cadabe53b3b4da]*/
+_winapi_CreateFileMapping_impl(PyObject *module, HANDLE file_handle,
+                               LPSECURITY_ATTRIBUTES security_attributes,
+                               DWORD protect, DWORD max_size_high,
+                               DWORD max_size_low, LPCWSTR name)
+/*[clinic end generated code: output=6c0a4d5cf7f6fcc6 input=3dc5cf762a74dee8]*/
 {
     HANDLE handle;
 
@@ -502,7 +502,7 @@ _winapi_CreateFileMappingW_impl(PyObject *module, HANDLE file_handle,
     Py_END_ALLOW_THREADS
 
     if (handle == NULL) {
-        PyErr_SetFromWindowsErr(0);
+        PyErr_SetFromWindowsErrWithUnicodeFilename(0, name);
         handle = INVALID_HANDLE_VALUE;
     }
 
@@ -1371,7 +1371,7 @@ _winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map,
 }
 
 /*[clinic input]
-_winapi.OpenFileMappingW -> HANDLE
+_winapi.OpenFileMapping -> HANDLE
 
     desired_access: DWORD
     inherit_handle: BOOL
@@ -1380,9 +1380,9 @@ _winapi.OpenFileMappingW -> HANDLE
 [clinic start generated code]*/
 
 static HANDLE
-_winapi_OpenFileMappingW_impl(PyObject *module, DWORD desired_access,
-                              BOOL inherit_handle, LPCWSTR name)
-/*[clinic end generated code: output=ad829d0e68cac379 input=68fa4e0f2d5d5c42]*/
+_winapi_OpenFileMapping_impl(PyObject *module, DWORD desired_access,
+                             BOOL inherit_handle, LPCWSTR name)
+/*[clinic end generated code: output=08cc44def1cb11f1 input=131f2a405359de7f]*/
 {
     HANDLE handle;
 
@@ -1848,7 +1848,7 @@ static PyMethodDef winapi_functions[] = {
     _WINAPI_CLOSEHANDLE_METHODDEF
     _WINAPI_CONNECTNAMEDPIPE_METHODDEF
     _WINAPI_CREATEFILE_METHODDEF
-    _WINAPI_CREATEFILEMAPPINGW_METHODDEF
+    _WINAPI_CREATEFILEMAPPING_METHODDEF
     _WINAPI_CREATENAMEDPIPE_METHODDEF
     _WINAPI_CREATEPIPE_METHODDEF
     _WINAPI_CREATEPROCESS_METHODDEF
@@ -1862,7 +1862,7 @@ static PyMethodDef winapi_functions[] = {
     _WINAPI_GETSTDHANDLE_METHODDEF
     _WINAPI_GETVERSION_METHODDEF
     _WINAPI_MAPVIEWOFFILE_METHODDEF
-    _WINAPI_OPENFILEMAPPINGW_METHODDEF
+    _WINAPI_OPENFILEMAPPING_METHODDEF
     _WINAPI_OPENPROCESS_METHODDEF
     _WINAPI_PEEKNAMEDPIPE_METHODDEF
     _WINAPI_READFILE_METHODDEF
@@ -1932,11 +1932,34 @@ PyInit__winapi(void)
     WINAPI_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED);
     WINAPI_CONSTANT(F_DWORD, FILE_GENERIC_READ);
     WINAPI_CONSTANT(F_DWORD, FILE_GENERIC_WRITE);
+    WINAPI_CONSTANT(F_DWORD, FILE_MAP_ALL_ACCESS);
+    WINAPI_CONSTANT(F_DWORD, FILE_MAP_COPY);
+    WINAPI_CONSTANT(F_DWORD, FILE_MAP_EXECUTE);
+    WINAPI_CONSTANT(F_DWORD, FILE_MAP_READ);
+    WINAPI_CONSTANT(F_DWORD, FILE_MAP_WRITE);
     WINAPI_CONSTANT(F_DWORD, GENERIC_READ);
     WINAPI_CONSTANT(F_DWORD, GENERIC_WRITE);
     WINAPI_CONSTANT(F_DWORD, INFINITE);
+    WINAPI_CONSTANT(F_HANDLE, INVALID_HANDLE_VALUE);
+    WINAPI_CONSTANT(F_DWORD, MEM_COMMIT);
+    WINAPI_CONSTANT(F_DWORD, MEM_FREE);
+    WINAPI_CONSTANT(F_DWORD, MEM_IMAGE);
+    WINAPI_CONSTANT(F_DWORD, MEM_MAPPED);
+    WINAPI_CONSTANT(F_DWORD, MEM_PRIVATE);
+    WINAPI_CONSTANT(F_DWORD, MEM_RESERVE);
     WINAPI_CONSTANT(F_DWORD, NMPWAIT_WAIT_FOREVER);
     WINAPI_CONSTANT(F_DWORD, OPEN_EXISTING);
+    WINAPI_CONSTANT(F_DWORD, PAGE_EXECUTE);
+    WINAPI_CONSTANT(F_DWORD, PAGE_EXECUTE_READ);
+    WINAPI_CONSTANT(F_DWORD, PAGE_EXECUTE_READWRITE);
+    WINAPI_CONSTANT(F_DWORD, PAGE_EXECUTE_WRITECOPY);
+    WINAPI_CONSTANT(F_DWORD, PAGE_GUARD);
+    WINAPI_CONSTANT(F_DWORD, PAGE_NOACCESS);
+    WINAPI_CONSTANT(F_DWORD, PAGE_NOCACHE);
+    WINAPI_CONSTANT(F_DWORD, PAGE_READONLY);
+    WINAPI_CONSTANT(F_DWORD, PAGE_READWRITE);
+    WINAPI_CONSTANT(F_DWORD, PAGE_WRITECOMBINE);
+    WINAPI_CONSTANT(F_DWORD, PAGE_WRITECOPY);
     WINAPI_CONSTANT(F_DWORD, PIPE_ACCESS_DUPLEX);
     WINAPI_CONSTANT(F_DWORD, PIPE_ACCESS_INBOUND);
     WINAPI_CONSTANT(F_DWORD, PIPE_READMODE_MESSAGE);
@@ -1945,6 +1968,12 @@ PyInit__winapi(void)
     WINAPI_CONSTANT(F_DWORD, PIPE_WAIT);
     WINAPI_CONSTANT(F_DWORD, PROCESS_ALL_ACCESS);
     WINAPI_CONSTANT(F_DWORD, PROCESS_DUP_HANDLE);
+    WINAPI_CONSTANT(F_DWORD, SEC_COMMIT);
+    WINAPI_CONSTANT(F_DWORD, SEC_IMAGE);
+    WINAPI_CONSTANT(F_DWORD, SEC_LARGE_PAGES);
+    WINAPI_CONSTANT(F_DWORD, SEC_NOCACHE);
+    WINAPI_CONSTANT(F_DWORD, SEC_RESERVE);
+    WINAPI_CONSTANT(F_DWORD, SEC_WRITECOMBINE);
     WINAPI_CONSTANT(F_DWORD, STARTF_USESHOWWINDOW);
     WINAPI_CONSTANT(F_DWORD, STARTF_USESTDHANDLES);
     WINAPI_CONSTANT(F_DWORD, STD_INPUT_HANDLE);
diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
index 1bb5ab3c45d5ca..e21f2bc2b6fd6f 100644
--- a/Modules/clinic/_winapi.c.h
+++ b/Modules/clinic/_winapi.c.h
@@ -168,23 +168,23 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
     return return_value;
 }
 
-PyDoc_STRVAR(_winapi_CreateFileMappingW__doc__,
-"CreateFileMappingW($module, file_handle, security_attributes, protect,\n"
-"                   max_size_high, max_size_low, name, /)\n"
+PyDoc_STRVAR(_winapi_CreateFileMapping__doc__,
+"CreateFileMapping($module, file_handle, security_attributes, protect,\n"
+"                  max_size_high, max_size_low, name, /)\n"
 "--\n"
 "\n");
 
-#define _WINAPI_CREATEFILEMAPPINGW_METHODDEF    \
-    {"CreateFileMappingW", (PyCFunction)(void(*)(void))_winapi_CreateFileMappingW, METH_FASTCALL, _winapi_CreateFileMappingW__doc__},
+#define _WINAPI_CREATEFILEMAPPING_METHODDEF    \
+    {"CreateFileMapping", (PyCFunction)(void(*)(void))_winapi_CreateFileMapping, METH_FASTCALL, _winapi_CreateFileMapping__doc__},
 
 static HANDLE
-_winapi_CreateFileMappingW_impl(PyObject *module, HANDLE file_handle,
-                                LPSECURITY_ATTRIBUTES security_attributes,
-                                DWORD protect, DWORD max_size_high,
-                                DWORD max_size_low, LPCWSTR name);
+_winapi_CreateFileMapping_impl(PyObject *module, HANDLE file_handle,
+                               LPSECURITY_ATTRIBUTES security_attributes,
+                               DWORD protect, DWORD max_size_high,
+                               DWORD max_size_low, LPCWSTR name);
 
 static PyObject *
-_winapi_CreateFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+_winapi_CreateFileMapping(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
     HANDLE file_handle;
@@ -195,11 +195,11 @@ _winapi_CreateFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t n
     LPCWSTR name;
     HANDLE _return_value;
 
-    if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkku:CreateFileMappingW",
+    if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkku:CreateFileMapping",
         &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, &name)) {
         goto exit;
     }
-    _return_value = _winapi_CreateFileMappingW_impl(module, file_handle, security_attributes, protect, max_size_high, max_size_low, name);
+    _return_value = _winapi_CreateFileMapping_impl(module, file_handle, security_attributes, protect, max_size_high, max_size_low, name);
     if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
         goto exit;
     }
@@ -685,20 +685,20 @@ _winapi_MapViewOfFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
     return return_value;
 }
 
-PyDoc_STRVAR(_winapi_OpenFileMappingW__doc__,
-"OpenFileMappingW($module, desired_access, inherit_handle, name, /)\n"
+PyDoc_STRVAR(_winapi_OpenFileMapping__doc__,
+"OpenFileMapping($module, desired_access, inherit_handle, name, /)\n"
 "--\n"
 "\n");
 
-#define _WINAPI_OPENFILEMAPPINGW_METHODDEF    \
-    {"OpenFileMappingW", (PyCFunction)(void(*)(void))_winapi_OpenFileMappingW, METH_FASTCALL, _winapi_OpenFileMappingW__doc__},
+#define _WINAPI_OPENFILEMAPPING_METHODDEF    \
+    {"OpenFileMapping", (PyCFunction)(void(*)(void))_winapi_OpenFileMapping, METH_FASTCALL, _winapi_OpenFileMapping__doc__},
 
 static HANDLE
-_winapi_OpenFileMappingW_impl(PyObject *module, DWORD desired_access,
-                              BOOL inherit_handle, LPCWSTR name);
+_winapi_OpenFileMapping_impl(PyObject *module, DWORD desired_access,
+                             BOOL inherit_handle, LPCWSTR name);
 
 static PyObject *
-_winapi_OpenFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+_winapi_OpenFileMapping(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
     DWORD desired_access;
@@ -706,11 +706,11 @@ _winapi_OpenFileMappingW(PyObject *module, PyObject *const *args, Py_ssize_t nar
     LPCWSTR name;
     HANDLE _return_value;
 
-    if (!_PyArg_ParseStack(args, nargs, "kiu:OpenFileMappingW",
+    if (!_PyArg_ParseStack(args, nargs, "kiu:OpenFileMapping",
         &desired_access, &inherit_handle, &name)) {
         goto exit;
     }
-    _return_value = _winapi_OpenFileMappingW_impl(module, desired_access, inherit_handle, name);
+    _return_value = _winapi_OpenFileMapping_impl(module, desired_access, inherit_handle, name);
     if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
         goto exit;
     }
@@ -1097,4 +1097,4 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=392caab421348fe7 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=f3897898ea1da99d input=a9049054013a1b77]*/

From 0d3d06f7cbcb47b2dee6c027c35265795052c899 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 21:57:33 -0600
Subject: [PATCH 37/41] Adopt simpler SharedMemory API, collapsing
 PosixSharedMemory and WindowsNamedSharedMemory into one.

---
 Doc/library/multiprocessing.shared_memory.rst |  70 ++--
 Lib/multiprocessing/shared_memory.py          | 321 ++++++++----------
 Lib/test/_test_multiprocessing.py             |  44 +--
 3 files changed, 189 insertions(+), 246 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index fd76ea5cafe031..607022061df3d4 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -35,41 +35,33 @@ or other communications requiring the serialization/deserialization and
 copying of data.
 
 
-.. class:: SharedMemory(name, flags=None, mode=0o600, size=0, read_only=False)
+.. class:: SharedMemory(name=None, create=False, size=0)
 
-   This class creates and returns an instance of either a
-   :class:`PosixSharedMemory` or :class:`NamedSharedMemory` class depending
-   upon their availability on the local system.
+   Creates a new shared memory block or attaches to an existing shared
+   memory block.  Each shared memory block is assigned a unique name.
+   In this way, one process can create a shared memory block with a
+   particular name and a different process can attach to that same shared
+   memory block using that same name.
+
+   As a resource for sharing data across processes, shared memory blocks
+   may outlive the original process that created them.  When one process
+   no longer needs access to a shared memory block that might still be
+   needed by other processes, the :meth:`close()` method should be called.
+   When a shared memory block is no longer needed by any process, the
+   :meth:`unlink()` method should be called to ensure proper cleanup.
 
    *name* is the unique name for the requested shared memory, specified as
-   a string.  If ``None`` is supplied for the name, a new shared memory
-   block with a novel name will be created without needing to also specify
-   ``flags``.
-
-   *flags* is set to ``None`` when attempting to attach to an existing shared
-   memory block by its unique name but if no existing block has that name, an
-   exception will be raised.  To request the creation of a new shared
-   memory block, set to ``O_CREX``.  To request the optional creation of a
-   new shared memory block or attach to an existing one by the same name,
-   set to ``O_CREAT``.
-
-   *mode* controls user/group/all-based read/write permissions on the
-   shared memory block.  Its specification is not enforceable on all platforms.
-
-   *size* specifies the number of bytes requested for a shared memory block.
-   Because some platforms choose to allocate chunks of memory based upon
-   that platform's memory page size, the exact size of the shared memory
-   block may be larger or equal to the size requested.  When attaching to an
-   existing shared memory block, set to ``0`` (which is the default).
-   Requesting a size greater than the original when attaching to an existing
-   shared memory block will attempt a resize of the shared memory block
-   which may or may not be successful.  Requesting a size smaller than the
-   original will attempt to attach to the first N bytes of the existing
-   shared memory block but may still give access to the full allocated size.
-
-   *read_only* controls whether a shared memory block is to be available
-   for only reading or for both reading and writing.  Its specification is
-   not enforceable on all platforms.
+   a string.  When creating a new shared memory block, if ``None`` (the
+   default) is supplied for the name, a novel name will be generated.
+
+   *create* controls whether a new shared memory block is created (``True``)
+   or an existing shared memory block is attached (``False``).
+
+   *size* specifies the requested number of bytes when creating a new shared
+   memory block.  Because some platforms choose to allocate chunks of memory
+   based upon that platform's memory page size, the exact size of the shared
+   memory block may be larger or equal to the size requested.  When attaching
+   to an existing shared memory block, the ``size`` parameter is ignored.
 
    .. method:: close()
 
@@ -100,10 +92,6 @@ copying of data.
 
       Read-only access to the unique name of the shared memory block.
 
-   .. attribute:: mode
-
-      Read-only access to access permissions mode of the shared memory block.
-
    .. attribute:: size
 
       Read-only access to size in bytes of the shared memory block.
@@ -113,7 +101,7 @@ The following example demonstrates low-level use of :class:`SharedMemory`
 instances::
 
    >>> from multiprocessing import shared_memory
-   >>> shm_a = shared_memory.SharedMemory(None, size=10)
+   >>> shm_a = shared_memory.SharedMemory(create=True, size=10)
    >>> type(shm_a.buf)
    <class 'memoryview'>
    >>> buffer = shm_a.buf
@@ -146,7 +134,7 @@ same ``numpy.ndarray`` from two distinct Python shells:
    >>> import numpy as np
    >>> a = np.array([1, 1, 2, 3, 5, 8])  # Start with an existing NumPy array
    >>> from multiprocessing import shared_memory
-   >>> shm = shared_memory.SharedMemory(None, size=a.nbytes)
+   >>> shm = shared_memory.SharedMemory(create=True, size=a.nbytes)
    >>> # Now create a NumPy array backed by shared memory
    >>> b = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf)
    >>> b[:] = a[:]  # Copy the original data into shared memory
@@ -163,7 +151,7 @@ same ``numpy.ndarray`` from two distinct Python shells:
    >>> import numpy as np
    >>> from multiprocessing import shared_memory
    >>> # Attach to the existing shared memory block
-   >>> existing_shm = shared_memory.SharedMemory('psm_21467_46075')
+   >>> existing_shm = shared_memory.SharedMemory(name='psm_21467_46075')
    >>> # Note that a.shape is (6,) and a.dtype is np.int64 in this example
    >>> c = np.ndarray((6,), dtype=np.int64, buffer=existing_shm.buf)
    >>> c
@@ -252,8 +240,8 @@ needed:
    >>> with shared_memory.SharedMemoryManager() as smm:
    ...     sl = smm.ShareableList(range(2000))
    ...     # Divide the work among two processes, storing partial results in sl
-   ...     p1 = Process(target=do_work, args=(sl.shm.name, 0, 1000))
-   ...     p2 = Process(target=do_work, args=(sl.shm.name, 1000, 2000))
+   ...     p1 = Process(target=do_work, args=(sl, 0, 1000))
+   ...     p2 = Process(target=do_work, args=(sl, 1000, 2000))
    ...     p1.start()
    ...     p2.start()  # A multiprocessing.Pool might be more efficient
    ...     p1.join()
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 24c382c8e9db97..7428b6561957ea 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -5,8 +5,7 @@
 """
 
 
-__all__ = [ 'SharedMemory', 'PosixSharedMemory', 'WindowsNamedSharedMemory',
-            'ShareableList',
+__all__ = [ 'SharedMemory', 'ShareableList',
             'SharedMemoryServer', 'SharedMemoryManager' ]
 
 
@@ -18,124 +17,29 @@
 import errno
 import struct
 import secrets
-try:
-    import  _posixshmem
-    from os import O_CREAT, O_EXCL, O_TRUNC
-except ImportError as ie:
-    O_CREAT, O_EXCL, O_TRUNC = 64, 128, 512
-
-O_CREX = O_CREAT | O_EXCL
 
 if os.name == "nt":
     import _winapi
+    _USE_POSIX = False
+else:
+    import _posixshmem
+    _USE_POSIX = True
 
 
-class WindowsNamedSharedMemory:
-
-    def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
-        if name is None:
-            name = _make_filename()
-
-        if size == 0:
-            # Attempt to dynamically determine the existing named shared
-            # memory block's size which is likely a multiple of mmap.PAGESIZE.
-            h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False, name)
-            try:
-                p_buf = _winapi.MapViewOfFile(
-                    h_map,
-                    _winapi.FILE_MAP_READ,
-                    0,
-                    0,
-                    0
-                )
-            finally:
-                _winapi.CloseHandle(h_map)
-            size = _winapi.VirtualQuerySize(p_buf)
-
-        if flags == O_CREX:
-            # Create and reserve shared memory block with this name until
-            # it can be attached to by mmap.
-            h_map = _winapi.CreateFileMapping(
-                _winapi.INVALID_HANDLE_VALUE,
-                _winapi.NULL,
-                _winapi.PAGE_READWRITE,
-                (size >> 32) & 0xFFFFFFFF,
-                size & 0xFFFFFFFF,
-                name
-            )
-            try:
-                last_error_code = _winapi.GetLastError()
-                if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
-                    raise FileExistsError(
-                        errno.EEXIST,
-                        os.strerror(errno.EEXIST),
-                        name,
-                        _winapi.ERROR_ALREADY_EXISTS
-                    )
-                self._mmap = mmap.mmap(-1, size, tagname=name)
-            finally:
-                _winapi.CloseHandle(h_map)
-
-        else:
-            self._mmap = mmap.mmap(-1, size, tagname=name)
-
-        self._buf = memoryview(self._mmap)
-        self.name = name
-        self.mode = mode
-        self._size = size
-
-    @property
-    def size(self):
-        "Size in bytes."
-        return self._size
-
-    @property
-    def buf(self):
-        "A memoryview of contents of the shared memory block."
-        return self._buf
-
-    def __reduce__(self):
-        return (
-            self.__class__,
-            (
-                self.name,
-                None,
-                self.mode,
-                0,
-                False,
-            ),
-        )
-
-    def __repr__(self):
-        return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
-
-    def close(self):
-        if self._buf is not None:
-            self._buf.release()
-            self._buf = None
-        if self._mmap is not None:
-            self._mmap.close()
-            self._mmap = None
-
-    def unlink(self):
-        """Windows ensures that destruction of the last reference to this
-        named shared memory block will result in the release of this memory."""
-        pass
-
+O_CREX = os.O_CREAT | os.O_EXCL
 
 # FreeBSD (and perhaps other BSDs) limit names to 14 characters.
 _SHM_SAFE_NAME_LENGTH = 14
 
-# shared object name prefix
-if os.name == "nt":
-    _SHM_NAME_PREFIX = 'wnsm_'
+# Shared memory block name prefix
+if _USE_POSIX:
+    _SHM_NAME_PREFIX = 'psm_'
 else:
-    _SHM_NAME_PREFIX = '/psm_'
+    _SHM_NAME_PREFIX = 'wnsm_'
 
 
 def _make_filename():
-    """Create a random filename for the shared memory object.
-    """
+    "Create a random filename for the shared memory object."
     # number of random bytes to use for name
     nbytes = (_SHM_SAFE_NAME_LENGTH - len(_SHM_NAME_PREFIX)) // 2
     assert nbytes >= 2, '_SHM_NAME_PREFIX too long'
@@ -144,112 +48,173 @@ def _make_filename():
     return name
 
 
-class PosixSharedMemory:
+class SharedMemory:
 
-    # defaults so close() and unlink() can run without errors
-    fd = -1
-    name = None
+    # Defaults; enables close() and unlink() to run without errors.
+    _name = None
+    _fd = -1
     _mmap = None
     _buf = None
+    _flags = os.O_RDWR
+    _mode = 0o600
 
-    def __init__(self, name, flags=None, mode=384, size=0, read_only=False):
-        if name and (flags is None):
-            flags = 0
-        else:
-            flags = O_CREX if flags is None else flags
-        if flags & O_EXCL and not flags & O_CREAT:
-            raise ValueError("O_EXCL must be combined with O_CREAT")
-        if name is None and not flags & O_EXCL:
-            raise ValueError("'name' can only be None if O_EXCL is set")
-        flags |= os.O_RDONLY if read_only else os.O_RDWR
-        self.flags = flags
-        self.mode = mode
+    def __init__(self, name=None, create=False, size=0):
         if not size >= 0:
             raise ValueError("'size' must be a positive integer")
-        if name is None:
-            self._open_retry()
-        else:
-            self.name = name
-            self.fd = _posixshmem.shm_open(name, flags, mode=mode)
-        if size:
+        if create:
+            self._flags = O_CREX | os.O_RDWR
+        if name is None and not self._flags & os.O_EXCL:
+            raise ValueError("'name' can only be None if create=True")
+
+        if _USE_POSIX:
+
+            # POSIX Shared Memory
+
+            if name is None:
+                while True:
+                    name = _make_filename()
+                    try:
+                        self._fd = _posixshmem.shm_open(
+                            name,
+                            self._flags,
+                            mode=self._mode
+                        )
+                    except FileExistsError:
+                        continue
+                    self._name = name
+                    break
+            else:
+                self._fd = _posixshmem.shm_open(
+                    name,
+                    self._flags,
+                    mode=self._mode
+                )
+                self._name = name
             try:
-                os.ftruncate(self.fd, size)
+                if create and size:
+                    os.ftruncate(self._fd, size)
+                stats = os.fstat(self._fd)
+                size = stats.st_size
+                self._mmap = mmap.mmap(self._fd, size)
             except OSError:
                 self.unlink()
                 raise
-        self._mmap = mmap.mmap(self.fd, self.size)
-        self._buf = memoryview(self._mmap)
 
-    @property
-    def size(self):
-        "Size in bytes."
-        if self.fd >= 0:
-            return os.fstat(self.fd).st_size
         else:
-            return 0
 
-    @property
-    def buf(self):
-        "A memoryview of contents of the shared memory block."
-        return self._buf
+            # Windows Named Shared Memory
+
+            if create:
+                while True:
+                    temp_name = _make_filename() if name is None else name
+                    # Create and reserve shared memory block with this name
+                    # until it can be attached to by mmap.
+                    h_map = _winapi.CreateFileMapping(
+                        _winapi.INVALID_HANDLE_VALUE,
+                        _winapi.NULL,
+                        _winapi.PAGE_READWRITE,
+                        (size >> 32) & 0xFFFFFFFF,
+                        size & 0xFFFFFFFF,
+                        temp_name
+                    )
+                    try:
+                        last_error_code = _winapi.GetLastError()
+                        if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
+                            if name is not None:
+                                raise FileExistsError(
+                                    errno.EEXIST,
+                                    os.strerror(errno.EEXIST),
+                                    name,
+                                    _winapi.ERROR_ALREADY_EXISTS
+                                )
+                            else:
+                                continue
+                        self._mmap = mmap.mmap(-1, size, tagname=temp_name)
+                    finally:
+                        _winapi.CloseHandle(h_map)
+                    self._name = temp_name
+                    break
 
-    def _open_retry(self):
-        # generate a random name, open, retry if it exists
-        while True:
-            name = _make_filename()
-            try:
-                self.fd = _posixshmem.shm_open(name, self.flags,
-                                               mode=self.mode)
-            except FileExistsError:
-                continue
-            self.name = name
-            break
+            else:
+                self._name = name
+                # Dynamically determine the existing named shared memory
+                # block's size which is likely a multiple of mmap.PAGESIZE.
+                h_map = _winapi.OpenFileMapping(
+                    _winapi.FILE_MAP_READ,
+                    False,
+                    name
+                )
+                try:
+                    p_buf = _winapi.MapViewOfFile(
+                        h_map,
+                        _winapi.FILE_MAP_READ,
+                        0,
+                        0,
+                        0
+                    )
+                finally:
+                    _winapi.CloseHandle(h_map)
+                size = _winapi.VirtualQuerySize(p_buf)
+                self._mmap = mmap.mmap(-1, size, tagname=name)
+
+        self._size = size
+        self._buf = memoryview(self._mmap)
+
+    def __del__(self):
+        try:
+            self.close()
+        except OSError:
+            pass
 
     def __reduce__(self):
         return (
             self.__class__,
             (
                 self.name,
-                None,
-                self.mode,
-                0,
                 False,
+                self.size,
             ),
         )
 
     def __repr__(self):
         return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
 
-    def unlink(self):
-        if self.name:
-            _posixshmem.shm_unlink(self.name)
+    @property
+    def buf(self):
+        "A memoryview of contents of the shared memory block."
+        return self._buf
+
+    @property
+    def name(self):
+        "Unique name that identifies the shared memory block."
+        return self._name
+
+    @property
+    def size(self):
+        "Size in bytes."
+        return self._size
 
     def close(self):
+        """Closes access to the shared memory from this instance but does
+        not destroy the shared memory block."""
         if self._buf is not None:
             self._buf.release()
             self._buf = None
         if self._mmap is not None:
             self._mmap.close()
             self._mmap = None
-        if self.fd >= 0:
-            os.close(self.fd)
-            self.fd = -1
+        if _USE_POSIX and self._fd >= 0:
+            os.close(self._fd)
+            self._fd = -1
 
-    def __del__(self):
-        try:
-            self.close()
-        except OSError:
-            pass
-
-
-class SharedMemory:
+    def unlink(self):
+        """Requests that the underlying shared memory block be destroyed.
 
-    def __new__(cls, *args, **kwargs):
-        if os.name == 'nt':
-            cls = WindowsNamedSharedMemory
-        else:
-            cls = PosixSharedMemory
-        return cls(*args, **kwargs)
+        In order to ensure proper cleanup of resources, unlink should be
+        called once (and only once) across all processes which have access
+        to the shared memory block."""
+        if _USE_POSIX and self.name:
+            _posixshmem.shm_unlink(self.name)
 
 
 encoding = "utf8"
@@ -326,7 +291,7 @@ def __init__(self, sequence=None, *, name=None):
         if name is not None and sequence is None:
             self.shm = SharedMemory(name)
         else:
-            self.shm = SharedMemory(name, flags=O_CREX, size=requested_size)
+            self.shm = SharedMemory(name, create=True, size=requested_size)
 
         if sequence is not None:
             _enc = encoding
@@ -638,7 +603,7 @@ def SharedMemory(self, size):
         """Returns a new SharedMemory instance with the specified size in
         bytes, to be tracked by the manager."""
         with self._Client(self._address, authkey=self._authkey) as conn:
-            sms = SharedMemory(None, flags=O_CREX, size=size)
+            sms = SharedMemory(None, create=True, size=size)
             try:
                 dispatch(conn, None, 'track_segment', (sms.name,))
             except BaseException as e:
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c45463aca160cc..048b2bfe4e2826 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3633,18 +3633,13 @@ def _attach_existing_shmem_then_write(shmem_name_or_obj, binary_data):
         local_sms.close()
 
     def test_shared_memory_basics(self):
-        sms = shared_memory.SharedMemory(
-            'test01_tsmb',
-            flags=shared_memory.O_CREX,
-            size=512
-        )
+        sms = shared_memory.SharedMemory('test01_tsmb', create=True, size=512)
         self.addCleanup(sms.unlink)
 
         # Verify attributes are readable.
         self.assertEqual(sms.name, 'test01_tsmb')
         self.assertGreaterEqual(sms.size, 512)
         self.assertGreaterEqual(len(sms.buf), sms.size)
-        self.assertEqual(sms.mode, 0o600)
 
         # Modify contents of shared memory segment through memoryview.
         sms.buf[0] = 42
@@ -3655,7 +3650,7 @@ def test_shared_memory_basics(self):
         self.assertEqual(also_sms.buf[0], 42)
         also_sms.close()
 
-        if isinstance(sms, shared_memory.PosixSharedMemory):
+        if shared_memory._USE_POSIX:
             # Posix Shared Memory can only be unlinked once.  Here we
             # test an implementation detail that is not observed across
             # all supported platforms (since WindowsNamedSharedMemory
@@ -3665,7 +3660,7 @@ def test_shared_memory_basics(self):
             with self.assertRaises(FileNotFoundError):
                 sms_uno = shared_memory.SharedMemory(
                     'test01_dblunlink',
-                    flags=shared_memory.O_CREX,
+                    create=True,
                     size=5000
                 )
 
@@ -3688,23 +3683,22 @@ def test_shared_memory_basics(self):
             # name that is already in use triggers an exception.
             there_can_only_be_one_sms = shared_memory.SharedMemory(
                 'test01_tsmb',
-                flags=shared_memory.O_CREX,
+                create=True,
                 size=512
             )
 
-        # Requesting creation of a shared memory segment with the option
-        # to attach to an existing segment, if that name is currently in
-        # use, should not trigger an exception.
-        # Note:  Using a smaller size could possibly cause truncation of
-        # the existing segment but is OS platform dependent.  In the
-        # case of MacOS/darwin, requesting a smaller size is disallowed.
-        ok_if_exists_sms = shared_memory.SharedMemory(
-            'test01_tsmb',
-            flags=shared_memory.O_CREAT,
-            size=sms.size if sys.platform != 'darwin' else 0
-        )
-        self.assertEqual(ok_if_exists_sms.size, sms.size)
-        ok_if_exists_sms.close()
+        if shared_memory._USE_POSIX:
+            # Requesting creation of a shared memory segment with the option
+            # to attach to an existing segment, if that name is currently in
+            # use, should not trigger an exception.
+            # Note:  Using a smaller size could possibly cause truncation of
+            # the existing segment but is OS platform dependent.  In the
+            # case of MacOS/darwin, requesting a smaller size is disallowed.
+            class OptionalAttachSharedMemory(shared_memory.SharedMemory):
+                _flags = os.O_CREAT | os.O_RDWR
+            ok_if_exists_sms = OptionalAttachSharedMemory('test01_tsmb')
+            self.assertEqual(ok_if_exists_sms.size, sms.size)
+            ok_if_exists_sms.close()
 
         # Attempting to attach to an existing shared memory segment when
         # no segment exists with the supplied name triggers an exception.
@@ -3715,11 +3709,7 @@ def test_shared_memory_basics(self):
         sms.close()
 
     def test_shared_memory_across_processes(self):
-        sms = shared_memory.SharedMemory(
-            'test02_tsmap',
-            flags=shared_memory.O_CREX,
-            size=512
-        )
+        sms = shared_memory.SharedMemory('test02_tsmap', True, size=512)
         self.addCleanup(sms.unlink)
 
         # Verify remote attachment to existing block by name is working.

From 05e26dddbf9c2bbdb9a550b6cf411e15e24bcc88 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 22:23:02 -0600
Subject: [PATCH 38/41] Fix missing docstring on class, add test for ignoring
 size when attaching.

---
 Lib/multiprocessing/shared_memory.py | 14 ++++++++++++++
 Lib/test/_test_multiprocessing.py    |  8 +++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 7428b6561957ea..e7eb5f2f8ab86b 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -49,6 +49,20 @@ def _make_filename():
 
 
 class SharedMemory:
+    """Creates a new shared memory block or attaches to an existing
+    shared memory block.
+
+    Every shared memory block is assigned a unique name.  This enables
+    one process to create a shared memory block with a particular name 
+    so that a different process can attach to that same shared memory
+    block using that same name.
+
+    As a resource for sharing data across processes, shared memory blocks
+    may outlive the original process that created them.  When one process
+    no longer needs access to a shared memory block that might still be
+    needed by other processes, the close() method should be called.
+    When a shared memory block is no longer needed by any process, the
+    unlink() method should be called to ensure proper cleanup."""
 
     # Defaults; enables close() and unlink() to run without errors.
     _name = None
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 048b2bfe4e2826..2aef69d4f54138 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3650,6 +3650,11 @@ def test_shared_memory_basics(self):
         self.assertEqual(also_sms.buf[0], 42)
         also_sms.close()
 
+        # Attach to existing shared memory segment but specify a new size.
+        same_sms = shared_memory.SharedMemory('test01_tsmb', size=20*sms.size)
+        self.assertEqual(same_sms.size, sms.size)  # Size was ignored.
+        same_sms.close()
+
         if shared_memory._USE_POSIX:
             # Posix Shared Memory can only be unlinked once.  Here we
             # test an implementation detail that is not observed across
@@ -3675,9 +3680,6 @@ def test_shared_memory_basics(self):
                 finally:
                     sms_uno.unlink()  # A second shm_unlink() call is bad.
 
-        # Enforcement of `mode` and `read_only` is OS platform dependent
-        # and as such will not be tested here.
-
         with self.assertRaises(FileExistsError):
             # Attempting to create a new shared memory segment with a
             # name that is already in use triggers an exception.

From 7a3c7e5a1bd26c44d95126728eaf33f3213b12c6 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 22:48:25 -0600
Subject: [PATCH 39/41] Moved SharedMemoryManager to managers module, tweak to
 fragile test.

---
 Doc/library/multiprocessing.shared_memory.rst |   3 +-
 Lib/multiprocessing/managers.py               | 151 +++++++++++++++++-
 Lib/multiprocessing/shared_memory.py          | 143 +----------------
 Lib/test/_test_multiprocessing.py             |   6 +-
 4 files changed, 156 insertions(+), 147 deletions(-)

diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst
index 607022061df3d4..426fef62fa75c2 100644
--- a/Doc/library/multiprocessing.shared_memory.rst
+++ b/Doc/library/multiprocessing.shared_memory.rst
@@ -20,7 +20,8 @@ and management of shared memory to be accessed by one or more processes
 on a multicore or symmetric multiprocessor (SMP) machine.  To assist with
 the life-cycle management of shared memory especially across distinct
 processes, a :class:`~multiprocessing.managers.BaseManager` subclass,
-:class:`SharedMemoryManager`, is also provided.
+:class:`SharedMemoryManager`, is also provided in the
+``multiprocessing.managers`` module.
 
 In this module, shared memory refers to "System V style" shared memory blocks
 (though is not necessarily implemented explicitly as such) and does not refer
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index dbed993a38d65a..8ef6f59f099072 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -1,5 +1,5 @@
 #
-# Module providing the `SyncManager` class for dealing
+# Module providing manager classes for dealing
 # with shared objects
 #
 # multiprocessing/managers.py
@@ -8,7 +8,8 @@
 # Licensed to PSF under a Contributor Agreement.
 #
 
-__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
+__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token',
+            'SharedMemoryManager' ]
 
 #
 # Imports
@@ -19,6 +20,7 @@
 import array
 import queue
 import time
+from os import getpid
 
 from traceback import format_exc
 
@@ -28,6 +30,11 @@
 from . import process
 from . import util
 from . import get_context
+try:
+    from . import shared_memory
+    HAS_SHMEM = True
+except ImportError:
+    HAS_SHMEM = False
 
 #
 # Register some things for pickling
@@ -1200,3 +1207,143 @@ class SyncManager(BaseManager):
 # types returned by methods of PoolProxy
 SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
 SyncManager.register('AsyncResult', create_method=False)
+
+#
+# Definition of SharedMemoryManager and SharedMemoryServer
+#
+
+if HAS_SHMEM:
+    class _SharedMemoryTracker:
+        "Manages one or more shared memory segments."
+
+        def __init__(self, name, segment_names=[]):
+            self.shared_memory_context_name = name
+            self.segment_names = segment_names
+
+        def register_segment(self, segment_name):
+            "Adds the supplied shared memory block name to tracker."
+            util.debug(f"Register segment {segment_name!r} in pid {getpid()}")
+            self.segment_names.append(segment_name)
+
+        def destroy_segment(self, segment_name):
+            """Calls unlink() on the shared memory block with the supplied name
+            and removes it from the list of blocks being tracked."""
+            util.debug(f"Destroy segment {segment_name!r} in pid {getpid()}")
+            self.segment_names.remove(segment_name)
+            segment = shared_memory.SharedMemory(segment_name)
+            segment.close()
+            segment.unlink()
+
+        def unlink(self):
+            "Calls destroy_segment() on all tracked shared memory blocks."
+            for segment_name in self.segment_names[:]:
+                self.destroy_segment(segment_name)
+
+        def __del__(self):
+            util.debug(f"Call {self.__class__.__name__}.__del__ in {getpid()}")
+            self.unlink()
+
+        def __getstate__(self):
+            return (self.shared_memory_context_name, self.segment_names)
+
+        def __setstate__(self, state):
+            self.__init__(*state)
+
+
+    class SharedMemoryServer(Server):
+
+        public = Server.public + \
+                 ['track_segment', 'release_segment', 'list_segments']
+
+        def __init__(self, *args, **kwargs):
+            Server.__init__(self, *args, **kwargs)
+            self.shared_memory_context = \
+                _SharedMemoryTracker(f"shmm_{self.address}_{getpid()}")
+            util.debug(f"SharedMemoryServer started by pid {getpid()}")
+
+        def create(self, c, typeid, *args, **kwargs):
+            """Create a new distributed-shared object (not backed by a shared
+            memory block) and return its id to be used in a Proxy Object."""
+            # Unless set up as a shared proxy, don't make shared_memory_context
+            # a standard part of kwargs.  This makes things easier for supplying
+            # simple functions.
+            if hasattr(self.registry[typeid][-1], "_shared_memory_proxy"):
+                kwargs['shared_memory_context'] = self.shared_memory_context
+            return Server.create(self, c, typeid, *args, **kwargs)
+
+        def shutdown(self, c):
+            "Call unlink() on all tracked shared memory, terminate the Server."
+            self.shared_memory_context.unlink()
+            return Server.shutdown(self, c)
+
+        def track_segment(self, c, segment_name):
+            "Adds the supplied shared memory block name to Server's tracker."
+            self.shared_memory_context.register_segment(segment_name)
+
+        def release_segment(self, c, segment_name):
+            """Calls unlink() on the shared memory block with the supplied name
+            and removes it from the tracker instance inside the Server."""
+            self.shared_memory_context.destroy_segment(segment_name)
+
+        def list_segments(self, c):
+            """Returns a list of names of shared memory blocks that the Server
+            is currently tracking."""
+            return self.shared_memory_context.segment_names
+
+
+    class SharedMemoryManager(BaseManager):
+        """Like SyncManager but uses SharedMemoryServer instead of Server.
+
+        It provides methods for creating and returning SharedMemory instances
+        and for creating a list-like object (ShareableList) backed by shared
+        memory.  It also provides methods that create and return Proxy Objects
+        that support synchronization across processes (i.e. multi-process-safe
+        locks and semaphores).
+        """
+
+        _Server = SharedMemoryServer
+
+        def __init__(self, *args, **kwargs):
+            BaseManager.__init__(self, *args, **kwargs)
+            util.debug(f"{self.__class__.__name__} created by pid {getpid()}")
+
+        def __del__(self):
+            util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}")
+            pass
+
+        def get_server(self):
+            'Better than monkeypatching for now; merge into Server ultimately'
+            if self._state.value != State.INITIAL:
+                if self._state.value == State.STARTED:
+                    raise ProcessError("Already started SharedMemoryServer")
+                elif self._state.value == State.SHUTDOWN:
+                    raise ProcessError("SharedMemoryManager has shut down")
+                else:
+                    raise ProcessError(
+                        "Unknown state {!r}".format(self._state.value))
+            return self._Server(self._registry, self._address,
+                                self._authkey, self._serializer)
+
+        def SharedMemory(self, size):
+            """Returns a new SharedMemory instance with the specified size in
+            bytes, to be tracked by the manager."""
+            with self._Client(self._address, authkey=self._authkey) as conn:
+                sms = shared_memory.SharedMemory(None, create=True, size=size)
+                try:
+                    dispatch(conn, None, 'track_segment', (sms.name,))
+                except BaseException as e:
+                    sms.unlink()
+                    raise e
+            return sms
+
+        def ShareableList(self, sequence):
+            """Returns a new ShareableList instance populated with the values
+            from the input sequence, to be tracked by the manager."""
+            with self._Client(self._address, authkey=self._authkey) as conn:
+                sl = shared_memory.ShareableList(sequence)
+                try:
+                    dispatch(conn, None, 'track_segment', (sl.shm.name,))
+                except BaseException as e:
+                    sl.shm.unlink()
+                    raise e
+            return sl
diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index e7eb5f2f8ab86b..185bb0c8b82936 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -5,14 +5,11 @@
 """
 
 
-__all__ = [ 'SharedMemory', 'ShareableList',
-            'SharedMemoryServer', 'SharedMemoryManager' ]
+__all__ = [ 'SharedMemory', 'ShareableList' ]
 
 
 from functools import partial
 import mmap
-from .managers import dispatch, BaseManager, Server, State, ProcessError
-from . import util
 import os
 import errno
 import struct
@@ -53,7 +50,7 @@ class SharedMemory:
     shared memory block.
 
     Every shared memory block is assigned a unique name.  This enables
-    one process to create a shared memory block with a particular name 
+    one process to create a shared memory block with a particular name
     so that a different process can attach to that same shared memory
     block using that same name.
 
@@ -500,139 +497,3 @@ def index(self, value):
                 return position
         else:
             raise ValueError(f"{value!r} not in this container")
-
-
-class _SharedMemoryTracker:
-    "Manages one or more shared memory segments."
-
-    def __init__(self, name, segment_names=[]):
-        self.shared_memory_context_name = name
-        self.segment_names = segment_names
-
-    def register_segment(self, segment_name):
-        "Adds the supplied shared memory block name to tracker."
-        util.debug(f"Registering segment {segment_name!r} in pid {os.getpid()}")
-        self.segment_names.append(segment_name)
-
-    def destroy_segment(self, segment_name):
-        """Calls unlink() on the shared memory block with the supplied name
-        and removes it from the list of blocks being tracked."""
-        util.debug(f"Destroying segment {segment_name!r} in pid {os.getpid()}")
-        self.segment_names.remove(segment_name)
-        segment = SharedMemory(segment_name)
-        segment.close()
-        segment.unlink()
-
-    def unlink(self):
-        "Calls destroy_segment() on all currently tracked shared memory blocks."
-        for segment_name in self.segment_names[:]:
-            self.destroy_segment(segment_name)
-
-    def __del__(self):
-        util.debug(f"Called {self.__class__.__name__}.__del__ in {os.getpid()}")
-        self.unlink()
-
-    def __getstate__(self):
-        return (self.shared_memory_context_name, self.segment_names)
-
-    def __setstate__(self, state):
-        self.__init__(*state)
-
-
-class SharedMemoryServer(Server):
-
-    public = Server.public + \
-             ['track_segment', 'release_segment', 'list_segments']
-
-    def __init__(self, *args, **kwargs):
-        Server.__init__(self, *args, **kwargs)
-        self.shared_memory_context = \
-            _SharedMemoryTracker(f"shmm_{self.address}_{os.getpid()}")
-        util.debug(f"SharedMemoryServer started by pid {os.getpid()}")
-
-    def create(self, c, typeid, *args, **kwargs):
-        """Create a new distributed-shared object (not backed by a shared
-        memory block) and return its id to be used in a Proxy Object."""
-        # Unless set up as a shared proxy, don't make shared_memory_context
-        # a standard part of kwargs.  This makes things easier for supplying
-        # simple functions.
-        if hasattr(self.registry[typeid][-1], "_shared_memory_proxy"):
-            kwargs['shared_memory_context'] = self.shared_memory_context
-        return Server.create(self, c, typeid, *args, **kwargs)
-
-    def shutdown(self, c):
-        "Call unlink() on all tracked shared memory then terminate the Server."
-        self.shared_memory_context.unlink()
-        return Server.shutdown(self, c)
-
-    def track_segment(self, c, segment_name):
-        "Adds the supplied shared memory block name to Server's tracker."
-        self.shared_memory_context.register_segment(segment_name)
-
-    def release_segment(self, c, segment_name):
-        """Calls unlink() on the shared memory block with the supplied name
-        and removes it from the tracker instance inside the Server."""
-        self.shared_memory_context.destroy_segment(segment_name)
-
-    def list_segments(self, c):
-        """Returns a list of names of shared memory blocks that the Server
-        is currently tracking."""
-        return self.shared_memory_context.segment_names
-
-
-class SharedMemoryManager(BaseManager):
-    """Like SyncManager but uses SharedMemoryServer instead of Server.
-
-    It provides methods for creating and returning SharedMemory instances
-    and for creating a list-like object (ShareableList) backed by shared
-    memory.  It also provides methods that create and return Proxy Objects
-    that support synchronization across processes (i.e. multi-process-safe
-    locks and semaphores).
-    """
-
-    _Server = SharedMemoryServer
-
-    def __init__(self, *args, **kwargs):
-        BaseManager.__init__(self, *args, **kwargs)
-        util.debug(f"{self.__class__.__name__} created by pid {os.getpid()}")
-
-    def __del__(self):
-        util.debug(f"{self.__class__.__name__} told die by pid {os.getpid()}")
-        pass
-
-    def get_server(self):
-        'Better than monkeypatching for now; merge into Server ultimately'
-        if self._state.value != State.INITIAL:
-            if self._state.value == State.STARTED:
-                raise ProcessError("Already started SharedMemoryServer")
-            elif self._state.value == State.SHUTDOWN:
-                raise ProcessError("SharedMemoryManager has shut down")
-            else:
-                raise ProcessError(
-                    "Unknown state {!r}".format(self._state.value))
-        return self._Server(self._registry, self._address,
-                            self._authkey, self._serializer)
-
-    def SharedMemory(self, size):
-        """Returns a new SharedMemory instance with the specified size in
-        bytes, to be tracked by the manager."""
-        with self._Client(self._address, authkey=self._authkey) as conn:
-            sms = SharedMemory(None, create=True, size=size)
-            try:
-                dispatch(conn, None, 'track_segment', (sms.name,))
-            except BaseException as e:
-                sms.unlink()
-                raise e
-        return sms
-
-    def ShareableList(self, sequence):
-        """Returns a new ShareableList instance populated with the values
-        from the input sequence, to be tracked by the manager."""
-        with self._Client(self._address, authkey=self._authkey) as conn:
-            sl = ShareableList(sequence)
-            try:
-                dispatch(conn, None, 'track_segment', (sl.shm.name,))
-            except BaseException as e:
-                sl.shm.unlink()
-                raise e
-        return sl
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 2aef69d4f54138..7f31ddd4623050 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3652,7 +3652,7 @@ def test_shared_memory_basics(self):
 
         # Attach to existing shared memory segment but specify a new size.
         same_sms = shared_memory.SharedMemory('test01_tsmb', size=20*sms.size)
-        self.assertEqual(same_sms.size, sms.size)  # Size was ignored.
+        self.assertLess(same_sms.size, 20*sms.size)  # Size was ignored.
         same_sms.close()
 
         if shared_memory._USE_POSIX:
@@ -3737,7 +3737,7 @@ def test_shared_memory_across_processes(self):
         sms.close()
 
     def test_shared_memory_SharedMemoryManager_basics(self):
-        smm1 = shared_memory.SharedMemoryManager()
+        smm1 = multiprocessing.managers.SharedMemoryManager()
         with self.assertRaises(ValueError):
             smm1.SharedMemory(size=9)  # Fails if SharedMemoryServer not started
         smm1.start()
@@ -3756,7 +3756,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
                 # No longer there to be attached to again.
                 absent_shm = shared_memory.SharedMemory(name=held_name)
 
-        with shared_memory.SharedMemoryManager() as smm2:
+        with multiprocessing.managers.SharedMemoryManager() as smm2:
             sl = smm2.ShareableList("howdy")
             shm = smm2.SharedMemory(size=128)
             held_name = sl.shm.name

From caf0a5d9cdfd569c438ce85ece952eacdf4100b8 Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Wed, 20 Feb 2019 23:27:48 -0600
Subject: [PATCH 40/41] Tweak to exception in OpenFileMapping suggested by
 eryksun.

---
 Modules/_winapi.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index 1996bd59c03365..e7b221d888ef8d 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -1391,7 +1391,7 @@ _winapi_OpenFileMapping_impl(PyObject *module, DWORD desired_access,
     Py_END_ALLOW_THREADS
 
     if (handle == NULL) {
-        PyErr_SetFromWindowsErr(0);
+        PyErr_SetFromWindowsErrWithUnicodeFilename(0, name);
         handle = INVALID_HANDLE_VALUE;
     }
 

From 12c097d88e7d1ceb159a1b726f229654e6218eed Mon Sep 17 00:00:00 2001
From: Davin Potts <python@discontinuity.net>
Date: Fri, 22 Feb 2019 13:10:50 -0600
Subject: [PATCH 41/41] Mark a few dangling bits as private as suggested by
 Giampaolo.

---
 Lib/multiprocessing/shared_memory.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 185bb0c8b82936..e4fe822cc641cf 100644
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -23,7 +23,7 @@
     _USE_POSIX = True
 
 
-O_CREX = os.O_CREAT | os.O_EXCL
+_O_CREX = os.O_CREAT | os.O_EXCL
 
 # FreeBSD (and perhaps other BSDs) limit names to 14 characters.
 _SHM_SAFE_NAME_LENGTH = 14
@@ -73,7 +73,7 @@ def __init__(self, name=None, create=False, size=0):
         if not size >= 0:
             raise ValueError("'size' must be a positive integer")
         if create:
-            self._flags = O_CREX | os.O_RDWR
+            self._flags = _O_CREX | os.O_RDWR
         if name is None and not self._flags & os.O_EXCL:
             raise ValueError("'name' can only be None if create=True")
 
@@ -228,7 +228,7 @@ def unlink(self):
             _posixshmem.shm_unlink(self.name)
 
 
-encoding = "utf8"
+_encoding = "utf8"
 
 class ShareableList:
     """Pattern for a mutable list-like object shareable via a shared
@@ -251,7 +251,7 @@ class ShareableList:
     _alignment = 8
     _back_transforms_mapping = {
         0: lambda value: value,                   # int, float, bool
-        1: lambda value: value.rstrip(b'\x00').decode(encoding),  # str
+        1: lambda value: value.rstrip(b'\x00').decode(_encoding),  # str
         2: lambda value: value.rstrip(b'\x00'),   # bytes
         3: lambda _value: None,                   # None
     }
@@ -305,7 +305,7 @@ def __init__(self, sequence=None, *, name=None):
             self.shm = SharedMemory(name, create=True, size=requested_size)
 
         if sequence is not None:
-            _enc = encoding
+            _enc = _encoding
             struct.pack_into(
                 "q" + self._format_size_metainfo,
                 self.shm.buf,
@@ -352,7 +352,7 @@ def _get_packing_format(self, position):
             self._offset_packing_formats + position * 8
         )[0]
         fmt = v.rstrip(b'\x00')
-        fmt_as_str = fmt.decode(encoding)
+        fmt_as_str = fmt.decode(_encoding)
 
         return fmt_as_str
 
@@ -384,7 +384,7 @@ def _set_packing_format_and_transform(self, position, fmt_as_str, value):
             "8s",
             self.shm.buf,
             self._offset_packing_formats + position * 8,
-            fmt_as_str.encode(encoding)
+            fmt_as_str.encode(_encoding)
         )
 
         transform_code = self._extract_recreation_code(value)
@@ -437,7 +437,7 @@ def __setitem__(self, position, value):
             new_format,
             value
         )
-        value = value.encode(encoding) if isinstance(value, str) else value
+        value = value.encode(_encoding) if isinstance(value, str) else value
         struct.pack_into(new_format, self.shm.buf, offset, value)
 
     def __reduce__(self):