From 5d2997e7329ffa53b8d6ea52d887562912ee82b2 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sun, 20 Oct 2024 22:10:37 +0200 Subject: [PATCH 1/8] Initial commit --- Lib/multiprocessing/synchronize.py | 78 ++++++++++++++++++++++++++++-- Lib/test/_test_multiprocessing.py | 24 +++++++-- 2 files changed, 93 insertions(+), 9 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 3ccbfe311c71f3..6831e87ea8e576 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -123,21 +123,89 @@ def _make_name(): return '%s-%s' % (process.current_process()._config['semprefix'], next(SemLock._rand)) +if sys.platform == 'darwin': + # + # Specific MacOSX Semaphore + # + + class _MacOSXSemaphore(SemLock): + """Dedicated class used only to workaround the missing + function 'sem_getvalue', when interpreter runs on MacOSX. + Add a shared counter for each [Bounded]Semaphore in order + to handle internal counter when acquire and release operations + are called. + """ + + def __init__(self, kind, value, maxvalue, *, ctx): + util.debug(f"_MacOSXSemaphore:: creation of a {self.__class__.__name__}"\ + f"with '{value = }'") + SemLock.__init__(self, kind, value, maxvalue, ctx=ctx) + self._count = ctx.Value('L', value) # May be more than 'L' ? + + def _acquire(self, *args, **kwargs) -> bool: + if self._semlock.acquire(*args, **kwargs): + with self._count: + util.debug(f"_MacOSXSemaphore: acquire {repr(self)}") + self._count.value -= 1 + return True + return False + + def _release(self): + with self._count: + self._count.value += 1 + self._semlock.release() + util.debug(f"_MacOSXSemaphore: release {repr(self)}") + + def _release_bounded(self): + if self._count.value + 1 > self._semlock.maxvalue: + raise ValueError(f"Cannot exceed initial value of"\ + f" {self._semlock.maxvalue!a}") + self._release() + + def _get_value(self) -> int: + return self._count.value + + def _make_methods(self): + super()._make_methods() + util.debug("_MacOSXSemaphore: _make_methods call") + self.acquire = self._acquire + if isinstance(self, BoundedSemaphore): + self.release = self._release_bounded + elif isinstance(self, Semaphore): + self.release = self._release + else: + raise RuntimeError("Class dedicated only to Semaphore or BoundedSemaphore OSX") + self.get_value = self._get_value + + def __setstate__(self, state): + self._count, state = state[-1], state[:-1] + super().__setstate__(state) + + def __getstate__(self) -> tuple: + return super().__getstate__() + (self._count,) + + + _SemClass = _MacOSXSemaphore +else: + _SemClass = SemLock + # # Semaphore # -class Semaphore(SemLock): +class Semaphore(_SemClass): def __init__(self, value=1, *, ctx): - SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) + _SemClass.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) def get_value(self): + """redefined when MacOSX. + """ return self._semlock._get_value() def __repr__(self): try: - value = self._semlock._get_value() + value = self.get_value() except Exception: value = 'unknown' return '<%s(value=%s)>' % (self.__class__.__name__, value) @@ -149,11 +217,11 @@ def __repr__(self): class BoundedSemaphore(Semaphore): def __init__(self, value=1, *, ctx): - SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx) + _SemClass.__init__(self, SEMAPHORE, value, value, ctx=ctx) def __repr__(self): try: - value = self._semlock._get_value() + value = self.get_value() except Exception: value = 'unknown' return '<%s(value=%s, maxvalue=%s)>' % \ diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 065fc27b770438..a95ff64163222b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1410,10 +1410,26 @@ def test_semaphore(self): def test_bounded_semaphore(self): sem = self.BoundedSemaphore(2) self._test_semaphore(sem) - # Currently fails on OS/X - #if HAVE_GETVALUE: - # self.assertRaises(ValueError, sem.release) - # self.assertReturnsIfImplemented(2, get_value, sem) + self.assertRaises(ValueError, sem.release) + self.assertReturnsIfImplemented(2, get_value, sem) + + @unittest.skipIf(sys.platform != 'darwin', 'Darwin only') + def test_detect_macosx_semaphore(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sem = self.Semaphore(2) + mro = sem.__class__.mro() + self.assertTrue(any('_MacOSXSemaphore' in cls.__name__ for cls in mro)) + + @unittest.skipIf(sys.platform != 'darwin', 'Darwin only') + def test_detect_macosx_boundedsemaphore(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sem = self.BoundedSemaphore(2) + mro = sem.__class__.mro() + self.assertTrue(any('_MacOSXSemaphore' in cls.__name__ for cls in mro)) def test_timeout(self): if self.TYPE != 'processes': From 6e2c56cef968cd0e83025bac423caa2bb5293856 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:18:30 +0000 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst diff --git a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst new file mode 100644 index 00000000000000..764a7e608bc7f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst @@ -0,0 +1,2 @@ +Fix the not implemented ``get_value`` for :class:`multiprocessing.Semaphore` on MacOSX +by adding a dedicated class in the ``synchronize.py`` file. From 5472ce2a6974293cd442c7e3e1e25586c9bd0c0e Mon Sep 17 00:00:00 2001 From: "Yves.Duprat" Date: Mon, 26 Jan 2026 17:13:54 +0100 Subject: [PATCH 3/8] Refactor code --- Lib/multiprocessing/synchronize.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index aca46cd0a65206..6c5c15980b21cb 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -139,15 +139,15 @@ class _MacOSXSemaphore(SemLock): """ def __init__(self, kind, value, maxvalue, *, ctx): - util.debug(f"_MacOSXSemaphore:: creation of a {self.__class__.__name__}"\ - f"with '{value = }'") + if not isinstance(self, Semaphore): + raise TypeError("_MacOSXSemaphore can only be used " + "as base class of Semaphore class") + self._count = ctx.Value('h', value) SemLock.__init__(self, kind, value, maxvalue, ctx=ctx) - self._count = ctx.Value('L', value) # May be more than 'L' ? def _acquire(self, *args, **kwargs) -> bool: if self._semlock.acquire(*args, **kwargs): with self._count: - util.debug(f"_MacOSXSemaphore: acquire {repr(self)}") self._count.value -= 1 return True return False @@ -156,27 +156,24 @@ def _release(self): with self._count: self._count.value += 1 self._semlock.release() - util.debug(f"_MacOSXSemaphore: release {repr(self)}") def _release_bounded(self): - if self._count.value + 1 > self._semlock.maxvalue: - raise ValueError(f"Cannot exceed initial value of"\ - f" {self._semlock.maxvalue!a}") - self._release() + with self._count: + if self._count.value + 1 > self._semlock.maxvalue: + raise ValueError(f"Cannot exceed initial value of"\ + f" {self._semlock.maxvalue!a}") + self._release() def _get_value(self) -> int: return self._count.value def _make_methods(self): super()._make_methods() - util.debug("_MacOSXSemaphore: _make_methods call") self.acquire = self._acquire if isinstance(self, BoundedSemaphore): self.release = self._release_bounded elif isinstance(self, Semaphore): self.release = self._release - else: - raise RuntimeError("Class dedicated only to Semaphore or BoundedSemaphore OSX") self.get_value = self._get_value def __setstate__(self, state): From a486952358addf11c0f765ea511e3236ee172546 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sat, 21 Mar 2026 16:36:06 +0100 Subject: [PATCH 4/8] Semaphore macosx multiprocessing fix (#6) * Refactor code * fix error on get_value, and nits --- Lib/multiprocessing/synchronize.py | 45 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 6c5c15980b21cb..71cf3c277dffa1 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -143,38 +143,33 @@ def __init__(self, kind, value, maxvalue, *, ctx): raise TypeError("_MacOSXSemaphore can only be used " "as base class of Semaphore class") self._count = ctx.Value('h', value) - SemLock.__init__(self, kind, value, maxvalue, ctx=ctx) + super().__init__(kind, value, maxvalue, ctx=ctx) - def _acquire(self, *args, **kwargs) -> bool: - if self._semlock.acquire(*args, **kwargs): + def acquire(self, blocking=True, timeout=None): + if self._semlock.acquire(blocking, timeout): with self._count: self._count.value -= 1 return True return False - def _release(self): + def release(self): + if isinstance(self, BoundedSemaphore): + with self._count: + if self._count.value + 1 > self._semlock.maxvalue: + raise ValueError(f"Cannot exceed initial value of"\ + f" {self._semlock.maxvalue!a}") with self._count: self._count.value += 1 self._semlock.release() - def _release_bounded(self): - with self._count: - if self._count.value + 1 > self._semlock.maxvalue: - raise ValueError(f"Cannot exceed initial value of"\ - f" {self._semlock.maxvalue!a}") - self._release() - - def _get_value(self) -> int: + def get_value(self): return self._count.value def _make_methods(self): - super()._make_methods() - self.acquire = self._acquire - if isinstance(self, BoundedSemaphore): - self.release = self._release_bounded - elif isinstance(self, Semaphore): - self.release = self._release - self.get_value = self._get_value + # Do not call the `Semlock._make_methods` method, + # as this breaks the reference to the local + # `acquire` and `release` methods. + pass def __setstate__(self, state): self._count, state = state[-1], state[:-1] @@ -186,7 +181,14 @@ def __getstate__(self) -> tuple: _SemClass = _MacOSXSemaphore else: - _SemClass = SemLock + class _NotMacOSXSemaphore(SemLock): + def __init__(self, kind, value, maxvalue, *, ctx): + super().__init__(kind, value, maxvalue, ctx=ctx) + + def get_value(self) -> int: + return self._semlock._get_value() + + _SemClass = _NotMacOSXSemaphore # # Semaphore @@ -197,9 +199,6 @@ class Semaphore(_SemClass): def __init__(self, value=1, *, ctx): _SemClass.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) - def get_value(self): - return self._semlock._get_value() - def __repr__(self): try: value = self.get_value() From e69bfd8c748ae7fec062b1b11b805a96e351bd36 Mon Sep 17 00:00:00 2001 From: "Yves.Duprat" Date: Thu, 7 May 2026 15:53:13 +0200 Subject: [PATCH 5/8] last modifications as: + Add `pending_acquires` counter + Update comments + Fix nits --- Lib/multiprocessing/synchronize.py | 38 +++++++++++++++++++----------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 71cf3c277dffa1..d445138fa9234f 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -133,50 +133,60 @@ def _make_name(): class _MacOSXSemaphore(SemLock): """Dedicated class used only to workaround the missing function 'sem_getvalue', when interpreter runs on MacOSX. - Add a shared counter for each [Bounded]Semaphore in order - to handle internal counter when acquire and release operations - are called. + Add two shared counters for each [Bounded]Semaphore in order + to calculate the internal value of the semaphore, + when acquire and release operations are called. """ def __init__(self, kind, value, maxvalue, *, ctx): if not isinstance(self, Semaphore): raise TypeError("_MacOSXSemaphore can only be used " "as base class of Semaphore class") - self._count = ctx.Value('h', value) + self._rlock = ctx.RLock() + self._count = ctx.Value('h', value, lock=self._rlock) + self._pending_acquires = ctx.Value('h', 0, lock=self._rlock) super().__init__(kind, value, maxvalue, ctx=ctx) def acquire(self, blocking=True, timeout=None): + with self._rlock: + self._pending_acquires.value += 1 if self._semlock.acquire(blocking, timeout): - with self._count: + with self._rlock: + self._pending_acquires.value -= 1 self._count.value -= 1 return True + with self._rlock: + self._pending_acquires.value -= 1 return False def release(self): if isinstance(self, BoundedSemaphore): - with self._count: - if self._count.value + 1 > self._semlock.maxvalue: - raise ValueError(f"Cannot exceed initial value of"\ - f" {self._semlock.maxvalue!a}") - with self._count: + with self._rlock: + if self.get_value() + 1 > self._semlock.maxvalue: + raise ValueError(f"semaphore released too many times") + with self._rlock: self._count.value += 1 self._semlock.release() def get_value(self): - return self._count.value + with self._rlock: + val = self._count.value - self._pending_acquires.value + return val if val > 0 else 0 def _make_methods(self): # Do not call the `Semlock._make_methods` method, - # as this breaks the reference to the local + # because that breaks the reference to the local # `acquire` and `release` methods. pass def __setstate__(self, state): - self._count, state = state[-1], state[:-1] + self._count, self._pending_acquires, self._rlock, state \ + = state[-3], state[-2], state[-1], state[:-3] super().__setstate__(state) def __getstate__(self) -> tuple: - return super().__getstate__() + (self._count,) + return super().__getstate__() \ + + (self._count, self._pending_acquires, self._rlock) _SemClass = _MacOSXSemaphore From ae275bf606d92634d57f26d4727cfeb08f454b4a Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 8 May 2026 16:13:02 +0200 Subject: [PATCH 6/8] Update news file Clarified the fix for the unimplemented get_value method and specified the class hierarchy for better understanding. --- .../Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst index 764a7e608bc7f1..7acd7f7f426111 100644 --- a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst +++ b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst @@ -1,2 +1 @@ -Fix the not implemented ``get_value`` for :class:`multiprocessing.Semaphore` on MacOSX -by adding a dedicated class in the ``synchronize.py`` file. +Fix the unimplemented ``get_value`` method of the ``_multiprocessing.SemLock`` class on MacOSX by adding a specific inherited class between ``multiprocessing.SemLock`` and :class:`multiprocessing.Semahore`. From 7080b1713d1b431c4c7afa5dde72a4ca7c687617 Mon Sep 17 00:00:00 2001 From: "Yves.Duprat" Date: Sun, 10 May 2026 17:35:09 +0200 Subject: [PATCH 7/8] =?UTF-8?q?Update=20following=20the=20execution=20of?= =?UTF-8?q?=20=E2=80=9Cregen-all=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst | 2 +- Programs/test_frozenmain.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst index 7acd7f7f426111..c6a98dda9c8e34 100644 --- a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst +++ b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst @@ -1 +1 @@ -Fix the unimplemented ``get_value`` method of the ``_multiprocessing.SemLock`` class on MacOSX by adding a specific inherited class between ``multiprocessing.SemLock`` and :class:`multiprocessing.Semahore`. +Fix the unimplemented ``get_value`` method of the ``_multiprocessing.SemLock`` class on MacOSX by adding a specific inherited class between ``multiprocessing.SemLock`` and :class:`multiprocessing.Semahore`. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 1bcf98a7600851..0f07d03b05a329 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -13,10 +13,10 @@ unsigned char M_test_frozenmain[] = { 92,2,31,0,81,4,92,6,12,0,81,5,92,5,92,6, 42,26,0,0,0,0,0,0,0,0,0,0,12,0,48,4, 50,1,0,0,0,0,0,0,29,0,74,26,0,0,9,0, - 28,0,80,7,33,0,41,7,233,0,0,0,0,122,18,70, + 28,0,80,7,33,0,41,7,233,0,0,0,0,218,18,70, 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, - 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, - 102,105,103,122,7,99,111,110,102,105,103,32,122,2,58,32, + 100,218,8,115,121,115,46,97,114,103,118,218,6,99,111,110, + 102,105,103,218,7,99,111,110,102,105,103,32,218,2,58,32, 41,5,218,12,112,114,111,103,114,97,109,95,110,97,109,101, 218,10,101,120,101,99,117,116,97,98,108,101,218,15,117,115, 101,95,101,110,118,105,114,111,110,109,101,110,116,218,17,99, @@ -25,15 +25,15 @@ unsigned char M_test_frozenmain[] = { 41,7,218,3,115,121,115,218,17,95,116,101,115,116,105,110, 116,101,114,110,97,108,99,97,112,105,218,5,112,114,105,110, 116,218,4,97,114,103,118,218,11,103,101,116,95,99,111,110, - 102,105,103,115,114,3,0,0,0,218,3,107,101,121,169,0, + 102,105,103,115,114,5,0,0,0,218,3,107,101,121,169,0, 243,0,0,0,0,218,18,116,101,115,116,95,102,114,111,122, 101,110,109,97,105,110,46,112,121,218,8,60,109,111,100,117, - 108,101,62,114,18,0,0,0,1,0,0,0,115,94,0,0, + 108,101,62,114,22,0,0,0,1,0,0,0,115,94,0,0, 0,241,3,1,1,1,243,8,0,1,11,219,0,24,225,0, 5,208,6,26,212,0,27,217,0,5,128,106,144,35,151,40, 145,40,212,0,27,216,9,26,215,9,38,210,9,38,211,9, 40,168,24,213,9,50,128,6,244,2,6,12,2,128,67,241, 14,0,5,10,136,71,144,67,144,53,152,2,152,54,160,35, 157,59,152,45,208,10,40,214,4,41,243,15,6,12,2,114, - 16,0,0,0, + 20,0,0,0, }; From 7fab36b6aa2ec6364eefd1bfcb67f9d47ac8274e Mon Sep 17 00:00:00 2001 From: Duprat Date: Sun, 10 May 2026 19:08:41 +0200 Subject: [PATCH 8/8] Fix bad class name --- .../next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst index c6a98dda9c8e34..ef02f1b55a9074 100644 --- a/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst +++ b/Misc/NEWS.d/next/Library/2026-01-26-16-18-19.gh-issue-125828.W6BLH8.rst @@ -1 +1 @@ -Fix the unimplemented ``get_value`` method of the ``_multiprocessing.SemLock`` class on MacOSX by adding a specific inherited class between ``multiprocessing.SemLock`` and :class:`multiprocessing.Semahore`. +Fix the unimplemented ``get_value`` method of the ``_multiprocessing.SemLock`` class on MacOSX by adding a specific inherited class between ``multiprocessing.SemLock`` and :class:`multiprocessing.Semaphore`.