From c1443801955f11e78860a2799d160abf7950b335 Mon Sep 17 00:00:00 2001 From: Pavel Polikashin Date: Sat, 22 Nov 2025 00:17:17 -0500 Subject: [PATCH 1/2] Fix semaphore leak warnings on Python 3.13+ When using Python 3.13+, loky creates semaphores that are registered with both loky's resource_tracker and Python's stdlib resource_tracker. However, loky only unregisters from its own tracker during cleanup, leaving the stdlib tracker to report 'leaked semaphore' warnings. This fix unregisters semaphores from Python's stdlib resource_tracker immediately after loky registers them, since loky takes full responsibility for cleanup via its own resource tracker. Fixes semaphore leak warnings like: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown: {'/loky-15609-fznn26z6'} Tested on: - Python 3.13.8 on macOS (Apple Silicon) - Fixes crash at training step 106 in joblib-based workflows --- loky/backend/synchronize.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/loky/backend/synchronize.py b/loky/backend/synchronize.py index 18db3e34..a1f7cd9c 100644 --- a/loky/backend/synchronize.py +++ b/loky/backend/synchronize.py @@ -21,6 +21,13 @@ from multiprocessing import process, util from multiprocessing.context import assert_spawning +# Import Python's stdlib resource_tracker to unregister semaphores from it +# This prevents "leaked semaphore" warnings on Python 3.13+ when loky manages cleanup +try: + from multiprocessing import resource_tracker as stdlib_resource_tracker +except ImportError: + stdlib_resource_tracker = None + from . import resource_tracker __all__ = [ @@ -96,6 +103,16 @@ def _after_fork(obj): # When the object is garbage collected or the # process shuts down we unlink the semaphore name resource_tracker.register(self._semlock.name, "semlock") + + # FIX: Unregister from Python's stdlib resource_tracker to prevent + # "leaked semaphore" warnings on Python 3.13+, since loky handles cleanup + if stdlib_resource_tracker is not None: + try: + stdlib_resource_tracker.unregister(self._semlock.name, "semlock") + except (KeyError, ValueError): + # Semaphore wasn't registered in stdlib tracker, that's fine + pass + util.Finalize( self, SemLock._cleanup, (self._semlock.name,), exitpriority=0 ) From 93d882b25e86fd8510b547e47b3820eb0b5bd328 Mon Sep 17 00:00:00 2001 From: Pavel Polikashin Date: Sat, 22 Nov 2025 00:59:18 -0500 Subject: [PATCH 2/2] fix: Unregister semaphores from stdlib tracker immediately after creation The previous fix attempted to unregister after loky's registration, but this was too late. The semaphore is registered with stdlib tracker when _SemLock() is created, so we must unregister immediately after creation in both code paths: - In the loop when name=None (after line 79-81) - When name is provided (after line 97) This prevents 'leaked semaphore' warnings on Python 3.13+ while loky handles cleanup properly through its own resource tracker. --- loky/backend/synchronize.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/loky/backend/synchronize.py b/loky/backend/synchronize.py index a1f7cd9c..21db21aa 100644 --- a/loky/backend/synchronize.py +++ b/loky/backend/synchronize.py @@ -79,6 +79,14 @@ def __init__(self, kind, value, maxvalue, name=None): self._semlock = _SemLock( kind, value, maxvalue, SemLock._make_name(), unlink_now ) + # FIX: Unregister from stdlib resource_tracker immediately after creation + # to prevent "leaked semaphore" warnings on Python 3.13+ + if stdlib_resource_tracker is not None: + try: + stdlib_resource_tracker.unregister(self._semlock.name, "semlock") + except (KeyError, ValueError): + # Semaphore wasn't registered in stdlib tracker, that's fine + pass except FileExistsError: # pragma: no cover pass else: @@ -87,6 +95,14 @@ def __init__(self, kind, value, maxvalue, name=None): raise FileExistsError("cannot find name for semaphore") else: self._semlock = _SemLock(kind, value, maxvalue, name, unlink_now) + # FIX: Unregister from stdlib resource_tracker immediately after creation + # to prevent "leaked semaphore" warnings on Python 3.13+ + if stdlib_resource_tracker is not None: + try: + stdlib_resource_tracker.unregister(self._semlock.name, "semlock") + except (KeyError, ValueError): + # Semaphore wasn't registered in stdlib tracker, that's fine + pass self.name = name util.debug( f"created semlock with handle {self._semlock.handle} and name " @@ -104,15 +120,6 @@ def _after_fork(obj): # process shuts down we unlink the semaphore name resource_tracker.register(self._semlock.name, "semlock") - # FIX: Unregister from Python's stdlib resource_tracker to prevent - # "leaked semaphore" warnings on Python 3.13+, since loky handles cleanup - if stdlib_resource_tracker is not None: - try: - stdlib_resource_tracker.unregister(self._semlock.name, "semlock") - except (KeyError, ValueError): - # Semaphore wasn't registered in stdlib tracker, that's fine - pass - util.Finalize( self, SemLock._cleanup, (self._semlock.name,), exitpriority=0 )