From 4f39bd3ee49f7b755f0f4b42735d42afa5c30c35 Mon Sep 17 00:00:00 2001 From: Ben Denham Date: Sat, 31 May 2025 16:01:08 +1200 Subject: [PATCH 1/2] Use mp_context so that correct start_method is used. --- labtech/runners/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labtech/runners/process.py b/labtech/runners/process.py index df4f0ce..1dba28e 100644 --- a/labtech/runners/process.py +++ b/labtech/runners/process.py @@ -151,7 +151,7 @@ def _start_processes(self): for future in futures_to_start: thunk = self._pending_future_to_thunk[future] del self._pending_future_to_thunk[future] - process = multiprocessing.Process( + process = self.mp_context.Process( target=_subprocess_target, kwargs=dict( future_id=future.id, From 20af9eea72ded154a56811562479d45230397ebe Mon Sep 17 00:00:00 2001 From: Ben Denham Date: Sat, 31 May 2025 16:11:34 +1200 Subject: [PATCH 2/2] Add graceful error message when task types are defined on __main__ in an interactive session --- docs/cookbook.md | 25 +++++++++++++++++-------- labtech/runners/process.py | 12 +++++++++++- labtech/utils.py | 6 ++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/cookbook.md b/docs/cookbook.md index 20688cd..c8e8816 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -926,20 +926,29 @@ if __name__ == '__main__': For details, see [Safe importing of main module](https://docs.python.org/3/library/multiprocessing.html#multiprocessing-safe-main-import). -### Why do I see the following error: `AttributeError: Can't get attribute 'YOUR_TASK_CLASS' on `? +
-You will see this error (as part of a very long stack trace) when -running a Lab with `runner_backend='spawn'` (the default on macOS and -Windows) from an interactive Python shell. +### Why do I see the following error: `RunnerError: Unable to submit YourTaskType tasks to SpawnProcessRunner because the task type is defined in the __main__ module from an interactive Python session`? -The solution to this error is to define all of your labtech `Task` -types in a separate `.py` Python module file which you can import into -your interactive shell session (e.g. `from my_module import MyTask`). +You may see this error when running a Lab with +`runner_backend='spawn'` (the default on macOS and Windows) from an +interactive Python shell (e.g. a Jupyter notebook session or a Python +script). + +The solution to this error is to define all of the classes you are +using from your labtech context and tasks (including task types) in a +separate `.py` Python module file which you can import into your +interactive shell session (e.g. `from my_module import MyClass`). The reason for this error is that "spawned" task subprocesses will not receive a copy the current state of your `__main__` module (which contains the variables you declare interactively in the Python shell, -including task definitions). This error does not occur with +including class definitions). This error does not occur with `runner_backend='fork'` (the default on Linux) because forked subprocesses *do* receive the current state of all modules (including `__main__`) from the parent process. + + +### Why do I see the following error: `AttributeError: Can't get attribute 'YOUR_CLASS' on `? + +[See the answer to the question directly above.](#spawn-interactive-main) diff --git a/labtech/runners/process.py b/labtech/runners/process.py index 1dba28e..2556a7f 100644 --- a/labtech/runners/process.py +++ b/labtech/runners/process.py @@ -22,7 +22,7 @@ from labtech.monitor import get_process_info from labtech.tasks import get_direct_dependencies from labtech.types import Runner, RunnerBackend -from labtech.utils import LoggerFileProxy, get_supported_start_methods, logger +from labtech.utils import LoggerFileProxy, get_supported_start_methods, is_interactive, logger from .base import run_or_load_task @@ -461,6 +461,16 @@ def _get_mp_context(self) -> SpawnContext: def _submit_task(self, executor: ProcessExecutor, task: Task, task_name: str, use_cache: bool, process_event_queue: Queue, log_queue: Queue) -> Future: + if is_interactive() and task.__class__.__module__ == '__main__': + raise RunnerError( + (f'Unable to submit {task.__class__.__qualname__} tasks to ' + 'SpawnProcessRunner because the task type is defined in the ' + '__main__ module from an interactive Python session. ' + 'Please define your task types in a separate `.py` Python ' + 'module file. For details, see: ' + 'https://ben-denham.github.io/labtech/cookbook/#spawn-interactive-main') + ) + filtered_context: LabContext = {} results_map: dict[Task, TaskResult] = {} if not use_cache: diff --git a/labtech/utils.py b/labtech/utils.py index 537fe10..7676d16 100644 --- a/labtech/utils.py +++ b/labtech/utils.py @@ -5,6 +5,7 @@ import logging import platform import re +import sys from multiprocessing import get_all_start_methods from typing import TYPE_CHECKING, Generic, TypeVar, cast @@ -123,6 +124,10 @@ def ensure_dict_key_str(value, *, exception_type: type[Exception]) -> str: return cast('str', value) +def is_interactive() -> bool: + return hasattr(sys, 'ps1') + + def is_ipython() -> bool: return hasattr(builtins, '__IPYTHON__') @@ -151,6 +156,7 @@ class tqdm_notebook(base_tqdm_notebook): 'OrderedSet', 'LoggerFileProxy', 'ensure_dict_key_str', + 'is_interactive', 'is_ipython', 'tqdm', 'tqdm_notebook',