-
-
Notifications
You must be signed in to change notification settings - Fork 25
Description
(pid=1402671) import pynvml # type: ignore[import]
(raylet) [2025-12-01 09:52:46,653 E 1391285 1391351] (raylet) file_system_monitor.cc:116: /tmp/ray/session_2025-12-01_09-52-20_833392_1378572 is over 95% full, available space: 30.1922 GB; capacity: 1756.89 GB. Object creation will fail if spilling is required.
!!! Exception during processing !!! The actor died because of an error raised in its creation task, ray::RayWorker:0:RayWorker.init() (pid=1402670, ip=172.17.0.5, actor_id=8dadc85f822c76b45f9717ec01000000, repr=<raylight.distributed_worker.ray_worker.RayWorker object at 0x7f78eb9f2ed0>)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-12-01_09-52-20_833392_1378572/runtime_resources/py_modules_files/_ray_pkg_7f83f26fc34f1941/raylight/distributed_worker/ray_worker.py", line 100, in init
initialize_model_parallel(
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/parallel_state.py", line 438, in initialize_model_parallel
_DP = init_model_parallel_group(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/parallel_state.py", line 302, in init_model_parallel_group
return GroupCoordinator(
^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/group_coordinator.py", line 141, in init
assert self.cpu_group is not None
AssertionError
Traceback (most recent call last):
File "/data/test_comfyui/ComfyUI/execution.py", line 510, in execute
output_data, output_ui, has_subgraph, has_pending_tasks = await get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, hidden_inputs=hidden_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/test_comfyui/ComfyUI/execution.py", line 324, in get_output_data
return_values = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, obj.FUNCTION, allow_interrupt=True, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, hidden_inputs=hidden_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/test_comfyui/ComfyUI/execution.py", line 298, in _async_map_node_over_list
await process_inputs(input_dict, i)
File "/data/test_comfyui/ComfyUI/execution.py", line 286, in process_inputs
result = f(**inputs)
^^^^^^^^^^^
File "/data/test_comfyui/ComfyUI/custom_nodes/raylight/src/raylight/nodes.py", line 130, in spawn_actor
ray_actors = ray_actor_fn()
^^^^^^^^^^^^^^
File "/data/test_comfyui/ComfyUI/custom_nodes/raylight/src/raylight/distributed_worker/ray_worker.py", line 496, in _init_ray_actor
ray.get(actor.ray_ready.remote())
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
values, debugger_breakpoint = worker.get_objects(
^^^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/ray/_private/worker.py", line 1033, in get_objects
raise value
ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::RayWorker:0:RayWorker.init() (pid=1402670, ip=172.17.0.5, actor_id=8dadc85f822c76b45f9717ec01000000, repr=<raylight.distributed_worker.ray_worker.RayWorker object at 0x7f78eb9f2ed0>)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-12-01_09-52-20_833392_1378572/runtime_resources/py_modules_files/_ray_pkg_7f83f26fc34f1941/raylight/distributed_worker/ray_worker.py", line 100, in init
initialize_model_parallel(
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/parallel_state.py", line 438, in initialize_model_parallel
_DP = init_model_parallel_group(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/parallel_state.py", line 302, in init_model_parallel_group
return GroupCoordinator(
^^^^^^^^^^^^^^^^^
File "/home/adminuser/miniconda3/envs/comfyui/lib/python3.12/site-packages/xfuser/core/distributed/group_coordinator.py", line 141, in init
assert self.cpu_group is not None
AssertionError
When testing want2v, there have been continuous errors. Does anyone know how to solve this problem? Is there anyone who has published the workflow for wan2.2 i2v? I'm looking for an open-source version.