From 8ae6fb875951487597caa0dcf04b23d61722cc4d Mon Sep 17 00:00:00 2001 From: biantianling Date: Thu, 8 Sep 2022 00:32:11 +0800 Subject: [PATCH 1/8] single progress bar works --- pandarallel/core.py | 9 +++++++-- pandarallel/progress_bars.py | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pandarallel/core.py b/pandarallel/core.py index 8eaaf12..f5e6786 100644 --- a/pandarallel/core.py +++ b/pandarallel/core.py @@ -6,6 +6,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Type, cast +from functools import partial import dill import pandas as pd @@ -205,6 +206,7 @@ def parallelize_with_memory_file_system( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, + single_bar ): def closure( data: Any, @@ -239,7 +241,7 @@ def closure( show_progress_bars = progress_bars_type != ProgressBarsType.No - progress_bars = get_progress_bars(progresses_length, show_progress_bars) + progress_bars = get_progress_bars(progresses_length, show_progress_bars, single_bar) progresses = [0] * nb_workers workers_status = [WorkerStatus.Running] * nb_workers @@ -344,6 +346,7 @@ def parallelize_with_pipe( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, + single_bar ): def closure( data: Any, @@ -380,7 +383,7 @@ def closure( show_progress_bars = progress_bars_type != ProgressBarsType.No - progress_bars = get_progress_bars(progresses_length, show_progress_bars) + progress_bars = get_progress_bars(progresses_length, show_progress_bars, single_bar) progresses = [0] * nb_workers workers_status = [WorkerStatus.Running] * nb_workers @@ -444,6 +447,7 @@ def initialize( shm_size_mb=None, nb_workers=NB_PHYSICAL_CORES, progress_bar=False, + single_progress_bar=False, verbose=2, use_memory_fs: Optional[bool] = None, ) -> None: @@ -459,6 +463,7 @@ def initialize( if use_memory_fs else parallelize_with_pipe ) + parallelize = partial(parallelize, single_bar=single_progress_bar) if use_memory_fs and not is_memory_fs_available: raise SystemError("Memory file system is not available") diff --git a/pandarallel/progress_bars.py b/pandarallel/progress_bars.py index dfe9190..2ee47a0 100644 --- a/pandarallel/progress_bars.py +++ b/pandarallel/progress_bars.py @@ -54,8 +54,11 @@ def is_notebook_lab() -> bool: class ProgressBarsConsole(ProgressBars): - def __init__(self, maxs: List[int], show: bool) -> None: + def __init__(self, maxs: List[int], show: bool, single_bar=True) -> None: self.__show = show + self.__single_bar = single_bar + if self.__single_bar: + maxs = [sum(maxs)] self.__bars = [[0, max] for max in maxs] self.__width = self.__get_width() @@ -107,6 +110,8 @@ def update(self, values: List[int]) -> None: if not self.__show: return + if self.__single_bar: + values = [sum(values)] for index, value in enumerate(values): self.__bars[index][0] = value @@ -118,7 +123,7 @@ def update(self, values: List[int]) -> None: class ProgressBarsNotebookLab(ProgressBars): - def __init__(self, maxs: List[int], show: bool) -> None: + def __init__(self, maxs: List[int], show: bool, single_bar=True) -> None: """Initialization. Positional argument: maxs - List containing the max value of each progress bar @@ -131,6 +136,9 @@ def __init__(self, maxs: List[int], show: bool) -> None: from IPython.display import display from ipywidgets import HBox, IntProgress, Label, VBox + self.__single_bar = single_bar + if self.__single_bar: + maxs = [sum(maxs)] self.__bars = [ HBox( [ @@ -150,7 +158,8 @@ def update(self, values: List[int]) -> None: """ if not self.__show: return - + if self.__single_bar: + values = [sum(values)] for index, value in enumerate(values): bar, label = self.__bars[index].children @@ -172,12 +181,13 @@ def set_error(self, index: int) -> None: def get_progress_bars( - maxs: List[int], show + maxs: List[int], show, single_bar ) -> Union[ProgressBarsNotebookLab, ProgressBarsConsole]: + print(f"Single progress bar: {single_bar}") return ( - ProgressBarsNotebookLab(maxs, show) + ProgressBarsNotebookLab(maxs, show, single_bar) if is_notebook_lab() - else ProgressBarsConsole(maxs, show) + else ProgressBarsConsole(maxs, show, single_bar) ) From 5888db10f6f6f0d0aaced8616d77f41d58063147 Mon Sep 17 00:00:00 2001 From: biantianling Date: Thu, 8 Sep 2022 00:48:03 +0800 Subject: [PATCH 2/8] support error state --- pandarallel/progress_bars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandarallel/progress_bars.py b/pandarallel/progress_bars.py index 2ee47a0..3044e97 100644 --- a/pandarallel/progress_bars.py +++ b/pandarallel/progress_bars.py @@ -175,7 +175,7 @@ def set_error(self, index: int) -> None: """Set a bar on error""" if not self.__show: return - + if self.__single_bar: index = 0 bar, _ = self.__bars[index].children bar.bar_style = "danger" From 1a8ff07e9240ab921ac1a9e7fb86cc8cdc9d4306 Mon Sep 17 00:00:00 2001 From: biantianling Date: Thu, 8 Sep 2022 00:54:35 +0800 Subject: [PATCH 3/8] remove debug prints --- pandarallel/progress_bars.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandarallel/progress_bars.py b/pandarallel/progress_bars.py index 3044e97..c7ff979 100644 --- a/pandarallel/progress_bars.py +++ b/pandarallel/progress_bars.py @@ -183,7 +183,6 @@ def set_error(self, index: int) -> None: def get_progress_bars( maxs: List[int], show, single_bar ) -> Union[ProgressBarsNotebookLab, ProgressBarsConsole]: - print(f"Single progress bar: {single_bar}") return ( ProgressBarsNotebookLab(maxs, show, single_bar) if is_notebook_lab() From 51f2568f5227de6a03ec8eaab3a48c54a14d6484 Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 23 May 2023 10:30:10 +0200 Subject: [PATCH 4/8] Update core.py added type to single_bar argument --- pandarallel/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandarallel/core.py b/pandarallel/core.py index f5e6786..b1ff0b2 100644 --- a/pandarallel/core.py +++ b/pandarallel/core.py @@ -206,7 +206,7 @@ def parallelize_with_memory_file_system( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, - single_bar + single_bar: bool ): def closure( data: Any, From 30481487d6e4f1942a54d13dad2be3990533abec Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 23 May 2023 12:30:16 +0200 Subject: [PATCH 5/8] Update core.py added type --- pandarallel/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandarallel/core.py b/pandarallel/core.py index b1ff0b2..cc90a62 100644 --- a/pandarallel/core.py +++ b/pandarallel/core.py @@ -346,7 +346,7 @@ def parallelize_with_pipe( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, - single_bar + single_bar: bool ): def closure( data: Any, From e48349e087678b80ed7acb08a2b7dcf8ef01f43d Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 23 May 2023 12:47:32 +0200 Subject: [PATCH 6/8] Update test_pandarallel.py added fixture for single_bar --- tests/test_pandarallel.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py index a9cce19..bd0f048 100644 --- a/tests/test_pandarallel.py +++ b/tests/test_pandarallel.py @@ -21,6 +21,11 @@ def use_memory_fs(request): return request.param +@pytest.fixture(params=(False, True)) +def single_bar(request): + return request.param + + @pytest.fixture(params=("named", "anonymous")) def func_dataframe_apply_axis_0(request): def func(x): @@ -152,9 +157,9 @@ def func(x): @pytest.fixture -def pandarallel_init(progress_bar, use_memory_fs): +def pandarallel_init(progress_bar, single_bar, use_memory_fs): pandarallel.initialize( - progress_bar=progress_bar, use_memory_fs=use_memory_fs, nb_workers=2 + progress_bar=progress_bar, single_bar=single_bar, use_memory_fs=use_memory_fs, nb_workers=2 ) From d03922dc54e1f44ffe99b16f87c44b8616191ab5 Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 23 May 2023 12:52:50 +0200 Subject: [PATCH 7/8] Update test_pandarallel.py fixed name --- tests/test_pandarallel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py index f779058..de98e6f 100644 --- a/tests/test_pandarallel.py +++ b/tests/test_pandarallel.py @@ -23,7 +23,7 @@ def use_memory_fs(request): @pytest.fixture(params=(False, True)) -def single_bar(request): +def single_progress_bar(request): return request.param @@ -163,9 +163,9 @@ def func(x): @pytest.fixture -def pandarallel_init(progress_bar, single_bar, use_memory_fs): +def pandarallel_init(progress_bar, single_progress_bar, use_memory_fs): pandarallel.initialize( - progress_bar=progress_bar, single_bar=single_bar, use_memory_fs=use_memory_fs, nb_workers=2 + progress_bar=progress_bar, single_progress_bar=single_progress_bar, use_memory_fs=use_memory_fs, nb_workers=2 ) From 545ac80a3bc7dfd4547c21afbd8ced80fb672359 Mon Sep 17 00:00:00 2001 From: Luca Puglia Date: Thu, 22 Jun 2023 08:38:18 +0100 Subject: [PATCH 8/8] using single name for single_bar --- pandarallel/core.py | 4 ++-- tests/test_pandarallel.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandarallel/core.py b/pandarallel/core.py index 146e3fe..f5a2c1f 100644 --- a/pandarallel/core.py +++ b/pandarallel/core.py @@ -458,7 +458,7 @@ def initialize( shm_size_mb=None, nb_workers=NB_PHYSICAL_CORES, progress_bar=False, - single_progress_bar=False, + single_bar=False, verbose=2, use_memory_fs: Optional[bool] = None, ) -> None: @@ -474,7 +474,7 @@ def initialize( if use_memory_fs else parallelize_with_pipe ) - parallelize = partial(parallelize, single_bar=single_progress_bar) + parallelize = partial(parallelize, single_bar=single_bar) if use_memory_fs and not is_memory_fs_available: raise SystemError("Memory file system is not available") diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py index de98e6f..f779058 100644 --- a/tests/test_pandarallel.py +++ b/tests/test_pandarallel.py @@ -23,7 +23,7 @@ def use_memory_fs(request): @pytest.fixture(params=(False, True)) -def single_progress_bar(request): +def single_bar(request): return request.param @@ -163,9 +163,9 @@ def func(x): @pytest.fixture -def pandarallel_init(progress_bar, single_progress_bar, use_memory_fs): +def pandarallel_init(progress_bar, single_bar, use_memory_fs): pandarallel.initialize( - progress_bar=progress_bar, single_progress_bar=single_progress_bar, use_memory_fs=use_memory_fs, nb_workers=2 + progress_bar=progress_bar, single_bar=single_bar, use_memory_fs=use_memory_fs, nb_workers=2 )