diff --git a/docs/docs/user_guide.md b/docs/docs/user_guide.md index fbc61b2..29383c7 100644 --- a/docs/docs/user_guide.md +++ b/docs/docs/user_guide.md @@ -37,6 +37,9 @@ This method takes 5 optional parameters: not available. * If set to `False`, `pandarallel` will use multiprocessing data transfer (pipe) to transfer data between the main process and workers. +- `leave`: (bool, `True` by default): whether to keep the progress bar after running on a notebook or not. + * If set to `True`, will leave the notebook progress bar after running. + * If set to `False`, will delete the notebook progress bar after running. Using memory file system reduces data transfer time between the main process and workers, especially for big data. diff --git a/pandarallel/core.py b/pandarallel/core.py index f96359c..5977fb2 100644 --- a/pandarallel/core.py +++ b/pandarallel/core.py @@ -205,6 +205,7 @@ def parallelize_with_memory_file_system( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, + leave: bool = True, ): def closure( data: Any, @@ -239,7 +240,7 @@ def closure( show_progress_bars = progress_bars_type != ProgressBarsType.No - progress_bars = get_progress_bars(progresses_length, show_progress_bars) + progress_bars = get_progress_bars(progresses_length, show_progress_bars, leave) progresses = [0] * nb_workers workers_status = [WorkerStatus.Running] * nb_workers @@ -344,6 +345,7 @@ def parallelize_with_pipe( nb_requested_workers: int, data_type: Type[DataType], progress_bars_type: ProgressBarsType, + leave: bool = True, ): def closure( data: Any, @@ -380,7 +382,7 @@ def closure( show_progress_bars = progress_bars_type != ProgressBarsType.No - progress_bars = get_progress_bars(progresses_length, show_progress_bars) + progress_bars = get_progress_bars(progresses_length, show_progress_bars, leave) progresses = [0] * nb_workers workers_status = [WorkerStatus.Running] * nb_workers @@ -446,6 +448,7 @@ def initialize( progress_bar=False, verbose=2, use_memory_fs: Optional[bool] = None, + leave: Optional[bool] = True, ) -> None: show_progress_bars = progress_bar is_memory_fs_available = Path(MEMORY_FS_ROOT).exists() @@ -510,36 +513,37 @@ def initialize( # DataFrame pd.DataFrame.parallel_apply = parallelize( - nb_workers, DataFrame.Apply, progress_bars_in_user_defined_function + nb_workers, DataFrame.Apply, progress_bars_in_user_defined_function, leave ) pd.DataFrame.parallel_applymap = parallelize( nb_workers, DataFrame.ApplyMap, progress_bars_in_user_defined_function_multiply_by_number_of_columns, + leave, ) # DataFrame GroupBy PandaDataFrameGroupBy.parallel_apply = parallelize( - nb_workers, DataFrameGroupBy.Apply, progress_bars_in_user_defined_function + nb_workers, DataFrameGroupBy.Apply, progress_bars_in_user_defined_function, leave ) # Expanding GroupBy PandasExpandingGroupby.parallel_apply = parallelize( - nb_workers, ExpandingGroupBy.Apply, progress_bars_in_work_function + nb_workers, ExpandingGroupBy.Apply, progress_bars_in_work_function, leave ) # Rolling GroupBy PandasRollingGroupby.parallel_apply = parallelize( - nb_workers, RollingGroupBy.Apply, progress_bars_in_work_function + nb_workers, RollingGroupBy.Apply, progress_bars_in_work_function, leave ) # Series pd.Series.parallel_apply = parallelize( - nb_workers, Series.Apply, progress_bars_in_user_defined_function + nb_workers, Series.Apply, progress_bars_in_user_defined_function, leave ) - pd.Series.parallel_map = parallelize(nb_workers, Series.Map, show_progress_bars) + pd.Series.parallel_map = parallelize(nb_workers, Series.Map, show_progress_bars, leave) # Series Rolling pd.core.window.Rolling.parallel_apply = parallelize( - nb_workers, SeriesRolling.Apply, progress_bars_in_user_defined_function + nb_workers, SeriesRolling.Apply, progress_bars_in_user_defined_function, leave ) diff --git a/pandarallel/progress_bars.py b/pandarallel/progress_bars.py index dfe9190..f3ebe69 100644 --- a/pandarallel/progress_bars.py +++ b/pandarallel/progress_bars.py @@ -118,12 +118,13 @@ def update(self, values: List[int]) -> None: class ProgressBarsNotebookLab(ProgressBars): - def __init__(self, maxs: List[int], show: bool) -> None: + def __init__(self, maxs: List[int], show: bool, leave: bool = True) -> None: """Initialization. Positional argument: maxs - List containing the max value of each progress bar """ self.__show = show + self.__leave = leave if not show: return @@ -159,6 +160,9 @@ def update(self, values: List[int]) -> None: if value >= bar.max: bar.bar_style = "success" + if not self.__leave: + bar.layout.display = 'none' + label.layout.display = 'none' label.value = "{} / {}".format(value, bar.max) @@ -172,10 +176,10 @@ def set_error(self, index: int) -> None: def get_progress_bars( - maxs: List[int], show + maxs: List[int], show, leave=True, ) -> Union[ProgressBarsNotebookLab, ProgressBarsConsole]: return ( - ProgressBarsNotebookLab(maxs, show) + ProgressBarsNotebookLab(maxs, show, leave) if is_notebook_lab() else ProgressBarsConsole(maxs, show) )