From 952ba565d41e5f172bc65054a3653d95417c8d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Mon, 26 Jun 2023 12:07:42 +0100 Subject: [PATCH 1/2] bug fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/data_indices/utils.py | 9 +++++++++ deepsearch/documents/core/convert.py | 21 +++++++++++++++++++++ deepsearch/documents/core/create_report.py | 5 +++++ deepsearch/documents/core/utils.py | 5 +++++ 4 files changed, 40 insertions(+) diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index a45cda81..3f7a0561 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -74,6 +74,11 @@ def process_url_input( task_ids = [] # submit urls count_urls = len(urls) + + # Check if there are valid targets to iterate over + if count_urls == 0: + print("No urls resolved from input") + return with tqdm( total=count_urls, desc=f"{'Submitting input:': <{progressbar.padding}}", @@ -133,6 +138,10 @@ def process_local_file( # container for task_ids task_ids = [] + # Check if there are valid targets to iterate over + if count_total_files == 0: + print("No files resolved from input") + return # start loop with tqdm( total=count_total_files, diff --git a/deepsearch/documents/core/convert.py b/deepsearch/documents/core/convert.py index 10b26a39..cc0be911 100644 --- a/deepsearch/documents/core/convert.py +++ b/deepsearch/documents/core/convert.py @@ -116,6 +116,11 @@ def send_files_for_conversion( # container for task_ids task_ids = [] + # Check if there are valid targets to iterate over + if len(files_zip) == 0: + print("No files resolved from input") + return [] + # start loop with tqdm( total=len(files_zip), @@ -163,6 +168,11 @@ def check_status_running_tasks( ) statuses = [] + # Check if there are valid targets to iterate over + if count_total == 0: + print("No task_ids resolved from input") + return [] + with tqdm( total=count_total, desc=f"{'Converting input:': <{progressbar.padding}}", @@ -227,6 +237,11 @@ def download_converted_documents( shows progress bar if True """ + # Check if there are valid targets to iterate over + if len(download_urls) == 0: + print("No urls resolved from input") + return + with tqdm( total=len(download_urls), desc=f"{'Downloading result:': <{progressbar.padding}}", @@ -280,6 +295,12 @@ def send_urls_for_conversion( """ count_urls = len(urls) task_ids = [] + + # Check if there are valid targets to iterate over + if count_urls == 0: + print("No urls resolved from input") + return [] + with tqdm( total=count_urls, desc=f"{'Submitting input:': <{progressbar.padding}}", diff --git a/deepsearch/documents/core/create_report.py b/deepsearch/documents/core/create_report.py index 976c13c2..80f604b1 100644 --- a/deepsearch/documents/core/create_report.py +++ b/deepsearch/documents/core/create_report.py @@ -71,6 +71,11 @@ def get_multiple_reports( writer = csv.writer(csvfile) writer.writerow(["batch_number", "task_id", "status", "document"]) + # Check if there are valid targets to iterate over + if len(task_ids) == 0: + print("No task_ids resolved from input") + return + # start loop with tqdm( total=len(task_ids), diff --git a/deepsearch/documents/core/utils.py b/deepsearch/documents/core/utils.py index 46c1d20e..1246ac77 100644 --- a/deepsearch/documents/core/utils.py +++ b/deepsearch/documents/core/utils.py @@ -103,6 +103,11 @@ def batch_single_files( # catch all filenames and batch names batched_files = [] + # Check if there are valid targets to iterate over + if len(files_to_upload) == 0: + print("No files resolved from input") + return [] + if len(files_to_upload) != 0: with tqdm( total=len(files_to_upload), From 40386d64061c63df2d8f346d6734ad219fb23df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Tue, 27 Jun 2023 14:20:55 +0100 Subject: [PATCH 2/2] Now raising exceptions instead of printing the issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/data_indices/utils.py | 6 ++---- deepsearch/documents/core/convert.py | 12 ++++-------- deepsearch/documents/core/create_report.py | 3 +-- deepsearch/documents/core/utils.py | 5 ----- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index 3f7a0561..cefb3509 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -77,8 +77,7 @@ def process_url_input( # Check if there are valid targets to iterate over if count_urls == 0: - print("No urls resolved from input") - return + raise ValueError("No urls resolved from input") with tqdm( total=count_urls, desc=f"{'Submitting input:': <{progressbar.padding}}", @@ -140,8 +139,7 @@ def process_local_file( # Check if there are valid targets to iterate over if count_total_files == 0: - print("No files resolved from input") - return + raise ValueError("No files resolved from input") # start loop with tqdm( total=count_total_files, diff --git a/deepsearch/documents/core/convert.py b/deepsearch/documents/core/convert.py index cc0be911..a3362208 100644 --- a/deepsearch/documents/core/convert.py +++ b/deepsearch/documents/core/convert.py @@ -118,8 +118,7 @@ def send_files_for_conversion( # Check if there are valid targets to iterate over if len(files_zip) == 0: - print("No files resolved from input") - return [] + raise ValueError("No files resolved from input") # start loop with tqdm( @@ -170,8 +169,7 @@ def check_status_running_tasks( # Check if there are valid targets to iterate over if count_total == 0: - print("No task_ids resolved from input") - return [] + raise ValueError("No task_ids resolved from input") with tqdm( total=count_total, @@ -239,8 +237,7 @@ def download_converted_documents( # Check if there are valid targets to iterate over if len(download_urls) == 0: - print("No urls resolved from input") - return + raise ValueError("No urls resolved from input") with tqdm( total=len(download_urls), @@ -298,8 +295,7 @@ def send_urls_for_conversion( # Check if there are valid targets to iterate over if count_urls == 0: - print("No urls resolved from input") - return [] + raise ValueError("No urls resolved from input") with tqdm( total=count_urls, diff --git a/deepsearch/documents/core/create_report.py b/deepsearch/documents/core/create_report.py index 80f604b1..348f41fe 100644 --- a/deepsearch/documents/core/create_report.py +++ b/deepsearch/documents/core/create_report.py @@ -73,8 +73,7 @@ def get_multiple_reports( # Check if there are valid targets to iterate over if len(task_ids) == 0: - print("No task_ids resolved from input") - return + raise ValueError("No task_ids resolved from input") # start loop with tqdm( diff --git a/deepsearch/documents/core/utils.py b/deepsearch/documents/core/utils.py index 1246ac77..46c1d20e 100644 --- a/deepsearch/documents/core/utils.py +++ b/deepsearch/documents/core/utils.py @@ -103,11 +103,6 @@ def batch_single_files( # catch all filenames and batch names batched_files = [] - # Check if there are valid targets to iterate over - if len(files_to_upload) == 0: - print("No files resolved from input") - return [] - if len(files_to_upload) != 0: with tqdm( total=len(files_to_upload),