diff --git a/pandaharvester/harvestercore/worker_errors.py b/pandaharvester/harvestercore/worker_errors.py index 7cf6a974..c97cf018 100644 --- a/pandaharvester/harvestercore/worker_errors.py +++ b/pandaharvester/harvestercore/worker_errors.py @@ -2,6 +2,7 @@ # Integer values for errors must be not less than 1000 import re +from typing import Optional ######## Error Code Message Pattern Maps (CMPMs) ######## @@ -172,7 +173,7 @@ def __init__(self, code_pattern_map: dict): self._code_pattern_map = code_pattern_map.copy() self._pattern_code_map = {v: k for k, v in self._code_pattern_map.items()} - def get_error_code(self, message: str) -> int | None: + def get_error_code(self, message: str) -> Optional[int]: """ Get the error code for a given message based on the defined patterns. diff --git a/pandaharvester/harvestersubmitter/submitter_common.py b/pandaharvester/harvestersubmitter/submitter_common.py index a3e1c19b..33cabc61 100644 --- a/pandaharvester/harvestersubmitter/submitter_common.py +++ b/pandaharvester/harvestersubmitter/submitter_common.py @@ -1,5 +1,6 @@ import random from math import log1p +from typing import List, Optional, Tuple ######################### # Pilot related functions @@ -115,8 +116,11 @@ def get_resource_type(resource_type_name, is_unified_queue, all_resource_types, # Compute weight of each CE according to worker stat, return tuple(dict, total weight score) def get_ce_weighting( - ce_endpoint_list: list | None = None, worker_ce_all_tuple: tuple | None = None, is_slave_queue: bool = False, fairshare_percent: int = 50 -) -> tuple: + ce_endpoint_list: Optional[List] = None, + worker_ce_all_tuple: Optional[Tuple] = None, + is_slave_queue: bool = False, + fairshare_percent: int = 50, +) -> Tuple: """ Compute the weighting of each CE based on worker statistics and throughput. diff --git a/pandaharvester/harvestersweeper/superfacility_sweeper.py b/pandaharvester/harvestersweeper/superfacility_sweeper.py index d7cff594..b4883c88 100644 --- a/pandaharvester/harvestersweeper/superfacility_sweeper.py +++ b/pandaharvester/harvestersweeper/superfacility_sweeper.py @@ -1,11 +1,9 @@ import os -import json -import requests +import shutil -from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.superfacility_utils import SuperfacilityClient - +from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper baseLogger = core_utils.setup_logger("superfacility_sweeper") @@ -13,7 +11,7 @@ class SuperfacilitySweeper(BaseSweeper): def __init__(self, **kwargs): BaseSweeper.__init__(self, **kwargs) - self.cred_dir = kwarg.get("superfacility_cred_dir") + self.cred_dir = kwargs.get("superfacility_cred_dir") self.sf_client = SuperfacilityClient(self.cred_dir) def kill_worker(self, workspec): @@ -30,7 +28,7 @@ def kill_worker(self, workspec): tmpLog.error(errStr) return False, errStr - if data.get('status') == 'success': + if data.get("status") == "success": tmpLog.info(f"Succeeded to kill workerID={workspec.workerID} batchID={workspec.workerID}") else: errStr = f"Failed to cancel job {jobid}: status: {data.get('status')}" @@ -44,11 +42,11 @@ def sweep_worker(self, workspec): if ap and os.path.exists(ap): try: shutil.rmtree(ap) - logger.info(f"Removed directory {ap}") + tmpLog.info(f"Removed directory {ap}") except Exception as e: err = f"Failed to remove {ap}: {e}" - logger.error(err) + tmpLog.error(err) return False, err else: - logger.info("Access point already removed or none provided.") + tmpLog.info("Access point already removed or none provided.") return True, "" diff --git a/pandaharvester/harvestertest/stageInTest_dpb.py b/pandaharvester/harvestertest/stageInTest_dpb.py index 62ff3fe2..a74e969d 100644 --- a/pandaharvester/harvestertest/stageInTest_dpb.py +++ b/pandaharvester/harvestertest/stageInTest_dpb.py @@ -1,4 +1,5 @@ import sys +import time from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.plugin_factory import PluginFactory diff --git a/pandaharvester/harvesterzipper/base_zipper.py b/pandaharvester/harvesterzipper/base_zipper.py index b047e7de..226006cb 100644 --- a/pandaharvester/harvesterzipper/base_zipper.py +++ b/pandaharvester/harvesterzipper/base_zipper.py @@ -297,7 +297,7 @@ def ssh_make_one_zip(self, arg_dict): retCode = p0.returncode if retCode != 0: msgStr = f"failed to make tmpargfile remotely with {stdOut}:{stdErr}" - tmp_log.error(msgStr) + self.zip_tmp_log.error(msgStr) return False, f"failed to zip with {msgStr}" stdOut_str = stdOut if (isinstance(stdOut, str) or stdOut is None) else stdOut.decode() tmpargfile_name = stdOut_str.strip("\n") @@ -333,7 +333,7 @@ def ssh_make_one_zip(self, arg_dict): retCode = p1a.returncode if retCode != 0: msgStr = f"failed to delete tmpargfile remotely with {stdOut}:{stdErr}" - tmp_log.error(msgStr) + self.zip_tmp_log.error(msgStr) del p1a, stdOut, stdErr gc.collect() # avoid overwriting diff --git a/pyproject.toml b/pyproject.toml index 1b7d8f8c..21068cfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,12 +27,6 @@ dependencies = [ 'panda-pilot >= 2.7.2.1', ] -requires-python = ">=3.10" -classifiers = [ - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] - [project.optional-dependencies] kubernetes = ['kubernetes', 'pyyaml'] mysql = ['mysqlclient'] @@ -83,4 +77,4 @@ aggressive = 3 profile = "black" [tool.flynt] -line-length = 160 +line-length = 160 \ No newline at end of file