From 25d6b948adcd1641a3e33e017c3e53a13ce7721d Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 10:59:44 +0100 Subject: [PATCH 01/10] Removed type hints with | syntax --- pandaharvester/harvestercore/worker_errors.py | 3 ++- pandaharvester/harvestersubmitter/submitter_common.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pandaharvester/harvestercore/worker_errors.py b/pandaharvester/harvestercore/worker_errors.py index 7cf6a974..c97cf018 100644 --- a/pandaharvester/harvestercore/worker_errors.py +++ b/pandaharvester/harvestercore/worker_errors.py @@ -2,6 +2,7 @@ # Integer values for errors must be not less than 1000 import re +from typing import Optional ######## Error Code Message Pattern Maps (CMPMs) ######## @@ -172,7 +173,7 @@ def __init__(self, code_pattern_map: dict): self._code_pattern_map = code_pattern_map.copy() self._pattern_code_map = {v: k for k, v in self._code_pattern_map.items()} - def get_error_code(self, message: str) -> int | None: + def get_error_code(self, message: str) -> Optional[int]: """ Get the error code for a given message based on the defined patterns. diff --git a/pandaharvester/harvestersubmitter/submitter_common.py b/pandaharvester/harvestersubmitter/submitter_common.py index a3e1c19b..33cabc61 100644 --- a/pandaharvester/harvestersubmitter/submitter_common.py +++ b/pandaharvester/harvestersubmitter/submitter_common.py @@ -1,5 +1,6 @@ import random from math import log1p +from typing import List, Optional, Tuple ######################### # Pilot related functions @@ -115,8 +116,11 @@ def get_resource_type(resource_type_name, is_unified_queue, all_resource_types, # Compute weight of each CE according to worker stat, return tuple(dict, total weight score) def get_ce_weighting( - ce_endpoint_list: list | None = None, worker_ce_all_tuple: tuple | None = None, is_slave_queue: bool = False, fairshare_percent: int = 50 -) -> tuple: + ce_endpoint_list: Optional[List] = None, + worker_ce_all_tuple: Optional[Tuple] = None, + is_slave_queue: bool = False, + fairshare_percent: int = 50, +) -> Tuple: """ Compute the weighting of each CE based on worker statistics and throughput. From 9ffa60a6f5996bb55d2adddac94dcafd02ae05db Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 11:03:40 +0100 Subject: [PATCH 02/10] Added python 3.9 syntax check action on PR --- .github/workflows/syntax-check-39.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/syntax-check-39.yml diff --git a/.github/workflows/syntax-check-39.yml b/.github/workflows/syntax-check-39.yml new file mode 100644 index 00000000..d931f65e --- /dev/null +++ b/.github/workflows/syntax-check-39.yml @@ -0,0 +1,21 @@ +name: syntax-check-39.yml + +on: + pull_request: + branches: ["main"] + +jobs: + syntax-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Install Python 3.9 + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + # Syntax check + - name: Run syntax check + run: | + python -m compileall -q . \ No newline at end of file From 8c65cb66bad86d7f41a395955c1194932927edcf Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 11:04:49 +0100 Subject: [PATCH 03/10] Renamed 311 syntax check action --- .github/workflows/{syntax-check.yml => syntax-check-311.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{syntax-check.yml => syntax-check-311.yml} (100%) diff --git a/.github/workflows/syntax-check.yml b/.github/workflows/syntax-check-311.yml similarity index 100% rename from .github/workflows/syntax-check.yml rename to .github/workflows/syntax-check-311.yml From 2c3a3b1f508707e7068e702d2744f0d0b6db2036 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 11:19:34 +0100 Subject: [PATCH 04/10] Dummy commit to trigger PR actions --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a2753bc7..e9b643de 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ # Setup for Harvester # # + import sys from setuptools import find_packages, setup From f973ee21b4c3c3ad8c870a7cf6ef957e2af99ac7 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 11:25:42 +0100 Subject: [PATCH 05/10] Dummy commit to trigger PR actions --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index e9b643de..a2753bc7 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ # Setup for Harvester # # - import sys from setuptools import find_packages, setup From 31accc9ecead6b8d109f59b0d9dbf41e61927d86 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 12:40:07 +0100 Subject: [PATCH 06/10] Bug in base_zipper.py: undeclared tmp_logger variable --- pandaharvester/harvesterzipper/base_zipper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaharvester/harvesterzipper/base_zipper.py b/pandaharvester/harvesterzipper/base_zipper.py index b047e7de..226006cb 100644 --- a/pandaharvester/harvesterzipper/base_zipper.py +++ b/pandaharvester/harvesterzipper/base_zipper.py @@ -297,7 +297,7 @@ def ssh_make_one_zip(self, arg_dict): retCode = p0.returncode if retCode != 0: msgStr = f"failed to make tmpargfile remotely with {stdOut}:{stdErr}" - tmp_log.error(msgStr) + self.zip_tmp_log.error(msgStr) return False, f"failed to zip with {msgStr}" stdOut_str = stdOut if (isinstance(stdOut, str) or stdOut is None) else stdOut.decode() tmpargfile_name = stdOut_str.strip("\n") @@ -333,7 +333,7 @@ def ssh_make_one_zip(self, arg_dict): retCode = p1a.returncode if retCode != 0: msgStr = f"failed to delete tmpargfile remotely with {stdOut}:{stdErr}" - tmp_log.error(msgStr) + self.zip_tmp_log.error(msgStr) del p1a, stdOut, stdErr gc.collect() # avoid overwriting From 71ff7f2de9d9fe5ff89171863f5a23f500c1ec54 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 12:41:33 +0100 Subject: [PATCH 07/10] Bug in stageInTest_dpb.py: missing time import --- pandaharvester/harvestertest/stageInTest_dpb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandaharvester/harvestertest/stageInTest_dpb.py b/pandaharvester/harvestertest/stageInTest_dpb.py index 62ff3fe2..a74e969d 100644 --- a/pandaharvester/harvestertest/stageInTest_dpb.py +++ b/pandaharvester/harvestertest/stageInTest_dpb.py @@ -1,4 +1,5 @@ import sys +import time from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.plugin_factory import PluginFactory From c3eb073c0ac264401fdb3ac0270aa3cf168778f4 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 12:44:39 +0100 Subject: [PATCH 08/10] Multiple bugs in superfacility_sweeper.py --- .../harvestersweeper/superfacility_sweeper.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pandaharvester/harvestersweeper/superfacility_sweeper.py b/pandaharvester/harvestersweeper/superfacility_sweeper.py index d7cff594..b4883c88 100644 --- a/pandaharvester/harvestersweeper/superfacility_sweeper.py +++ b/pandaharvester/harvestersweeper/superfacility_sweeper.py @@ -1,11 +1,9 @@ import os -import json -import requests +import shutil -from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.superfacility_utils import SuperfacilityClient - +from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper baseLogger = core_utils.setup_logger("superfacility_sweeper") @@ -13,7 +11,7 @@ class SuperfacilitySweeper(BaseSweeper): def __init__(self, **kwargs): BaseSweeper.__init__(self, **kwargs) - self.cred_dir = kwarg.get("superfacility_cred_dir") + self.cred_dir = kwargs.get("superfacility_cred_dir") self.sf_client = SuperfacilityClient(self.cred_dir) def kill_worker(self, workspec): @@ -30,7 +28,7 @@ def kill_worker(self, workspec): tmpLog.error(errStr) return False, errStr - if data.get('status') == 'success': + if data.get("status") == "success": tmpLog.info(f"Succeeded to kill workerID={workspec.workerID} batchID={workspec.workerID}") else: errStr = f"Failed to cancel job {jobid}: status: {data.get('status')}" @@ -44,11 +42,11 @@ def sweep_worker(self, workspec): if ap and os.path.exists(ap): try: shutil.rmtree(ap) - logger.info(f"Removed directory {ap}") + tmpLog.info(f"Removed directory {ap}") except Exception as e: err = f"Failed to remove {ap}: {e}" - logger.error(err) + tmpLog.error(err) return False, err else: - logger.info("Access point already removed or none provided.") + tmpLog.info("Access point already removed or none provided.") return True, "" From 8d3bfd9a56df6882df6f99a40e2c35c67c6d483c Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 12:47:47 +0100 Subject: [PATCH 09/10] Removed syntax check for 3.9 since not working --- .github/workflows/syntax-check-39.yml | 21 ------------------- ...{syntax-check-311.yml => syntax-check.yml} | 0 pyproject.toml | 2 +- 3 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 .github/workflows/syntax-check-39.yml rename .github/workflows/{syntax-check-311.yml => syntax-check.yml} (100%) diff --git a/.github/workflows/syntax-check-39.yml b/.github/workflows/syntax-check-39.yml deleted file mode 100644 index d931f65e..00000000 --- a/.github/workflows/syntax-check-39.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: syntax-check-39.yml - -on: - pull_request: - branches: ["main"] - -jobs: - syntax-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - # Install Python 3.9 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - # Syntax check - - name: Run syntax check - run: | - python -m compileall -q . \ No newline at end of file diff --git a/.github/workflows/syntax-check-311.yml b/.github/workflows/syntax-check.yml similarity index 100% rename from .github/workflows/syntax-check-311.yml rename to .github/workflows/syntax-check.yml diff --git a/pyproject.toml b/pyproject.toml index 1b7d8f8c..ec2c2d29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,4 +83,4 @@ aggressive = 3 profile = "black" [tool.flynt] -line-length = 160 +line-length = 160 \ No newline at end of file From aa1660a2d089afcb6203f39735a5a41e88b2d88b Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 17 Nov 2025 14:14:22 +0100 Subject: [PATCH 10/10] Removed python3.10 requirement --- pyproject.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ec2c2d29..21068cfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,12 +27,6 @@ dependencies = [ 'panda-pilot >= 2.7.2.1', ] -requires-python = ">=3.10" -classifiers = [ - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] - [project.optional-dependencies] kubernetes = ['kubernetes', 'pyyaml'] mysql = ['mysqlclient']