From 753604275577c9888e2a7d4fc883d90c3b2d9bac Mon Sep 17 00:00:00 2001 From: mightqxc Date: Mon, 20 Oct 2025 10:51:06 +0200 Subject: [PATCH 1/3] fix --- pandaharvester/commit_timestamp.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaharvester/commit_timestamp.py b/pandaharvester/commit_timestamp.py index ed7812af..3bedabd0 100644 --- a/pandaharvester/commit_timestamp.py +++ b/pandaharvester/commit_timestamp.py @@ -1 +1 @@ -timestamp = "20-10-2025 08:35:35 on flin (by mightqxc)" +timestamp = "20-10-2025 08:51:07 on flin (by mightqxc)" diff --git a/setup.py b/setup.py index a2753bc7..e70194d2 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ extras_require={ "kubernetes": ["kubernetes", "pyyaml"], "mysql": ["mysqlclient"], - "atlasgrid": ["uWSGI >= 2.0.20", "htcondor>=10.3.0,<25.0,", "mysqlclient >= 2.1.1"], + "atlasgrid": ["uWSGI >= 2.0.20", "htcondor >= 10.3.0, < 25.0", "mysqlclient >= 2.1.1"], }, data_files=[ # config and cron files From 5c90e9754a73ce175f243d225d93e826a5f81031 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 11 Nov 2025 08:58:36 +0100 Subject: [PATCH 2/3] log --- pandaharvester/commit_timestamp.py | 2 +- pandaharvester/harvesterbody/cacher.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandaharvester/commit_timestamp.py b/pandaharvester/commit_timestamp.py index 3bedabd0..4fc7e154 100644 --- a/pandaharvester/commit_timestamp.py +++ b/pandaharvester/commit_timestamp.py @@ -1 +1 @@ -timestamp = "20-10-2025 08:51:07 on flin (by mightqxc)" +timestamp = "11-11-2025 07:58:37 on flin (by mightqxc)" diff --git a/pandaharvester/harvesterbody/cacher.py b/pandaharvester/harvesterbody/cacher.py index 3b177db0..2d5df84f 100644 --- a/pandaharvester/harvesterbody/cacher.py +++ b/pandaharvester/harvesterbody/cacher.py @@ -63,7 +63,9 @@ def _refresh_cache(inputs): return # get information tmpStat, newInfo = self.get_data(infoURL, mainLog) - if not tmpStat: + if tmpStat: + mainLog.debug(f"got data for key={mainKey} subKey={subKey} from {infoURL}") + else: mainLog.error(f"failed to get info for key={mainKey} subKey={subKey}") return # update From 48f25c39f219ce33174744bf7651765f5dac30c5 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 20 Jan 2026 14:00:31 +0100 Subject: [PATCH 3/3] support per_core_attr in CRIC catchall --- pandaharvester/commit_timestamp.py | 2 +- pandaharvester/harvestermisc/info_utils.py | 35 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pandaharvester/commit_timestamp.py b/pandaharvester/commit_timestamp.py index 4fc7e154..90171e29 100644 --- a/pandaharvester/commit_timestamp.py +++ b/pandaharvester/commit_timestamp.py @@ -1 +1 @@ -timestamp = "11-11-2025 07:58:37 on flin (by mightqxc)" +timestamp = "20-01-2026 13:00:32 on flin (by mightqxc)" diff --git a/pandaharvester/harvestermisc/info_utils.py b/pandaharvester/harvestermisc/info_utils.py index b4af0350..83fe0f58 100644 --- a/pandaharvester/harvestermisc/info_utils.py +++ b/pandaharvester/harvestermisc/info_utils.py @@ -1,3 +1,4 @@ +import re import threading import time @@ -17,6 +18,12 @@ class PandaQueuesDict(dict, PluginBase, metaclass=SingletonWithID): Able to query with either PanDA Queue name or PanDA Resource name """ + candidate_per_core_attrs = ( + "maxrss", + "minrss", + "maxwdir", + ) + def __init__(self, **kwargs): dict.__init__(self) PluginBase.__init__(self, **kwargs) @@ -33,6 +40,27 @@ def _is_fresh(self): return True return False + @staticmethod + def has_value_in_catchall(panda_queues_dict, key): + """ + Check if specific value is in catchall attributes + """ + catchall_str = panda_queues_dict.get("catchall") + if catchall_str is None: + return False + for tmp_key in catchall_str.split(","): + tmp_match = re.search(f"^{key}(=|)*", tmp_key) + if tmp_match is not None: + return True + return False + + @staticmethod + def use_per_core_attr(panda_queues_dict): + """ + Check if treating all attributes as per-core + """ + return PandaQueuesDict.has_value_in_catchall(panda_queues_dict, "per_core_attr") + def _refresh(self): with self.lock: if self._is_fresh(): @@ -48,6 +76,13 @@ def _refresh(self): pass else: self[panda_resource] = v + # handle per-core attributes: scale with corecount if per-core + if PandaQueuesDict.use_per_core_attr(v): + core_count = v.get("corecount", 1) + for attr in self.candidate_per_core_attrs: + if attr in v and core_count > 0: + v[attr] = v[attr] * core_count + # successfully refreshed from cache self.last_refresh_ts = time.time() else: # not getting cache; shorten next period into 5 sec