support per_core_attr in CRIC catchall; minor fixes

pandaharvester/commit_timestamp.py

@@ -1 +1 @@
-timestamp = "20-10-2025 08:35:35 on flin (by mightqxc)"
+timestamp = "20-01-2026 13:00:32 on flin (by mightqxc)"

pandaharvester/harvesterbody/cacher.py

@@ -63,7 +63,9 @@ def _refresh_cache(inputs):
                     return
                 # get information
                 tmpStat, newInfo = self.get_data(infoURL, mainLog)
-                if not tmpStat:
+                if tmpStat:
+                    mainLog.debug(f"got data for key={mainKey} subKey={subKey} from {infoURL}")
+                else:
                     mainLog.error(f"failed to get info for key={mainKey} subKey={subKey}")
                     return
                 # update

pandaharvester/harvestermisc/info_utils.py

@@ -1,3 +1,4 @@
+import re
 import threading
 import time
 
@@ -17,6 +18,12 @@ class PandaQueuesDict(dict, PluginBase, metaclass=SingletonWithID):
     Able to query with either PanDA Queue name or PanDA Resource name
     """
 
+    candidate_per_core_attrs = (
+        "maxrss",
+        "minrss",
+        "maxwdir",
+    )
+
     def __init__(self, **kwargs):
         dict.__init__(self)
         PluginBase.__init__(self, **kwargs)
@@ -33,6 +40,27 @@ def _is_fresh(self):
             return True
         return False
 
+    @staticmethod
+    def has_value_in_catchall(panda_queues_dict, key):
+        """
+        Check if specific value is in catchall attributes
+        """
+        catchall_str = panda_queues_dict.get("catchall")
+        if catchall_str is None:
+            return False
+        for tmp_key in catchall_str.split(","):
+            tmp_match = re.search(f"^{key}(=|)*", tmp_key)
+            if tmp_match is not None:
+                return True
+        return False
+
+    @staticmethod
+    def use_per_core_attr(panda_queues_dict):
+        """
+        Check if treating all attributes as per-core
+        """
+        return PandaQueuesDict.has_value_in_catchall(panda_queues_dict, "per_core_attr")
+
     def _refresh(self):
         with self.lock:
             if self._is_fresh():
@@ -48,6 +76,13 @@ def _refresh(self):
                         pass
                     else:
                         self[panda_resource] = v
+                    # handle per-core attributes: scale with corecount if per-core
+                    if PandaQueuesDict.use_per_core_attr(v):
+                        core_count = v.get("corecount", 1)
+                        for attr in self.candidate_per_core_attrs:
+                            if attr in v and core_count > 0:
+                                v[attr] = v[attr] * core_count
+                # successfully refreshed from cache
                 self.last_refresh_ts = time.time()
             else:
                 # not getting cache; shorten next period into 5 sec

setup.py

@@ -41,7 +41,7 @@
     extras_require={
         "kubernetes": ["kubernetes", "pyyaml"],
         "mysql": ["mysqlclient"],
-        "atlasgrid": ["uWSGI >= 2.0.20", "htcondor>=10.3.0,<25.0,", "mysqlclient >= 2.1.1"],
+        "atlasgrid": ["uWSGI >= 2.0.20", "htcondor >= 10.3.0, < 25.0", "mysqlclient >= 2.1.1"],
     },
     data_files=[
         # config and cron files