Adding data field and updating the logger name.

productionsystem/sql/SQLTableBase.py

@@ -9,7 +9,10 @@
 from enum import Enum
 from datetime import datetime
 from abc import ABCMeta
-from collections import Mapping
+try:
+    from collections import Mapping
+except ImportError:
+    from collections.abc import Mapping
 from sqlalchemy import Column
 from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
 from sqlalchemy.orm.attributes import InstrumentedAttribute

productionsystem/sql/models/DiracJobs.py

@@ -36,7 +36,7 @@ class DiracJobs(SQLTableBase):
     parametricjob_id = Column(Integer, nullable=False)
     status = Column(Enum(DiracStatus), nullable=False, default=DiracStatus.UNKNOWN)
     reschedules = Column(Integer, nullable=False, default=0)
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild(__qualname__)
 
     @classmethod
     def get(cls, diracjob_id=None, request_id=None, parametricjob_id=None, user_id=None):

productionsystem/sql/models/ParametricJobs.py

@@ -7,13 +7,17 @@
 import os
 import logging
 from datetime import datetime
-from collections import defaultdict, Counter, Iterable
+from collections import defaultdict, Counter
+try:
+    from collections import Iterable
+except ImportError:
+    from collections.abc import Iterable
 from copy import deepcopy
 from operator import attrgetter
 
 from future.utils import native
 import cherrypy
-from sqlalchemy import (Column, SmallInteger, Integer, Boolean, TEXT, TIMESTAMP,
+from sqlalchemy import (Column, SmallInteger, Integer, Boolean, TEXT, TIMESTAMP, PickleType,
                         ForeignKey, Enum, CheckConstraint, event, inspect)
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
@@ -37,6 +41,16 @@ def subdict(dct, keys, **kwargs):
     return out
 
 
+def split_args(dct, inclusive_keys):
+    out = {"data": {}}
+    for key, value in dct.items():
+        if key in inclusive_keys:
+            out[key] = value
+        else:
+            out["data"][key] = value
+    return out
+
+
 @cherrypy.expose
 @cherrypy.popargs('parametricjob_id')
 class ParametricJobs(SQLTableBase):
@@ -62,10 +76,11 @@ class ParametricJobs(SQLTableBase):
     num_submitted = Column(Integer, nullable=False, default=0)
     num_running = Column(Integer, nullable=False, default=0)
     log = Column(TEXT, nullable=False, default="")
+    data = Column(PickleType, nullable=True, default={})
     dirac_jobs = relationship("DiracJobs", cascade="all, delete-orphan",
                               primaryjoin="and_(ParametricJobs.request_id==DiracJobs.request_id, "
                                           "ParametricJobs.id==DiracJobs.parametricjob_id)")
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild(__qualname__)
 
     @hybrid_property
     def num_other(self):
@@ -79,9 +94,11 @@ def __init__(self, **kwargs):
         """Initialise."""
         required_args = set(self.required_columns).difference(kwargs)  # pylint: disable=no-member
         if required_args:
-            raise ValueError("Missing required keyword args: %s" % list(required_args))
+            raise ValueError(f"Missing required keyword args: {list(required_args)!s}")
         # pylint: disable=no-member
-        super(ParametricJobs, self).__init__(**subdict(kwargs, self.allowed_columns))
+        # disallow users to specify the data attribute directly
+        kwargs.pop("data", None)
+        super().__init__(**split_args(kwargs, self.allowed_columns))
 
     def update(self):
         """Update DB with current values."""

productionsystem/sql/models/Requests.py

@@ -10,7 +10,7 @@
 
 from future.utils import native
 import cherrypy
-from sqlalchemy import Column, Integer, TIMESTAMP, TEXT, ForeignKey, Enum, event, inspect
+from sqlalchemy import Column, Integer, TIMESTAMP, TEXT, PickleType, ForeignKey, Enum, event, inspect
 # from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import relationship, joinedload
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
@@ -28,6 +28,16 @@ def subdict(dct, keys, **kwargs):
     return out
 
 
+def split_args(dct, inclusive_keys):
+    out = {"data": {}}
+    for key, value in dct.items():
+        if key in inclusive_keys:
+            out[key] = value
+        else:
+            out["data"][key] = value
+    return out
+
+
 @cherrypy.expose
 @cherrypy.popargs('request_id')
 class Requests(SQLTableBase):
@@ -45,20 +55,25 @@ class Requests(SQLTableBase):
     status = Column(Enum(LocalStatus), nullable=False, default=LocalStatus.REQUESTED)
     timestamp = Column(TIMESTAMP, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
     log = Column(TEXT, nullable=False, default="")
+    data = Column(PickleType, nullable=True, default={})
     parametric_jobs = relationship("ParametricJobs", cascade="all, delete-orphan")
     requester = relationship("Users")
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild(__qualname__)
 
     def __init__(self, **kwargs):
         """Initialise."""
         required_args = set(self.required_columns).difference(kwargs)  # pylint: disable=no-member
         if required_args:
-            raise ValueError("Missing required keyword args: %s" % list(required_args))
-        # pylint: disable=no-member
-        super(Requests, self).__init__(**subdict(kwargs, self.allowed_columns))
-        parametricjobs = kwargs.get('parametricjobs', [])
+            raise ValueError(f"Missing required keyword args: {list(required_args)!s}")
+
+        # disallow direct setting of the data attribute
+        kwargs.pop("data", None)
+        parametricjobs = kwargs.pop('parametricjobs', [])
         if not parametricjobs:
             self.logger.warning("No parametricjobs associated with new request.")
+        # pylint: disable=no-member
+        super().__init__(**split_args(kwargs, self.allowed_columns))
+
         for job_id, parametricjob in enumerate(parametricjobs):
             parametricjob.pop('requester_id', None)
             parametricjob.pop('request_id', None)

productionsystem/sql/models/Services.py

@@ -26,7 +26,7 @@ class Services(SQLTableBase):
     name = Column(String(30), nullable=False, unique=True)
     status = Column(Enum(ServiceStatus), nullable=False, default=ServiceStatus.UNKNOWN)
     timestamp = Column(TIMESTAMP, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild(__qualname__)
 
     def add(self):
         """Add self to the DB."""

productionsystem/sql/models/Users.py

@@ -26,7 +26,7 @@ class Users(SQLTableBase):
     email = Column(TEXT, nullable=False)
     suspended = Column(Boolean, nullable=False)
     admin = Column(Boolean, nullable=False)
-    logger = logging.getLogger(__name__)
+    logger = logging.getLogger(__name__).getChild(__qualname__)
 
     @property
     def name(self):