script scanner load experiments from directories

lib/clients/script_scanner_gui/experiment_selector_widget.py

@@ -265,8 +265,7 @@ def setupLayout(self):
         self.repeat_button = QtGui.QPushButton("Repeat")
         self.scan_button = QtGui.QPushButton("Scan")
         self.schedule_button = QtGui.QPushButton("Schedule")
-        self.refresh_button = QtGui.QPushButton()
-        self.refresh_button.setIcon(QtGui.QIcon.fromTheme('view-refresh'))
+        self.refresh_button = QtGui.QPushButton("Refresh")
         layout.addWidget(label, 0, 0, 1, 1)
         layout.addWidget(self.dropdown, 0, 1, 1, 3)
         layout.addWidget(self.refresh_button, 0, 4, 1, 1)

lib/clients/script_scanner_gui/script_scanner_gui.py

@@ -357,7 +357,7 @@ def refresh_script(self, ident):
         sc = yield self.cxn.get_server('ScriptScanner')
         pv = yield self.cxn.get_server('ParameterVault')
         ident = int(ident)
-        sc.reload_available_scripts()
+        yield sc.reload_available_scripts()
         pv.reload_parameters()
         sc.refresh()
         self.reinitialize_scriptscanner()

lib/servers/script_scanner/scheduler.py

@@ -80,7 +80,7 @@ class scheduler(object):
 
     TODO: proper class name
     """
-    def __init__(self, signals):
+    def __init__(self, signals, allowed_concurrent):
         self.signals = signals
         # dict[identification] = running_script_instance
         self.running = {}
@@ -89,6 +89,7 @@ def __init__(self, signals):
         self.scheduled = {}
         self.scheduled_ID_counter = 0
         self.scan_ID_counter = 0
+        self.allowed_concurrent = allowed_concurrent
 
     def running_deferred_list(self):
         return [script.defer_on_done for script in self.running.itervalues() if not script.externally_launched]
@@ -180,7 +181,7 @@ def get_non_conflicting(self):
         non_conflicting = []
         for running, script in self.running.iteritems():
             cls_name = script.scan.script_cls.name
-            non_conf = config.allowed_concurrent.get(cls_name, None)
+            non_conf = self.allowed_concurrent.get(cls_name, None)
             if non_conf is not None:
                 non_conflicting.append(set(non_conf))
         if non_conflicting:

lib/servers/script_scanner/script_scanner.py

@@ -26,6 +26,11 @@
 import scan_methods
 from scheduler import scheduler
 import sys
+import os
+from six.moves import configparser
+import pkgutil
+import StringIO
+import importlib
 
 
 class script_class_parameters(object):
@@ -60,21 +65,53 @@ class ScriptScanner(ScriptSignalsServer):
 
     name = 'ScriptScanner'
 
+    @inlineCallbacks
     def initServer(self):
 
         # Dictionary with experiment.name as keys and
         # script_class_parameters instances are the values.
         self.script_parameters = {}
         # Instance of a complicated object
-        self.scheduler = scheduler(ScriptSignalsServer)
-        self.load_scripts()
+        yield self.load_scripts()
+        self.scheduler = scheduler(ScriptSignalsServer, self.allowed_concurrent)
 
+    @inlineCallbacks
     def load_scripts(self):
         '''
         loads script information from the configuration file
         '''
-        config = sc_config.config
-        for import_path, class_name in config.scripts:
+
+        self.allowed_concurrent = {}
+        scripts = []
+        reg_path = ["", "Servers", self.name]
+        reg = self.client.registry
+        p = reg.packet()
+        try:
+            p.cd(reg_path)
+            p.get("Directories")
+            ans = yield p.send()
+            paths = ans.get
+            if not len(paths):
+                print("No path found in registry")
+                raise Exception()
+        except:
+            print("Cannot load experiment paths from registry. " + 
+                  "Creating the registry path and checking config file now.")
+            p = reg.packet()
+            p.cd(reg_path, True)
+            p.set("Directories", [])
+            yield p.send()
+            config = sc_config.config
+            scripts = config.scripts
+            self.allowed_concurrent = config.allowed_concurrent
+        else:
+            experiments = self._get_all_experiments_from_basepaths(paths)
+            for experiment in experiments:
+                scripts.append((experiment[0], experiment[1]))
+                self.allowed_concurrent[experiment[1]] = experiment[2]
+
+        scripts = list(set(scripts))
+        for import_path, class_name in scripts:
             try:
                 __import__(import_path)
                 module = sys.modules[import_path]
@@ -98,6 +135,60 @@ def load_scripts(self):
                 else:
                     self.script_parameters[name] = script_class_parameters(name, cls, parameters)
 
+    def _get_all_experiments_from_basepaths(self, paths):
+        modules = []
+        for path in paths:
+            try:
+                imported_module = importlib.import_module(path)
+                self._list_submodules(modules, imported_module)
+            except Exception as e:
+                print("Exception when importing " + path + ". " + e)
+
+        modules = list(set(modules))
+        experiments = []
+        for module in modules:
+            try:
+                imported_module = importlib.import_module(module)
+                docstring = imported_module.__doc__
+                class_name, allow_concurrent = self._get_experiment_info(docstring)
+                if class_name is not None:
+                    experiments.append((module, class_name, allow_concurrent))
+            except Exception as e:
+                pass
+        return experiments
+
+    def _list_submodules(self, list, package_name):
+        for loader, module_name, is_pkg in pkgutil.walk_packages(package_name.__path__,
+                                                                 package_name.__name__ + '.'):
+            try:
+                list.append(module_name)
+                if is_pkg:
+                    module_name = importlib.import_module(module_name)
+                    self._list_submodules(list, module_name)
+            except Exception as e:
+                pass
+
+    def _get_experiment_info(self, docstring):
+        exp_info_start_str = "### BEGIN EXPERIMENT INFO"
+        exp_info_end_str = "### END EXPERIMENT INFO"
+        start = docstring.find(exp_info_start_str) + len(exp_info_start_str)
+        end = docstring.find(exp_info_end_str)
+        if end > start:
+            buf = StringIO.StringIO(docstring)
+            cp = configparser.ConfigParser()
+            cp.readfp(buf)
+            if eval(cp.get("info", "load_into_scriptscanner")):
+                name = cp.get("info", "name")
+                try:
+                    allow_concurrent = eval(cp.get("info", "allow_concurrent"))
+                except:
+                    allow_concurrent = []
+                return (name, allow_concurrent)
+            else:
+                return (None, None)
+        else:
+            return (None, None)
+
     @setting(0, "get_available_scripts", returns='*s')
     def get_available_scripts(self, c):
         return self.script_parameters.keys()
@@ -342,7 +433,7 @@ def error_finish_confirmed(self, c, script_ID, error_message):
     def reload_available_scripts(self, c):
         reload(sc_config)
         self.script_parameters = {}
-        self.load_scripts()
+        yield self.load_scripts()
 
     @inlineCallbacks
     def stopServer(self):