From ecf97c681cb17cf57b2578f0a4c7ec2dc7cb7be5 Mon Sep 17 00:00:00 2001 From: Mingyu Fan Date: Tue, 27 Mar 2018 23:47:45 -0700 Subject: [PATCH 1/7] add "refresh" label on refresh button --- lib/clients/script_scanner_gui/experiment_selector_widget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/clients/script_scanner_gui/experiment_selector_widget.py b/lib/clients/script_scanner_gui/experiment_selector_widget.py index f9c4981..236cb65 100644 --- a/lib/clients/script_scanner_gui/experiment_selector_widget.py +++ b/lib/clients/script_scanner_gui/experiment_selector_widget.py @@ -265,8 +265,7 @@ def setupLayout(self): self.repeat_button = QtGui.QPushButton("Repeat") self.scan_button = QtGui.QPushButton("Scan") self.schedule_button = QtGui.QPushButton("Schedule") - self.refresh_button = QtGui.QPushButton() - self.refresh_button.setIcon(QtGui.QIcon.fromTheme('view-refresh')) + self.refresh_button = QtGui.QPushButton("Refresh") layout.addWidget(label, 0, 0, 1, 1) layout.addWidget(self.dropdown, 0, 1, 1, 3) layout.addWidget(self.refresh_button, 0, 4, 1, 1) From af5e66d1e4861f9c83c536a3d2a8b078657633a7 Mon Sep 17 00:00:00 2001 From: Mingyu Fan Date: Wed, 28 Mar 2018 00:49:11 -0700 Subject: [PATCH 2/7] adds ability to load all experiments from basepaths --- lib/servers/script_scanner/script_scanner.py | 42 ++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index 85bc78b..e433280 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -26,6 +26,8 @@ import scan_methods from scheduler import scheduler import sys +import os +from six.moves import configparser class script_class_parameters(object): @@ -74,6 +76,14 @@ def load_scripts(self): loads script information from the configuration file ''' config = sc_config.config + + paths = config.paths + experiments_from_paths = self._get_all_experiments_from_basepaths(paths) + for experiment in experiments_from_paths: + config.scripts.append(experiment) + + config.scripts = list(set(config.scripts)) + for import_path, class_name in config.scripts: try: __import__(import_path) @@ -98,6 +108,38 @@ def load_scripts(self): else: self.script_parameters[name] = script_class_parameters(name, cls, parameters) + def _get_all_experiments_from_basepaths(self, basepaths): + modules = [] + for path in basepaths: + for root, dirs, files in os.walk(path): + for file in files: + if os.path.splitext(file)[1] == ".py" and file != "__init__.py": + modules.append((os.path.splitext(file)[0], os.path.join(root, file))) + + experiments = [] + import importlib + for module in modules: + try: + imported_module = importlib.import_module(modules[1]) + docstring = imported_module.__doc__ + class_name = self._get_experiment_class_name_from_docstring(docstring) + if class_name is not None: + experiments.append((modules[1], class_name)) + return experiments + + def _get_experiment_class_name_from_docstring(self, docstring): + exp_info_start_str = "### BEGIN EXPERIMENT INFO" + exp_info_end_str = "### END EXPERIMENT INFO" + start = docstring.find(exp_info_start_str) + len(exp_info_start_str) + end = docstring.find(exp_info_end_str) + cp = configparser.ConfigParser() + cp.read_string(docstring) + depreciated = cp["info"]["depreciated"] + if not depreciated: + return cp["info"]["name"] + else: + return None + @setting(0, "get_available_scripts", returns='*s') def get_available_scripts(self, c): return self.script_parameters.keys() From 56cb138457f101dd8fb70278a4fc030eb8ec2fc4 Mon Sep 17 00:00:00 2001 From: fanmingyu212 Date: Wed, 28 Mar 2018 01:39:02 -0700 Subject: [PATCH 3/7] add experiment by paths should work now. Imports needs to be cleaned. --- lib/servers/script_scanner/script_scanner.py | 44 +++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index e433280..13d5713 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -108,35 +108,51 @@ def load_scripts(self): else: self.script_parameters[name] = script_class_parameters(name, cls, parameters) - def _get_all_experiments_from_basepaths(self, basepaths): + def _get_all_experiments_from_basepaths(self, paths): modules = [] - for path in basepaths: - for root, dirs, files in os.walk(path): - for file in files: - if os.path.splitext(file)[1] == ".py" and file != "__init__.py": - modules.append((os.path.splitext(file)[0], os.path.join(root, file))) + import importlib + imported_module = importlib.import_module(paths[0]) + self._list_submodules(modules, imported_module) + modules = list(set(modules)) experiments = [] - import importlib for module in modules: try: - imported_module = importlib.import_module(modules[1]) + imported_module = importlib.import_module(module) docstring = imported_module.__doc__ class_name = self._get_experiment_class_name_from_docstring(docstring) if class_name is not None: - experiments.append((modules[1], class_name)) + experiments.append((module, class_name)) + except: + pass return experiments + + def _list_submodules(self, list, package_name): + import pkgutil + for loader, module_name, is_pkg in pkgutil.walk_packages(package_name.__path__, package_name.__name__+'.'): + try: + list.append(module_name) + if is_pkg: + module_name = __import__(module_name, fromlist='dummylist') + self._list_submodules(list, module_name) + except: + pass def _get_experiment_class_name_from_docstring(self, docstring): exp_info_start_str = "### BEGIN EXPERIMENT INFO" exp_info_end_str = "### END EXPERIMENT INFO" start = docstring.find(exp_info_start_str) + len(exp_info_start_str) end = docstring.find(exp_info_end_str) - cp = configparser.ConfigParser() - cp.read_string(docstring) - depreciated = cp["info"]["depreciated"] - if not depreciated: - return cp["info"]["name"] + if end > start: + import StringIO + buf = StringIO.StringIO(docstring) + cp = configparser.ConfigParser() + cp.readfp(buf) + depreciated = eval(cp.get("info", "depreciated")) + if not depreciated: + return cp.get("info", "name") + else: + return None else: return None From fcc140eef9fea5951dbef2eb52f76835567ed2c1 Mon Sep 17 00:00:00 2001 From: Mingyu Fan Date: Thu, 29 Mar 2018 00:29:37 -0700 Subject: [PATCH 4/7] backward compatibility and add optional allow_concurrent in module docstring --- lib/servers/script_scanner/scheduler.py | 5 +- lib/servers/script_scanner/script_scanner.py | 94 ++++++++++++-------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/lib/servers/script_scanner/scheduler.py b/lib/servers/script_scanner/scheduler.py index b24dd47..b08a37f 100644 --- a/lib/servers/script_scanner/scheduler.py +++ b/lib/servers/script_scanner/scheduler.py @@ -75,7 +75,7 @@ def __init__(self, scan, defer_on_done, status, priority=-1, self.externally_launched = externally_launched -class scheduler(object): +class scheduler(object, allowed_concurrent): """ TODO: proper class name @@ -89,6 +89,7 @@ def __init__(self, signals): self.scheduled = {} self.scheduled_ID_counter = 0 self.scan_ID_counter = 0 + self.allowed_concurrent = allowed_concurrent def running_deferred_list(self): return [script.defer_on_done for script in self.running.itervalues() if not script.externally_launched] @@ -180,7 +181,7 @@ def get_non_conflicting(self): non_conflicting = [] for running, script in self.running.iteritems(): cls_name = script.scan.script_cls.name - non_conf = config.allowed_concurrent.get(cls_name, None) + non_conf = self.allowed_concurrent.get(cls_name, None) if non_conf is not None: non_conflicting.append(set(non_conf)) if non_conflicting: diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index 13d5713..5dd48a6 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -28,6 +28,8 @@ import sys import os from six.moves import configparser +import pkgutil +import StringIO class script_class_parameters(object): @@ -62,29 +64,44 @@ class ScriptScanner(ScriptSignalsServer): name = 'ScriptScanner' + @inlineCallbacks def initServer(self): # Dictionary with experiment.name as keys and # script_class_parameters instances are the values. self.script_parameters = {} # Instance of a complicated object - self.scheduler = scheduler(ScriptSignalsServer) - self.load_scripts() + self.scheduler = scheduler(ScriptSignalsServer, self.allowed_concurrent) + yield self.load_scripts() + @inlineCallbacks def load_scripts(self): ''' loads script information from the configuration file ''' - config = sc_config.config - paths = config.paths - experiments_from_paths = self._get_all_experiments_from_basepaths(paths) - for experiment in experiments_from_paths: - config.scripts.append(experiment) - - config.scripts = list(set(config.scripts)) - - for import_path, class_name in config.scripts: + try: + self.allowed_concurrent = {} + reg = self.client.registry + reg_path = ["", "Servers", self.name] + p = reg.packet() + p.cd(path) + p.get("directories", "s") + ans = yield p.send() + paths = ans.get + experiments = self._get_all_experiments_from_basepaths(paths) + for experiment in experiments: + scripts.append((experiment[0], experiment[1])) + self.allowed_concurrent[experiment[1]] = experiment[2] + except: + print("Cannot load experiment paths from registry. " + + "Checking config file now.") + config = sc_config.config + scripts = config.scripts + self.allowed_concurrent = config.allowed_concurrent + + scripts = list(set(scripts)) + for import_path, class_name in scripts: try: __import__(import_path) module = sys.modules[import_path] @@ -110,51 +127,56 @@ def load_scripts(self): def _get_all_experiments_from_basepaths(self, paths): modules = [] - import importlib - imported_module = importlib.import_module(paths[0]) - self._list_submodules(modules, imported_module) - - modules = list(set(modules)) - experiments = [] - for module in modules: + for path in paths: try: - imported_module = importlib.import_module(module) - docstring = imported_module.__doc__ - class_name = self._get_experiment_class_name_from_docstring(docstring) - if class_name is not None: - experiments.append((module, class_name)) - except: - pass + imported_module = __import__(path) + self._list_submodules(modules, imported_module) + + modules = list(set(modules)) + experiments = [] + for module in modules: + try: + imported_module = __import__(module) + docstring = imported_module.__doc__ + class_name, allow_concurrent = self._get_experiment_info(docstring) + if class_name is not None: + if allow_concurrent is None: + allow_concurrent = [] + experiments.append((module, class_name, allow_concurrent)) + except Exception as e: + pass + except Exception as e: + print("Exception when importing " + path ". " + e) return experiments def _list_submodules(self, list, package_name): - import pkgutil - for loader, module_name, is_pkg in pkgutil.walk_packages(package_name.__path__, package_name.__name__+'.'): + for loader, module_name, is_pkg in pkgutil.walk_packages(package_name.__path__, + package_name.__name__ + '.'): try: list.append(module_name) if is_pkg: - module_name = __import__(module_name, fromlist='dummylist') + module_name = __import__(module_name) self._list_submodules(list, module_name) - except: + except Exception as e: pass - def _get_experiment_class_name_from_docstring(self, docstring): + def _get_experiment_info(self, docstring): exp_info_start_str = "### BEGIN EXPERIMENT INFO" exp_info_end_str = "### END EXPERIMENT INFO" start = docstring.find(exp_info_start_str) + len(exp_info_start_str) end = docstring.find(exp_info_end_str) if end > start: - import StringIO buf = StringIO.StringIO(docstring) cp = configparser.ConfigParser() cp.readfp(buf) - depreciated = eval(cp.get("info", "depreciated")) - if not depreciated: - return cp.get("info", "name") + if eval(cp.get("info", "load_into_script_scanner"): + name = cp.get("info", "name") + allow_concurrent = eval(cp.get("info", "allow_concurrent")) + return (name, allow_concurrent) else: - return None + return (None, None) else: - return None + return (None, None) @setting(0, "get_available_scripts", returns='*s') def get_available_scripts(self, c): From 41a8c3cd78c062c15a92eb12e261bb6b6a96bea2 Mon Sep 17 00:00:00 2001 From: fanmingyu212 Date: Thu, 29 Mar 2018 00:57:52 -0700 Subject: [PATCH 5/7] bug fixes --- lib/servers/script_scanner/scheduler.py | 4 +- lib/servers/script_scanner/script_scanner.py | 47 +++++++++++--------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/lib/servers/script_scanner/scheduler.py b/lib/servers/script_scanner/scheduler.py index b08a37f..a4d7a25 100644 --- a/lib/servers/script_scanner/scheduler.py +++ b/lib/servers/script_scanner/scheduler.py @@ -75,12 +75,12 @@ def __init__(self, scan, defer_on_done, status, priority=-1, self.externally_launched = externally_launched -class scheduler(object, allowed_concurrent): +class scheduler(object): """ TODO: proper class name """ - def __init__(self, signals): + def __init__(self, signals, allowed_concurrent): self.signals = signals # dict[identification] = running_script_instance self.running = {} diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index 5dd48a6..c8812d1 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -30,6 +30,7 @@ from six.moves import configparser import pkgutil import StringIO +import importlib class script_class_parameters(object): @@ -71,8 +72,8 @@ def initServer(self): # script_class_parameters instances are the values. self.script_parameters = {} # Instance of a complicated object - self.scheduler = scheduler(ScriptSignalsServer, self.allowed_concurrent) yield self.load_scripts() + self.scheduler = scheduler(ScriptSignalsServer, self.allowed_concurrent) @inlineCallbacks def load_scripts(self): @@ -82,11 +83,12 @@ def load_scripts(self): try: self.allowed_concurrent = {} + scripts = [] reg = self.client.registry reg_path = ["", "Servers", self.name] p = reg.packet() - p.cd(path) - p.get("directories", "s") + p.cd(reg_path) + p.get("Directories") ans = yield p.send() paths = ans.get experiments = self._get_all_experiments_from_basepaths(paths) @@ -129,24 +131,22 @@ def _get_all_experiments_from_basepaths(self, paths): modules = [] for path in paths: try: - imported_module = __import__(path) + imported_module = importlib.import_module(path) self._list_submodules(modules, imported_module) - - modules = list(set(modules)) - experiments = [] - for module in modules: - try: - imported_module = __import__(module) - docstring = imported_module.__doc__ - class_name, allow_concurrent = self._get_experiment_info(docstring) - if class_name is not None: - if allow_concurrent is None: - allow_concurrent = [] - experiments.append((module, class_name, allow_concurrent)) - except Exception as e: - pass except Exception as e: - print("Exception when importing " + path ". " + e) + print("Exception when importing " + path + ". " + e) + + modules = list(set(modules)) + experiments = [] + for module in modules: + try: + imported_module = importlib.import_module(module) + docstring = imported_module.__doc__ + class_name, allow_concurrent = self._get_experiment_info(docstring) + if class_name is not None: + experiments.append((module, class_name, allow_concurrent)) + except Exception as e: + pass return experiments def _list_submodules(self, list, package_name): @@ -155,7 +155,7 @@ def _list_submodules(self, list, package_name): try: list.append(module_name) if is_pkg: - module_name = __import__(module_name) + module_name = importlib.import_module(module_name) self._list_submodules(list, module_name) except Exception as e: pass @@ -169,9 +169,12 @@ def _get_experiment_info(self, docstring): buf = StringIO.StringIO(docstring) cp = configparser.ConfigParser() cp.readfp(buf) - if eval(cp.get("info", "load_into_script_scanner"): + if eval(cp.get("info", "load_into_scriptscanner")): name = cp.get("info", "name") - allow_concurrent = eval(cp.get("info", "allow_concurrent")) + try: + allow_concurrent = eval(cp.get("info", "allow_concurrent")) + except: + allow_concurrent = [] return (name, allow_concurrent) else: return (None, None) From 217672a1b2e908331d9e275ba2b1c5f4940ae987 Mon Sep 17 00:00:00 2001 From: fanmingyu212 Date: Mon, 9 Apr 2018 19:31:05 -0700 Subject: [PATCH 6/7] fix for refresh load experiment bug --- lib/clients/script_scanner_gui/script_scanner_gui.py | 2 +- lib/servers/script_scanner/script_scanner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/clients/script_scanner_gui/script_scanner_gui.py b/lib/clients/script_scanner_gui/script_scanner_gui.py index 52c0458..afd56c1 100644 --- a/lib/clients/script_scanner_gui/script_scanner_gui.py +++ b/lib/clients/script_scanner_gui/script_scanner_gui.py @@ -357,7 +357,7 @@ def refresh_script(self, ident): sc = yield self.cxn.get_server('ScriptScanner') pv = yield self.cxn.get_server('ParameterVault') ident = int(ident) - sc.reload_available_scripts() + yield sc.reload_available_scripts() pv.reload_parameters() sc.refresh() self.reinitialize_scriptscanner() diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index c8812d1..8f2a3eb 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -425,7 +425,7 @@ def error_finish_confirmed(self, c, script_ID, error_message): def reload_available_scripts(self, c): reload(sc_config) self.script_parameters = {} - self.load_scripts() + yield self.load_scripts() @inlineCallbacks def stopServer(self): From e565962d313d9425efe445d7b76d5bafd3bb9dfe Mon Sep 17 00:00:00 2001 From: fanmingyu212 Date: Mon, 9 Apr 2018 19:56:30 -0700 Subject: [PATCH 7/7] Auto creates the registry path. Bug fixes in loading from config. --- lib/servers/script_scanner/script_scanner.py | 28 +++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/servers/script_scanner/script_scanner.py b/lib/servers/script_scanner/script_scanner.py index 8f2a3eb..8793dfb 100644 --- a/lib/servers/script_scanner/script_scanner.py +++ b/lib/servers/script_scanner/script_scanner.py @@ -81,26 +81,34 @@ def load_scripts(self): loads script information from the configuration file ''' + self.allowed_concurrent = {} + scripts = [] + reg_path = ["", "Servers", self.name] + reg = self.client.registry + p = reg.packet() try: - self.allowed_concurrent = {} - scripts = [] - reg = self.client.registry - reg_path = ["", "Servers", self.name] - p = reg.packet() p.cd(reg_path) p.get("Directories") ans = yield p.send() paths = ans.get - experiments = self._get_all_experiments_from_basepaths(paths) - for experiment in experiments: - scripts.append((experiment[0], experiment[1])) - self.allowed_concurrent[experiment[1]] = experiment[2] + if not len(paths): + print("No path found in registry") + raise Exception() except: print("Cannot load experiment paths from registry. " + - "Checking config file now.") + "Creating the registry path and checking config file now.") + p = reg.packet() + p.cd(reg_path, True) + p.set("Directories", []) + yield p.send() config = sc_config.config scripts = config.scripts self.allowed_concurrent = config.allowed_concurrent + else: + experiments = self._get_all_experiments_from_basepaths(paths) + for experiment in experiments: + scripts.append((experiment[0], experiment[1])) + self.allowed_concurrent[experiment[1]] = experiment[2] scripts = list(set(scripts)) for import_path, class_name in scripts: