From fdf8cf4d0ea065b95195ce7ff26f25bf5fceb859 Mon Sep 17 00:00:00 2001 From: Eric Sevault Date: Wed, 10 Jul 2024 17:42:30 +0200 Subject: [PATCH 1/4] New directive import --- bin/fypp | 87 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/bin/fypp b/bin/fypp index 4207219..55aeb93 100755 --- a/bin/fypp +++ b/bin/fypp @@ -119,6 +119,9 @@ _SET_PARAM_REGEXP = re.compile( _DEL_PARAM_REGEXP = re.compile( r'^(?:[(]\s*)?[a-zA-Z_]\w*(?:\s*,\s*[a-zA-Z_]\w*)*(?:\s*[)])?$') +_IMPORT_PARAM_REGEXP = re.compile( + r'^(?:[(]\s*)?[a-zA-Z_]\w*(?:\s*,\s*[a-zA-Z_]\w*)*(?:\s*[)])?$') + _FOR_PARAM_REGEXP = re.compile( r'^(?P[a-zA-Z_]\w*(\s*,\s*[a-zA-Z_]\w*)*)\s+in\s+(?P.+)$') @@ -349,6 +352,18 @@ class Parser: self._log_event('del', span, name=name) + def handle_import(self, span, name): + '''Called when parser encounters an import directive. + + It is a dummy method and should be overridden for actual use. + + Args: + span (tuple of int): Start and end line of the directive. + name (str): Name of the variable to delete. + ''' + self._log_event('import', span, name=name) + + def handle_if(self, span, cond): '''Called when parser encounters an if directive. @@ -652,6 +667,9 @@ class Parser: elif directive == 'del': self._check_param_presence(True, 'del', param, span) self._process_del(param, span) + elif directive == 'import': + self._check_param_presence(True, 'import', param, span) + self._process_import(param, span) elif directive == 'for': self._check_param_presence(True, 'for', param, span) self._process_for(param, span) @@ -765,6 +783,14 @@ class Parser: self.handle_del(span, param) + def _process_import(self, param, span): + match = _IMPORT_PARAM_REGEXP.match(param) + if not match: + msg = "invalid module name specification '{0}'".format(param) + raise FyppFatalError(msg, self._curfile, span) + self.handle_import(span, param) + + def _process_for(self, param, span): match = _FOR_PARAM_REGEXP.match(param) if not match: @@ -1196,6 +1222,16 @@ class Builder: self._curnode.append(('del', self._curfile, span, name)) + def handle_import(self, span, name): + '''Should be called to signalize an import directive. + + Args: + span (tuple of int): Start and end line of the directive. + name (str): Name of the variable(s) to delete. + ''' + self._curnode.append(('import', self._curfile, span, name)) + + def handle_eval(self, span, expr): '''Should be called to signalize an eval directive. @@ -1424,6 +1460,8 @@ class Renderer: output.append(result) elif cmd == 'del': self._delete_variable(*node[1:4]) + elif cmd == 'import': + self._load_module(*node[1:4]) elif cmd == 'for': out, ieval, peval = self._get_iterated_content(*node[1:6]) eval_inds += _shiftinds(ieval, len(output)) @@ -1687,6 +1725,20 @@ class Renderer: return result + def _load_module(self, fname, span, name): + result = '' + try: + self._evaluator.load(name) + except Exception as exc: + msg = "exception occurred when importing module(s) '{0}'"\ + .format(name) + raise FyppFatalError(msg, fname, span) from exc + multiline = (span[0] != span[1]) + if self._linenums and not self._diverted and multiline: + result = self._linenumdir(span[1], fname) + return result + + def _add_global(self, fname, span, name): result = '' try: @@ -2059,6 +2111,18 @@ class Evaluator: raise FyppFatalError(msg) + def load(self, name): + '''Load modules. + + Args: + name (str): Name(s) of the module(s) to load. + ''' + modnames = self._get_variable_names(name) + for modname in modnames: + self._check_variable_name(modname) + self.import_module(modname) + + def addglobal(self, name): '''Define a given entity as global. @@ -2374,6 +2438,7 @@ class Processor: self._parser.handle_enddef = self._builder.handle_enddef self._parser.handle_set = self._builder.handle_set self._parser.handle_del = self._builder.handle_del + self._parser.handle_import = self._builder.handle_import self._parser.handle_global = self._builder.handle_global self._parser.handle_for = self._builder.handle_for self._parser.handle_endfor = self._builder.handle_endfor @@ -2496,18 +2561,22 @@ class Fypp: def __init__(self, options=None, evaluator_factory=Evaluator, parser_factory=Parser, builder_factory=Builder, renderer_factory=Renderer): - syspath = self._get_syspath_without_scriptdir() - self._adjust_syspath(syspath) if options is None: options = FyppOptions() + syspath = self._get_syspath_without_scriptdir() + lookuppath = [] + if options.moduledirs is not None: + lookuppath += [os.path.abspath(moddir) for moddir in options.moduledirs] + lookuppath.append(os.path.abspath('.')) + lookuppath += syspath + self._adjust_syspath(lookuppath) if inspect.signature(evaluator_factory) == inspect.signature(Evaluator): evaluator = evaluator_factory() else: raise FyppFatalError('evaluator_factory has incorrect signature') self._encoding = options.encoding if options.modules: - self._import_modules(options.modules, evaluator, syspath, - options.moduledirs) + self._import_modules(options.modules, evaluator) if options.defines: self._apply_definitions(options.defines, evaluator) if inspect.signature(parser_factory) == inspect.signature(Parser): @@ -2603,17 +2672,9 @@ class Fypp: evaluator.define(name, value) - def _import_modules(self, modules, evaluator, syspath, moduledirs): - lookuppath = [] - if moduledirs is not None: - lookuppath += [os.path.abspath(moddir) for moddir in moduledirs] - lookuppath.append(os.path.abspath('.')) - lookuppath += syspath - self._adjust_syspath(lookuppath) + def _import_modules(self, modules, evaluator): for module in modules: evaluator.import_module(module) - self._adjust_syspath(syspath) - @staticmethod def _get_syspath_without_scriptdir(): From eb5bd9edb941cc9802539a897c4985ae4912102e Mon Sep 17 00:00:00 2001 From: Eric Sevault Date: Wed, 18 Sep 2024 17:34:44 +0200 Subject: [PATCH 2/4] Some internal cosmetics for import --- bin/fypp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/fypp b/bin/fypp index 55aeb93..a825a3c 100755 --- a/bin/fypp +++ b/bin/fypp @@ -359,7 +359,7 @@ class Parser: Args: span (tuple of int): Start and end line of the directive. - name (str): Name of the variable to delete. + name (str): Name of the python module to import. ''' self._log_event('import', span, name=name) @@ -1227,7 +1227,7 @@ class Builder: Args: span (tuple of int): Start and end line of the directive. - name (str): Name of the variable(s) to delete. + name (str): Name of the module to import. ''' self._curnode.append(('import', self._curfile, span, name)) @@ -1728,7 +1728,7 @@ class Renderer: def _load_module(self, fname, span, name): result = '' try: - self._evaluator.load(name) + self._evaluator.loadmodules(name) except Exception as exc: msg = "exception occurred when importing module(s) '{0}'"\ .format(name) @@ -2111,8 +2111,8 @@ class Evaluator: raise FyppFatalError(msg) - def load(self, name): - '''Load modules. + def loadmodules(self, name): + '''Load modules in current space name. Args: name (str): Name(s) of the module(s) to load. @@ -2672,10 +2672,12 @@ class Fypp: evaluator.define(name, value) - def _import_modules(self, modules, evaluator): + @staticmethod + def _import_modules(modules, evaluator): for module in modules: evaluator.import_module(module) + @staticmethod def _get_syspath_without_scriptdir(): '''Remove the folder of the fypp binary from the search path''' From 2b164c28d318faae6629620aa887d13954a3ec09 Mon Sep 17 00:00:00 2001 From: Eric Sevault Date: Wed, 9 Oct 2024 10:55:31 +0200 Subject: [PATCH 3/4] New import regex and alias facility --- bin/fypp | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/bin/fypp b/bin/fypp index a825a3c..97dce68 100755 --- a/bin/fypp +++ b/bin/fypp @@ -120,7 +120,7 @@ _DEL_PARAM_REGEXP = re.compile( r'^(?:[(]\s*)?[a-zA-Z_]\w*(?:\s*,\s*[a-zA-Z_]\w*)*(?:\s*[)])?$') _IMPORT_PARAM_REGEXP = re.compile( - r'^(?:[(]\s*)?[a-zA-Z_]\w*(?:\s*,\s*[a-zA-Z_]\w*)*(?:\s*[)])?$') + r'^\s*(?P[a-zA-Z]\w*(?:\.[a-zA-Z]\w*)*)(?:\s+as\s+(?P[a-zA-Z]\w*))?\s*$') _FOR_PARAM_REGEXP = re.compile( r'^(?P[a-zA-Z_]\w*(\s*,\s*[a-zA-Z_]\w*)*)\s+in\s+(?P.+)$') @@ -788,7 +788,7 @@ class Parser: if not match: msg = "invalid module name specification '{0}'".format(param) raise FyppFatalError(msg, self._curfile, span) - self.handle_import(span, param) + self.handle_import(span, match.group('modname'), match.group('modalias')) def _process_for(self, param, span): @@ -1222,14 +1222,15 @@ class Builder: self._curnode.append(('del', self._curfile, span, name)) - def handle_import(self, span, name): + def handle_import(self, span, name, alias): '''Should be called to signalize an import directive. Args: span (tuple of int): Start and end line of the directive. name (str): Name of the module to import. + alias (str): Local name to be used. ''' - self._curnode.append(('import', self._curfile, span, name)) + self._curnode.append(('import', self._curfile, span, name, alias)) def handle_eval(self, span, expr): @@ -1461,7 +1462,7 @@ class Renderer: elif cmd == 'del': self._delete_variable(*node[1:4]) elif cmd == 'import': - self._load_module(*node[1:4]) + self._load_module(*node[1:5]) elif cmd == 'for': out, ieval, peval = self._get_iterated_content(*node[1:6]) eval_inds += _shiftinds(ieval, len(output)) @@ -1725,10 +1726,10 @@ class Renderer: return result - def _load_module(self, fname, span, name): + def _load_module(self, fname, span, name, alias): result = '' try: - self._evaluator.loadmodules(name) + self._evaluator.loadmodule(name, alias) except Exception as exc: msg = "exception occurred when importing module(s) '{0}'"\ .format(name) @@ -2026,7 +2027,7 @@ class Evaluator: return result - def import_module(self, module): + def import_module(self, module, alias=None): '''Import a module into the evaluator. Note: Import only trustworthy modules! Module imports are global, @@ -2035,15 +2036,19 @@ class Evaluator: Args: module (str): Python module to import. + alias (str): Local alias name for the module. Raises: FyppFatalError: If module could not be imported. ''' - rootmod = module.split('.', 1)[0] + rootmod, *xtramod = module.split('.', 1) + if alias is None: + alias = rootmod + xtramod = None try: - imported = __import__(module, self._scope) - self.define(rootmod, imported) + imported = __import__(module, globals=self._scope, fromlist=xtramod) + self.define(alias, imported) except Exception as exc: msg = "failed to import module '{0}'".format(module) raise FyppFatalError(msg) from exc @@ -2111,16 +2116,15 @@ class Evaluator: raise FyppFatalError(msg) - def loadmodules(self, name): + def loadmodule(self, modname, alias=None): '''Load modules in current space name. Args: - name (str): Name(s) of the module(s) to load. + modname (str): Name(s) of the module(s) to load. + alias (str): Local name for the module (optional). ''' - modnames = self._get_variable_names(name) - for modname in modnames: - self._check_variable_name(modname) - self.import_module(modname) + self._check_variable_name(modname) + self.import_module(modname, alias) def addglobal(self, name): From 992024ed66a88dd2445bd442dbfa485d3935692c Mon Sep 17 00:00:00 2001 From: Eric Sevault Date: Wed, 9 Oct 2024 15:53:11 +0200 Subject: [PATCH 4/4] Add check_module_name --- bin/fypp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/fypp b/bin/fypp index 97dce68..07d012e 100755 --- a/bin/fypp +++ b/bin/fypp @@ -2123,7 +2123,10 @@ class Evaluator: modname (str): Name(s) of the module(s) to load. alias (str): Local name for the module (optional). ''' - self._check_variable_name(modname) + if alias is None: + self._check_module_name(modname) + else: + self._check_module_name(alias) self.import_module(modname, alias) @@ -2271,6 +2274,16 @@ class Evaluator: .format(varname) raise FyppFatalError(msg, None, None) + @staticmethod + def _check_module_name(modname): + if modname.startswith(_RESERVED_PREFIX): + msg = "Local module name '{0}' starts with reserved prefix '{1}'"\ + .format(modname, _RESERVED_PREFIX) + raise FyppFatalError(msg, None, None) + if modname in _RESERVED_NAMES: + msg = "Name '{0}' is reserved and can not be redefined as a local module name"\ + .format(modname) + raise FyppFatalError(msg, None, None) def _func_defined(self, var): defined = var in self._scope