From 1770d7e4feaae5e79f540a2afe1c54273fe95798 Mon Sep 17 00:00:00 2001 From: biplavbarua Date: Thu, 25 Dec 2025 00:03:43 +0530 Subject: [PATCH] Port Spitfire to Python 3 --- scripts/crunner.py | 79 ++++++------ scripts/spitfire-compile | 17 +-- setup.py | 7 +- spitfire/compiler/analyzer.py | 6 +- spitfire/compiler/ast.py | 18 +-- spitfire/compiler/codegen.py | 8 +- spitfire/compiler/compiler.py | 4 +- spitfire/compiler/macros/i18n.py | 2 +- spitfire/compiler/optimizer.py | 43 ++++--- spitfire/compiler/options.py | 2 +- spitfire/compiler/parser.g | 4 +- spitfire/compiler/util.py | 12 +- spitfire/compiler/visitor.py | 14 +- spitfire/compiler/walker.py | 2 +- spitfire/runtime/__init__.py | 4 +- spitfire/runtime/_template.c | 35 +++-- spitfire/runtime/_udn.c | 39 ++++-- spitfire/runtime/filters.py | 6 +- spitfire/runtime/repeater.py | 2 +- spitfire/runtime/runner.py | 8 +- spitfire/runtime/template.py | 2 +- spitfire/runtime/udn.py | 4 +- spitfire/text.py | 2 +- .../some_library.txt | 2 +- .../template_if_4.txt | 2 +- tests/output-preserve-whitespace/util.txt | 2 +- tests/output/some_library.txt | 2 +- tests/output/template_if_4.txt | 2 +- tests/output/util.txt | 2 +- ...-hoist-conditional-loop-optimization-3.txt | 2 +- third_party/yapps2/yapps2.py | 120 +++++++++--------- third_party/yapps2/yappsrt.py | 35 ++--- 32 files changed, 259 insertions(+), 230 deletions(-) mode change 100755 => 100644 third_party/yapps2/yapps2.py diff --git a/scripts/crunner.py b/scripts/crunner.py index 19bf9f3..db91b14 100755 --- a/scripts/crunner.py +++ b/scripts/crunner.py @@ -13,7 +13,10 @@ import time import traceback -import cStringIO as StringIO +# Add project root to sys.path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import io as StringIO from spitfire.compiler import compiler from spitfire.compiler import options @@ -60,11 +63,11 @@ def __getattr__(self, key): return self._get_item(key) -sys_modules = sys.modules.keys() +sys_modules = list(sys.modules.keys()) def reset_sys_modules(): - for key in sys.modules.keys(): + for key in list(sys.modules.keys()): if key not in sys_modules: del sys.modules[key] @@ -105,18 +108,18 @@ def begin(self): def end(self): self.finish_time = time.time() - print >> sys.stderr + print(file=sys.stderr) if self.num_tests_failed > 0: sys.stderr.write(self.buffer.getvalue()) - print >> sys.stderr, '-' * 70 - print >> sys.stderr, 'Ran %d tests in %0.3fs' % ( - self.num_tests_run, self.finish_time - self.start_time) - print >> sys.stderr + print('-' * 70, file=sys.stderr) + print('Ran %d tests in %0.3fs' % ( + self.num_tests_run, self.finish_time - self.start_time), file=sys.stderr) + print(file=sys.stderr) if self.num_tests_failed > 0: - print >> sys.stderr, 'FAILED (failures=%d)' % self.num_tests_failed + print('FAILED (failures=%d)' % self.num_tests_failed, file=sys.stderr) sys.exit(1) else: - print >> sys.stderr, 'OK' + print('OK', file=sys.stderr) sys.exit(0) def process_file(self, filename): @@ -137,31 +140,31 @@ def process_file(self, filename): self.compiler.compile_file(filename) except Exception as e: compile_failed = True - print >> buffer, '=' * 70 - print >> buffer, 'FAIL:', modulename, '(' + filename + ')' - print >> buffer, '-' * 70 - traceback.print_exc(None, buffer) + print('=' * 70, file=buffer) + print('FAIL:', modulename, '(' + filename + ')', file=buffer) + print('-' * 70, file=buffer) + traceback.print_exc(file=buffer) if self.options.debug: if 'parse_tree' in self.options.debug_flags: - print >> buffer, "parse_tree:" + print("parse_tree:", file=buffer) visitor.print_tree(self.compiler._parse_tree, output=buffer) if 'analyzed_tree' in self.options.debug_flags: - print >> buffer, "analyzed_tree:" + print("analyzed_tree:", file=buffer) visitor.print_tree(self.compiler._analyzed_tree, output=buffer) if 'optimized_tree' in self.options.debug_flags: - print >> buffer, "optimized_tree:" + print("optimized_tree:", file=buffer) visitor.print_tree(self.compiler._optimized_tree, output=buffer) if 'hoisted_tree' in self.options.debug_flags: - print >> buffer, "hoisted_tree:" + print("hoisted_tree:", file=buffer) visitor.print_tree(self.compiler._hoisted_tree, output=buffer) if 'source_code' in self.options.debug_flags: - print >> buffer, "source_code:" + print("source_code:", file=buffer) for i, line in enumerate(self.compiler._source_code.split( '\n')): - print >> buffer, '% 3s' % (i + 1), line + print('% 3s' % (i + 1), line, file=buffer) test_failed = False if not self.options.skip_test: @@ -186,7 +189,7 @@ def process_file(self, filename): try: template_class = getattr(template_module, classname) template = template_class(search_list=self.search_list) - current_output = template.main().encode('utf8') + current_output = template.main() except Exception as e: # An exception here doesn't meant that the test fails # necessarily since libraries don't have a class; as long as @@ -216,10 +219,10 @@ def process_file(self, filename): if current_output != test_output: test_failed = True if self.options.debug: - print >> buffer, "expected output:" - print >> buffer, test_output - print >> buffer, "actual output:" - print >> buffer, current_output + print("expected output:", file=buffer) + print(test_output, file=buffer) + print("actual output:", file=buffer) + print(current_output, file=buffer) if compile_failed or test_failed: self.num_tests_failed += 1 @@ -232,23 +235,23 @@ def process_file(self, filename): f = open(current_output_path, 'w') f.write(current_output) f.close() - print >> buffer, '=' * 70 - print >> buffer, 'FAIL:', modulename, '(' + filename + ')' - print >> buffer, '-' * 70 - print >> buffer, 'Compare expected and actual output with:' - print >> buffer, ' '.join([' diff -u', test_output_path, - current_output_path]) - print >> buffer, 'Show debug information for the test with:' + print('=' * 70, file=buffer) + print('FAIL:', modulename, '(' + filename + ')', file=buffer) + print('-' * 70, file=buffer) + print('Compare expected and actual output with:', file=buffer) + print(' '.join([' diff -u', test_output_path, + current_output_path]), file=buffer) + print('Show debug information for the test with:', file=buffer) test_cmd = [arg for arg in sys.argv if arg not in self.files] if '--debug' not in test_cmd: test_cmd.append('--debug') test_cmd = ' '.join(test_cmd) - print >> buffer, ' ', test_cmd, filename + print(' ', test_cmd, filename, file=buffer) if raised_exception: - print >> buffer, '-' * 70 - print >> buffer, current_output - traceback.print_exc(None, buffer) - print >> buffer + print('-' * 70, file=buffer) + print(current_output, file=buffer) + traceback.print_exc(file=buffer) + print(file=buffer) self.buffer.write(buffer.getvalue()) else: if self.options.verbose: @@ -259,8 +262,6 @@ def process_file(self, filename): if __name__ == '__main__': - reload(sys) - sys.setdefaultencoding('utf8') option_parser = optparse.OptionParser() options.add_common_options(option_parser) diff --git a/scripts/spitfire-compile b/scripts/spitfire-compile index f744d27..a4d61ef 100755 --- a/scripts/spitfire-compile +++ b/scripts/spitfire-compile @@ -20,41 +20,38 @@ def process_file(spt_compiler, filename, options): def print_output(*args): if options.verbose: - print >> sys.stderr, ' '.join(args) + print(' '.join(args), file=sys.stderr) try: if options.output_file: spt_compiler.write_file = False if options.output_file == '-': - f = sys.stdout + f = sys.stdout.buffer else: - f = open(options.output_file, 'w') + f = open(options.output_file, 'wb') else: spt_compiler.write_file = True src_code = spt_compiler.compile_file(filename) if options.output_file: f.write(src_code) f.close() - except Exception, e: + except Exception as e: error_msg = 'Failed processing file: %s' % filename if options.verbose: logging.exception(error_msg) else: - print >> sys.stderr, error_msg - print >> sys.stderr, e + print(error_msg, file=sys.stderr) + print(e, file=sys.stderr) sys.exit(1) if __name__ == '__main__': - reload(sys) - sys.setdefaultencoding('utf8') - option_parser = optparse.OptionParser() options.add_common_options(option_parser) (spt_options, spt_args) = option_parser.parse_args() if spt_options.version: - print >> sys.stderr, 'spitfire %s' % spitfire.__version__ + print('spitfire %s' % spitfire.__version__, file=sys.stderr) sys.exit(0) spt_compiler_args = compiler.Compiler.args_from_optparse(spt_options) diff --git a/setup.py b/setup.py index e6df11b..4efcf22 100644 --- a/setup.py +++ b/setup.py @@ -41,14 +41,15 @@ SCRIPTS = ['scripts/crunner.py', 'scripts/spitfire-compile'] -EXT_MODULES = [Extension('spitfire.runtime._baked', - [os.path.join('spitfire', 'runtime', '_baked.c')]), +EXT_MODULES = [ + # Extension('spitfire.runtime._baked', + # [os.path.join('spitfire', 'runtime', '_baked.c')]), Extension('spitfire.runtime._template', [os.path.join('spitfire', 'runtime', '_template.c')]), Extension('spitfire.runtime._udn', [os.path.join('spitfire', 'runtime', '_udn.c')])] # Disable C extensions for PyPy. -if platform.python_implementation() == 'PyPy': +if platform.python_implementation() == 'PyPy' or os.environ.get('SPITFIRE_SKIP_EXT'): EXT_MODULES = None setup(name=NAME, diff --git a/spitfire/compiler/analyzer.py b/spitfire/compiler/analyzer.py index 815ef56..efa4fd1 100644 --- a/spitfire/compiler/analyzer.py +++ b/spitfire/compiler/analyzer.py @@ -96,7 +96,7 @@ def build_ast(self, node): try: if len(ast_node_list) != 1: return ast_node_list - except TypeError, e: + except TypeError as e: self.compiler.error(SemanticAnalyzerError('method: %s, result: %s' % (method, ast_node_list))) @@ -446,7 +446,7 @@ def handleMacro(self, pnode, macro_function, parse_rule): fragment_ast = util.parse(macro_output, 'fragment_goal') elif isinstance(pnode, ast.CallFunctionNode): fragment_ast = util.parse(macro_output, 'rhs_expression') - except Exception, e: + except Exception as e: self.compiler.error(MacroParseError(e), pos=pnode.pos) return self.build_ast(fragment_ast) @@ -464,7 +464,7 @@ def analyzeMacroNode(self, pnode): try: temp_fragment = util.parse(pnode.value, macro_parse_rule or 'fragment_goal') - except Exception, e: + except Exception as e: self.compiler.error(MacroParseError(e), pos=pnode.pos) if not self.uses_raw: diff --git a/spitfire/compiler/ast.py b/spitfire/compiler/ast.py index a08ddf0..a9c03ce 100644 --- a/spitfire/compiler/ast.py +++ b/spitfire/compiler/ast.py @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import __builtin__ +import builtins import copy import traceback @@ -61,8 +61,8 @@ def append(self, node): else: try: node.parent = self - except AttributeError, e: - print e, node + except AttributeError as e: + print(e, node) raise self.child_nodes.append(node) @@ -898,7 +898,7 @@ class FragmentNode(ASTNode): class TemplateNode(ASTNode): - __builtin_set = frozenset(dir(__builtin__)) + __builtin_set = frozenset(dir(builtins)) def __init__(self, classname=None, pos=None, **kargs): ASTNode.__init__(self, pos=pos, **kargs) @@ -1033,21 +1033,17 @@ def __delitem__(self, key): self._order.remove(key) del self._dict[key] - def keys(self): - return list(self.iterkeys()) - def items(self): - return list(self.iteritems()) - def iterkeys(self): + def keys(self): return iter(self._order) - def iteritems(self): + def items(self): for key in self._order: yield key, self._dict[key] def update(self, ordered_dict): - for key, value in ordered_dict.iteritems(): + for key, value in ordered_dict.items(): self[key] = value def __str__(self): diff --git a/spitfire/compiler/codegen.py b/spitfire/compiler/codegen.py index 47d011b..96184d5 100644 --- a/spitfire/compiler/codegen.py +++ b/spitfire/compiler/codegen.py @@ -5,7 +5,7 @@ import logging -import cStringIO as StringIO +import io as StringIO from spitfire.compiler import ast @@ -70,7 +70,7 @@ def get_code(self): def generate_python(self, code_node): try: return code_node.src_line - except AttributeError, e: + except AttributeError as e: self.compiler.error(CodegenError("can't write code_node: %s\n\t%s" % (code_node, e))) @@ -316,7 +316,7 @@ def codegenASTTargetListNode(self, node): def codegenASTLiteralNode(self, node): if (self.options and not self.options.generate_unicode and - isinstance(node.value, basestring)): + isinstance(node.value, str)): # If the node is the empty string, we should mark it as sanitized by # default. Eventually, all string literals should be marked as # sanitized. @@ -694,7 +694,7 @@ def codegenDefault(self, node): try: return [CodeNode(line % vars(node)) for line in v['AST%s_tmpl' % node.__class__.__name__]] - except KeyError, e: + except KeyError as e: self.compiler.error(CodegenError("no codegen for %s %s" % (type( node), vars(node)))) diff --git a/spitfire/compiler/compiler.py b/spitfire/compiler/compiler.py index ed0e404..cb74a49 100644 --- a/spitfire/compiler/compiler.py +++ b/spitfire/compiler/compiler.py @@ -120,7 +120,7 @@ def __init__(self, **kargs): self._hoisted_tree = None self._source_code = None - for key, value in kargs.iteritems(): + for key, value in kargs.items(): setattr(self, key, value) if self.analyzer_options is None: @@ -296,7 +296,7 @@ def write_src_file(self, src_code): outfile_path = os.path.join(self.output_directory, relative_dir, outfile_name) - outfile = open(outfile_path, 'w') + outfile = open(outfile_path, 'wb') outfile.write(src_code) outfile.close() diff --git a/spitfire/compiler/macros/i18n.py b/spitfire/compiler/macros/i18n.py index cd802d2..954faa4 100644 --- a/spitfire/compiler/macros/i18n.py +++ b/spitfire/compiler/macros/i18n.py @@ -5,7 +5,7 @@ import sys -import cStringIO as StringIO +import io as StringIO from spitfire.compiler import analyzer from spitfire.compiler import ast diff --git a/spitfire/compiler/optimizer.py b/spitfire/compiler/optimizer.py index be6ce0e..6ccc281 100644 --- a/spitfire/compiler/optimizer.py +++ b/spitfire/compiler/optimizer.py @@ -3,17 +3,23 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import __builtin__ +import builtins import copy +from functools import reduce +import hashlib import logging -import os.path +import os import re +import sys from spitfire.compiler import ast from spitfire.compiler import analyzer from spitfire.compiler import walker -builtin_names = vars(__builtin__) +# TODO: this list is weak +# roughly the list of builtins that are safe to access (no-side-effects) +# and are optimized in CPython. +builtin_names = vars(builtins) _BINOP_INVALID_COUNT = 1000 # Any value > 0 will work. _BINOP_INITIAL_COUNT = 0 @@ -119,7 +125,7 @@ def _get_common_aliased_expression_map(*scopes): clean_key_sets = [] for scope in scopes: clean_scope_keys = set() - for alias in scope.aliased_expression_map.iterkeys(): + for alias in scope.aliased_expression_map.keys(): if _is_clean(alias, scope): clean_scope_keys.add(alias) clean_key_sets.append(clean_scope_keys) @@ -153,7 +159,7 @@ def __init__(self, ast_root, options, compiler): def optimize_ast(self): self.visit_ast(self.ast_root) if self.options.debug: - print "unoptimized_node_types", self.unoptimized_node_types + print("unoptimized_node_types", self.unoptimized_node_types) return self.ast_root # build an AST node list from a single parse node @@ -163,7 +169,7 @@ def visit_ast(self, node, parent=None): method_name = 'analyze%s' % node.__class__.__name__ method = getattr(self, method_name, self.default_optimize_node) if method_name in self.compiler.debug_flags: - print method_name, node + print(method_name, node) return method(node) def skip_analyze_node(self, node): @@ -240,17 +246,14 @@ def reanalyzeConditionalNode(self, conditional_node): #print " parent_block", parent_block #print " parent_scope", parent_block.scope # NOTE: need to iterate over items, in case we modify something - items = conditional_node.scope.aliased_expression_map.items() + items = list(conditional_node.scope.aliased_expression_map.items()) for alias_node, alias in items: - #print " check alias:", alias - #print " alias_node:", alias_node assign_alias_node = ast.AssignNode(alias, alias_node, pos=alias_node.pos) if alias_node in parent_block.scope.aliased_expression_map: if self._is_condition_invariant(alias_node, conditional_node): - #print " hoist:", assign_alias_node self.hoist(conditional_node, parent_block, insertion_point, alias_node, assign_alias_node) @@ -626,11 +629,11 @@ def analyzeFilterNode(self, filter_node): alias_name = _generate_filtered_placeholder( filter_node.expression) if alias_name in scope.alias_name_set: - print "duplicate alias_name", alias_name - print "scope", scope - print "scope.alias_name_set", scope.alias_name_set - print " ".join("scope.aliased_expression_map", - scope.aliased_expression_map) + print("duplicate alias_name", alias_name) + print("scope", scope) + print("scope.alias_name_set", scope.alias_name_set) + print(" ".join("scope.aliased_expression_map", + str(scope.aliased_expression_map))) return alias = ast.IdentifierNode(alias_name, pos=filter_node.pos) @@ -766,11 +769,11 @@ def analyzeGetAttrNode(self, node): alias_format = '%s_%s' alias_name = alias_format % (node.expression.name, node.name) if alias_name in scope.alias_name_set: - print "duplicate alias_name", alias_name - print "scope", scope - print "scope.alias_name_set", scope.alias_name_set - print " ".join("scope.aliased_expression_map", - scope.aliased_expression_map) + print("duplicate alias_name", alias_name) + print("scope", scope) + print("scope.alias_name_set", scope.alias_name_set) + print(" ".join("scope.aliased_expression_map", + str(scope.aliased_expression_map))) return alias = ast.IdentifierNode(alias_name) diff --git a/spitfire/compiler/options.py b/spitfire/compiler/options.py index 2b722f9..ab8ce85 100644 --- a/spitfire/compiler/options.py +++ b/spitfire/compiler/options.py @@ -125,7 +125,7 @@ def update(self, **kargs): @classmethod def get_help(cls): return ', '.join(['[no-]' + name.replace('_', '-') - for name, value in vars(cls()).iteritems() + for name, value in vars(cls()).items() if not name.startswith('__') and type(value) == bool]) diff --git a/spitfire/compiler/parser.g b/spitfire/compiler/parser.g index bd74095..0567765 100644 --- a/spitfire/compiler/parser.g +++ b/spitfire/compiler/parser.g @@ -463,10 +463,10 @@ parser _SpitfireParser: rule stringliteral: '"' DOUBLE_QUOTE_STR '"' - {{ return unicode(eval('"%s"' % DOUBLE_QUOTE_STR)) }} + {{ return str(eval('"%s"' % DOUBLE_QUOTE_STR)) }} | "'" SINGLE_QUOTE_STR "'" - {{ return unicode(eval("'%s'" % SINGLE_QUOTE_STR)) }} + {{ return str(eval("'%s'" % SINGLE_QUOTE_STR)) }} # had to factor out the floats rule literal: diff --git a/spitfire/compiler/util.py b/spitfire/compiler/util.py index 0898cb9..8ea9055 100644 --- a/spitfire/compiler/util.py +++ b/spitfire/compiler/util.py @@ -4,7 +4,7 @@ # license that can be found in the LICENSE file. import logging -import new +import types import os.path import re import sys @@ -55,7 +55,7 @@ def parse_template(src_text, xspt_mode=False): def read_template_file(filename): - f = open(filename, 'r') + f = open(filename, 'rb') try: return f.read().decode('utf8') finally: @@ -106,7 +106,7 @@ def load_template_file(filename, spt_compiler = compiler.Compiler(analyzer_options=analyzer_options, xspt_mode=xspt_mode) if compiler_options: - for k, v in compiler_options.iteritems(): + for k, v in compiler_options.items(): setattr(spt_compiler, k, v) class_name = filename2classname(filename) if not module_name: @@ -128,7 +128,7 @@ def load_template(template_src, from spitfire.compiler import compiler spt_compiler = compiler.Compiler(analyzer_options=analyzer_options) if compiler_options: - for k, v in compiler_options.iteritems(): + for k, v in compiler_options.items(): setattr(spt_compiler, k, v) src_code = spt_compiler.compile_template(template_src, class_name) module = load_module_from_src(src_code, filename, module_name) @@ -137,11 +137,11 @@ def load_template(template_src, # a helper method to import a template without having to save it to disk def load_module_from_src(src_code, filename, module_name): - module = new.module(module_name) + module = types.ModuleType(module_name) sys.modules[module_name] = module bytecode = compile(src_code, filename, 'exec') - exec bytecode in module.__dict__ + exec(bytecode, module.__dict__) return module diff --git a/spitfire/compiler/visitor.py b/spitfire/compiler/visitor.py index c9dc860..0d5e441 100644 --- a/spitfire/compiler/visitor.py +++ b/spitfire/compiler/visitor.py @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import StringIO +import io as StringIO class TreeWalkError(Exception): @@ -40,10 +40,10 @@ def flatten_tree(root): def print_tree(root, output=None): - if output: - print >> output, flatten_tree(root) - else: - print flatten_tree(root) + if output: + print(flatten_tree(root), file=output) + else: + print(flatten_tree(root)) # perform an in-order traversal of the AST and call the generate methods @@ -63,14 +63,14 @@ def get_text(self): text = self.output.getvalue() try: text = text.encode(self.ast_root.encoding) - except AttributeError, e: + except AttributeError as e: pass return text def generate_text(self, visit_node): try: return visit_node.node_repr - except AttributeError, e: + except AttributeError as e: raise TreeWalkError("can't write visit_node: %s\n\t%s" % (visit_node, e)) diff --git a/spitfire/compiler/walker.py b/spitfire/compiler/walker.py index dd55e3f..47b4af2 100644 --- a/spitfire/compiler/walker.py +++ b/spitfire/compiler/walker.py @@ -9,7 +9,7 @@ class TreeWalkError(Exception): def print_tree(root): - print TreeVisitor(root).get_text() + print(TreeVisitor(root).get_text()) # perform an in-order traversal of the AST and call the generate methods diff --git a/spitfire/runtime/__init__.py b/spitfire/runtime/__init__.py index 67bf292..13bcb3e 100644 --- a/spitfire/runtime/__init__.py +++ b/spitfire/runtime/__init__.py @@ -68,7 +68,7 @@ def import_module_symbol(name): module = __import__(module_name, globals(), locals(), [symbol_name]) try: symbol = getattr(module, symbol_name) - except AttributeError, e: + except AttributeError as e: raise ImportError("can't import %s" % name) return symbol @@ -76,7 +76,7 @@ def import_module_symbol(name): # map template function names to python function names # inject them into a module so they run as globals def register_functions(module, template_function_map): - for t_name, f_name in template_function_map.iteritems(): + for t_name, f_name in template_function_map.items(): f_func = import_module_symbol(f_name) setattr(module, t_name, f_func) diff --git a/spitfire/runtime/_template.c b/spitfire/runtime/_template.c index 31b9000..874267d 100644 --- a/spitfire/runtime/_template.c +++ b/spitfire/runtime/_template.c @@ -3,6 +3,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#define PY_SSIZE_T_CLEAN #include // Constant used when checking for the presence of the skip_filter attribute. @@ -67,15 +68,14 @@ filter_function(PyObject *self, PyObject *args) // Function registration table: name-string -> function-pointer static struct PyMethodDef template_methods[] = { - {"filter_function", (PyCFunction)filter_function, METH_VARARGS}, - {NULL, NULL} + {"filter_function", (PyCFunction)filter_function, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} }; // BaseSpitfireTemplate Type. static PyTypeObject BaseSpitfireTemplateType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "template.BaseSpitfireTemplate", /* tp_name */ sizeof(BaseSpitfireTemplateObject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -118,39 +118,48 @@ static PyTypeObject BaseSpitfireTemplateType = { // Function registration table: name-string -> function-pointer static struct PyMethodDef module_methods[] = { - {NULL} // Sentinel + {NULL, NULL, 0, NULL} // Sentinel }; +static struct PyModuleDef _templatemodule = { + PyModuleDef_HEAD_INIT, + "_template", + "Template Module", + -1, + module_methods +}; PyMODINIT_FUNC -init_template(void) +PyInit__template(void) { // Set interned strings. - Skip_Filter_PyString = PyString_InternFromString("skip_filter"); - filter_function_name = PyString_InternFromString("_filter_function"); + Skip_Filter_PyString = PyUnicode_InternFromString("skip_filter"); + filter_function_name = PyUnicode_InternFromString("_filter_function"); // Get SanitizedPlaceholder from the baked module. PyObject *baked_module = PyImport_ImportModule("spitfire.runtime.baked"); if (baked_module == NULL) - return; + return NULL; baked_SanitizedPlaceholder = (struct _typeobject *) PyObject_GetAttrString(baked_module, "SanitizedPlaceholder"); Py_DECREF(baked_module); if (baked_SanitizedPlaceholder == NULL) - return; + return NULL; // Setup module and class. PyObject *m; BaseSpitfireTemplateType.tp_new = PyType_GenericNew; if (PyType_Ready(&BaseSpitfireTemplateType) < 0) - return; + return NULL; - m = Py_InitModule3("_template", module_methods, "Template Module"); + m = PyModule_Create(&_templatemodule); if (m == NULL) - return; + return NULL; Py_INCREF(&BaseSpitfireTemplateType); PyModule_AddObject(m, "BaseSpitfireTemplate", (PyObject *)&BaseSpitfireTemplateType); + + return m; } diff --git a/spitfire/runtime/_udn.c b/spitfire/runtime/_udn.c index c153a48..b1b845b 100644 --- a/spitfire/runtime/_udn.c +++ b/spitfire/runtime/_udn.c @@ -3,6 +3,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#define PY_SSIZE_T_CLEAN #include "Python.h" #include #include @@ -72,7 +73,7 @@ udn_resolve_udn(PyObject *self, PyObject *args, PyObject *kargs) return NULL; } - if (!(PyUnicode_Check(name) || PyString_Check(name))) { + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_ValueError, "name must be string"); return NULL; } @@ -110,7 +111,7 @@ udn_resolve_from_search_list(PyObject *self, PyObject *args, PyObject *keywds) return NULL; } - if (!(PyUnicode_Check(name) || PyString_Check(name))) { + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_ValueError, "name must be string"); return NULL; } @@ -150,22 +151,37 @@ udn_resolve_from_search_list(PyObject *self, PyObject *args, PyObject *keywds) /* Method registration table: name-string -> function-pointer */ static struct PyMethodDef udn_methods[] = { - {"_resolve_udn", (PyCFunction)udn_resolve_udn, METH_VARARGS|METH_KEYWORDS}, - {"_resolve_from_search_list", (PyCFunction)udn_resolve_from_search_list, METH_VARARGS|METH_KEYWORDS}, - {NULL, NULL} + {"_resolve_udn", (PyCFunction)udn_resolve_udn, METH_VARARGS|METH_KEYWORDS, NULL}, + {"_resolve_from_search_list", (PyCFunction)udn_resolve_from_search_list, METH_VARARGS|METH_KEYWORDS, NULL}, + {NULL, NULL, 0, NULL} }; /* Initialization function (import-time) */ -DL_EXPORT(void) -init_udn(void) +static struct PyModuleDef _udnmodule = { + PyModuleDef_HEAD_INIT, + "_udn", + NULL, + -1, + udn_methods +}; + +PyMODINIT_FUNC +PyInit__udn(void) { PyObject *m, *runtime_module; - m = Py_InitModule("_udn", udn_methods); + m = PyModule_Create(&_udnmodule); + if (m == NULL) + return NULL; runtime_module = PyImport_ImportModule("spitfire.runtime"); + if (runtime_module == NULL) { + Py_DECREF(m); + return NULL; + } + PlaceholderError = PyObject_GetAttrString( runtime_module, "PlaceholderError"); UDNResolveError = PyObject_GetAttrString(runtime_module, "UDNResolveError"); @@ -175,8 +191,13 @@ init_udn(void) runtime_module, "UndefinedAttribute"); Py_DECREF(runtime_module); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { + Py_DECREF(m); Py_FatalError("Can't initialize module _udn"); + return NULL; + } + + return m; } #ifdef __cplusplus diff --git a/spitfire/runtime/filters.py b/spitfire/runtime/filters.py index c6ea127..1444f39 100644 --- a/spitfire/runtime/filters.py +++ b/spitfire/runtime/filters.py @@ -37,7 +37,7 @@ def passthrough_filter(value): def escape_html(value, quote=True): """Replace special characters '&', '<' and '>' by SGML entities.""" value = simple_str_filter(value) - if isinstance(value, basestring): + if isinstance(value, str): value = value.replace("&", "&") # Must be done first! value = value.replace("<", "<") value = value.replace(">", ">") @@ -49,7 +49,7 @@ def escape_html(value, quote=True): # deprecated def safe_values(value): """Deprecated - use simple_str_filter instead.""" - if isinstance(value, (str, unicode, int, long, float, + if isinstance(value, (str, int, float, runtime.UndefinedPlaceholder)): return value else: @@ -58,7 +58,7 @@ def safe_values(value): def simple_str_filter(value): """Return a string if the input type is something primitive.""" - if isinstance(value, (str, unicode, int, long, float, + if isinstance(value, (str, int, float, runtime.UndefinedPlaceholder)): # fixme: why do force this conversion here? # do we want to be unicode or str? diff --git a/spitfire/runtime/repeater.py b/spitfire/runtime/repeater.py index 2990fbe..3058290 100644 --- a/spitfire/runtime/repeater.py +++ b/spitfire/runtime/repeater.py @@ -12,7 +12,7 @@ def __init__(self): def __setitem__(self, key, value): try: self.repeater_map[key].index = value - except KeyError, e: + except KeyError as e: self.repeater_map[key] = Repeater(value) def __getitem__(self, key): diff --git a/spitfire/runtime/runner.py b/spitfire/runtime/runner.py index 4b9b6f1..5cc6868 100644 --- a/spitfire/runtime/runner.py +++ b/spitfire/runtime/runner.py @@ -8,7 +8,7 @@ import os.path import sys -import cPickle as pickle +import pickle def run_template(class_object): @@ -27,15 +27,15 @@ def run_template(class_object): def load_search_list(filename): - f = open(filename) + f = open(filename, 'rb') raw_data = f.read() ext = os.path.splitext(filename)[-1] if ext == '.pkl': data = pickle.loads(raw_data) else: try: - data = eval(raw_data) - except Exception, e: + data = eval(raw_data.decode('utf-8')) + except Exception as e: logging.error('load_search_list\n%s', raw_data) raise return data diff --git a/spitfire/runtime/template.py b/spitfire/runtime/template.py index 7cb83a0..ab4f5dd 100644 --- a/spitfire/runtime/template.py +++ b/spitfire/runtime/template.py @@ -5,7 +5,7 @@ # an 'abstract' base class for a template, seems like a good idea for now -import cStringIO as StringIO +import io as StringIO from spitfire import runtime from spitfire.runtime import baked diff --git a/spitfire/runtime/udn.py b/spitfire/runtime/udn.py index dbe5aaf..bd9e824 100644 --- a/spitfire/runtime/udn.py +++ b/spitfire/runtime/udn.py @@ -9,7 +9,7 @@ # syntactically, 'name' will always be a valid identifier - so you won't get # name='my attribute' - it must be a legal python identifier -import __builtin__ +import builtins import inspect import logging import weakref @@ -180,7 +180,7 @@ def _resolve_placeholder(name, template, global_vars): # fixme: finally try to resolve builtins - this should be configurable # if you compile optimized modes, this isn't necessary try: - return getattr(__builtin__, name) + return getattr(builtins, name) except AttributeError: return UndefinedPlaceholder(name, search_list) diff --git a/spitfire/text.py b/spitfire/text.py index 9b135ac..43830c6 100644 --- a/spitfire/text.py +++ b/spitfire/text.py @@ -7,7 +7,7 @@ import string import unicodedata -normal_characters = string.lowercase + string.uppercase +normal_characters = string.ascii_lowercase + string.ascii_uppercase mangled_character_names = [ 'LATIN SMALL LETTER A WITH RING ABOVE', 'LATIN SMALL LETTER THORN', diff --git a/tests/output-preserve-whitespace/some_library.txt b/tests/output-preserve-whitespace/some_library.txt index 986fa1c..0f80db0 100644 --- a/tests/output-preserve-whitespace/some_library.txt +++ b/tests/output-preserve-whitespace/some_library.txt @@ -1 +1 @@ -'module' object has no attribute 'some_library' \ No newline at end of file +module 'tests.some_library' has no attribute 'some_library' \ No newline at end of file diff --git a/tests/output-preserve-whitespace/template_if_4.txt b/tests/output-preserve-whitespace/template_if_4.txt index 14471fc..2e4610f 100644 --- a/tests/output-preserve-whitespace/template_if_4.txt +++ b/tests/output-preserve-whitespace/template_if_4.txt @@ -1,2 +1,2 @@ - test u' ' + test ' ' \ No newline at end of file diff --git a/tests/output-preserve-whitespace/util.txt b/tests/output-preserve-whitespace/util.txt index 0c02a5e..58fdaa3 100644 --- a/tests/output-preserve-whitespace/util.txt +++ b/tests/output-preserve-whitespace/util.txt @@ -1 +1 @@ -'module' object has no attribute 'util' \ No newline at end of file +module 'tests.util' has no attribute 'util' \ No newline at end of file diff --git a/tests/output/some_library.txt b/tests/output/some_library.txt index 986fa1c..0f80db0 100644 --- a/tests/output/some_library.txt +++ b/tests/output/some_library.txt @@ -1 +1 @@ -'module' object has no attribute 'some_library' \ No newline at end of file +module 'tests.some_library' has no attribute 'some_library' \ No newline at end of file diff --git a/tests/output/template_if_4.txt b/tests/output/template_if_4.txt index afc8857..6f2393c 100644 --- a/tests/output/template_if_4.txt +++ b/tests/output/template_if_4.txt @@ -1 +1 @@ - test u' ' + test ' ' diff --git a/tests/output/util.txt b/tests/output/util.txt index 0c02a5e..58fdaa3 100644 --- a/tests/output/util.txt +++ b/tests/output/util.txt @@ -1 +1 @@ -'module' object has no attribute 'util' \ No newline at end of file +module 'tests.util' has no attribute 'util' \ No newline at end of file diff --git a/tests/test-hoist-conditional-loop-optimization-3.txt b/tests/test-hoist-conditional-loop-optimization-3.txt index 78c35c3..0198da6 100644 --- a/tests/test-hoist-conditional-loop-optimization-3.txt +++ b/tests/test-hoist-conditional-loop-optimization-3.txt @@ -3,7 +3,7 @@ ## the bug was in calculating the dependencies for $msg ## you have to take into account the dependency of the ## and conditions between you and the loop -#for $i in $xrange(2) +#for $i in $range(2) #if $i % 2 == 0 #set $msg = 'Success: even' #else diff --git a/third_party/yapps2/yapps2.py b/third_party/yapps2/yapps2.py old mode 100755 new mode 100644 index 89da486..8152060 --- a/third_party/yapps2/yapps2.py +++ b/third_party/yapps2/yapps2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Yapps 2.0 - yet another python parser system # Amit J Patel, January 1999 @@ -48,7 +48,7 @@ def __init__(self, name, options, tokens, rules): n = t self.ignore.append(n) if n in self.tokens.keys() and self.tokens[n] != t: - print 'Warning: token', n, 'multiply defined.' + print('Warning: token', n, 'multiply defined.') self.tokens[n] = t self.terminals.append(n) @@ -67,7 +67,7 @@ def __getitem__(self, name): return self.options.get(name, 0) def non_ignored_tokens(self): - return filter(lambda x, i=self.ignore: x not in i, self.terminals) + return list(filter(lambda x, i=self.ignore: x not in i, self.terminals)) def changed(self): self.change_count = 1+self.change_count @@ -104,7 +104,7 @@ def in_test(self, x, full, b): if len(b) == 1: return '%s == %s' % (x, repr(b[0])) if full and len(b) > len(full)/2: # Reverse the sense of the test. - not_b = filter(lambda x, b=b: x not in b, full) + not_b = list(filter(lambda x, b=b: x not in b, full)) return self.not_in_test(x, full, not_b) return '%s in %s' % (x, repr(b)) @@ -144,20 +144,20 @@ def calculate(self): def dump_information(self): self.calculate() for r in self.goals: - print ' _____' + '_'*len(r) - print ('___/Rule '+r+'\\' + '_'*80)[:79] + print(' _____' + '_'*len(r)) + print(('___/Rule '+r+'\\' + '_'*80)[:79]) queue = [self.rules[r]] while queue: top = queue[0] del queue[0] - print repr(top) + print(repr(top)) top.first.sort() top.follow.sort() eps = [] if top.accepts_epsilon: eps = ['(null)'] - print ' FIRST:', join(top.first+eps, ', ') - print ' FOLLOW:', join(top.follow, ', ') + print(' FIRST:', ', '.join(top.first+eps)) + print(' FOLLOW:', ', '.join(top.follow)) for x in top.get_children(): queue.append(x) def generate_output(self): @@ -167,17 +167,17 @@ def generate_output(self): self.write("from string import *\n") self.write("import re\n") self.write("from third_party.yapps2.yappsrt import *\n") - self.write("\n") - self.write("class ", self.name, "Scanner(Scanner):\n") + self.write("\n") + self.write("class ", self.name, "Scanner(Scanner):\n") self.write(" patterns = [\n") for p in self.terminals: self.write(" (%s, re.compile(%s)),\n" % ( repr(p), repr(self.tokens[p]))) self.write(" ]\n") - self.write(" def __init__(self, str):\n") - self.write(" Scanner.__init__(self,None,%s,str)\n" % + self.write(" def __init__(self, str):\n") + self.write(" Scanner.__init__(self,None,%s,str)\n" % repr(self.ignore)) - self.write("\n") + self.write("\n") self.write("class ", self.name, "(Parser):\n") for r in self.goals: @@ -202,8 +202,8 @@ def generate_output(self): self.write(INDENT*3, "f = open(argv[2],'r')\n") self.write(INDENT*2, "else:\n") self.write(INDENT*3, "f = stdin\n") - self.write(INDENT*2, "print parse(argv[1], f.read())\n") - self.write(INDENT, "else: print 'Args: []'\n") + self.write(INDENT*2, "print(parse(argv[1], f.read()))\n") + self.write(INDENT, "else: print('Args: []')\n") ###################################################################### class Node: @@ -271,10 +271,10 @@ def setup(self, gen, rule): gen.changed() def __str__(self): - return '{{ %s }}' % strip(self.expr) + return '{{ %s }}' % self.expr.strip() def output(self, gen, indent): - gen.write(indent, strip(self.expr), '\n') + gen.write(indent, self.expr.strip(), '\n') class NonTerminal(Node): def __init__(self, name, args): @@ -290,7 +290,7 @@ def setup(self, gen, rule): self.accepts_epsilon = self.target.accepts_epsilon gen.changed() except KeyError: # Oops, it's nonexistent - print 'Error: no rule <%s>' % self.name + print('Error: no rule <%s>' % self.name) self.target = self def __str__(self): @@ -328,7 +328,7 @@ def get_children(self): return self.children def __str__(self): - return '( %s )' % join(map(lambda x: str(x), self.children)) + return '( %s )' % " ".join(map(str, self.children)) def update(self, gen): Node.update(self, gen) @@ -378,7 +378,7 @@ def get_children(self): return self.children def __str__(self): - return '( %s )' % join(map(lambda x: str(x), self.children), ' | ') + return '( %s )' % ' | '.join(map(str, self.children)) def update(self, gen): Node.update(self, gen) @@ -413,11 +413,11 @@ def output(self, gen, indent): tokens_seen = tokens_seen + testset if removed: if not testset: - print 'Error in rule', self.rule+':', c, 'never matches.' + print('Error in rule', self.rule+':', c, 'never matches.') else: - print 'Warning:', self - print ' * These tokens are being ignored:', join(removed, ', ') - print ' due to previous choices using them.' + print('Warning:', self) + print(' * These tokens are being ignored:', ', '.join(removed)) + print(' due to previous choices using them.') if testset: if not tokens_unseen: # context sensitive scanners only! @@ -474,7 +474,7 @@ def __str__(self): def output(self, gen, indent): if self.child.accepts_epsilon: - print 'Warning in rule', self.rule+': contents may be empty.' + print('Warning in rule', self.rule+': contents may be empty.') gen.write(indent, "if %s:\n" % gen.peek_test(self.first, self.child.first)) self.child.output(gen, indent+INDENT) @@ -495,9 +495,9 @@ def update(self, gen): def output(self, gen, indent): if self.child.accepts_epsilon: - print 'Warning in rule', self.rule+':' - print ' * The repeated pattern could be empty. The resulting' - print ' parser may not work properly.' + print('Warning in rule', self.rule+':') + print(' * The repeated pattern could be empty. The resulting') + print(' parser may not work properly.') gen.write(indent, "while 1:\n") self.child.output(gen, indent+INDENT) union = self.first[:] @@ -517,9 +517,9 @@ def __str__(self): def output(self, gen, indent): if self.child.accepts_epsilon: - print 'Warning in rule', self.rule+':' - print ' * The repeated pattern could be empty. The resulting' - print ' parser probably will not work properly.' + print('Warning in rule', self.rule+':') + print(' * The repeated pattern could be empty. The resulting') + print(' parser probably will not work properly.') gen.write(indent, "while %s:\n" % gen.peek_test(self.follow, self.child.first)) self.child.output(gen, indent+INDENT) @@ -539,11 +539,11 @@ def add_inline_token(tokens, str): def cleanup_choice(lst): if len(lst) == 0: return Sequence([]) if len(lst) == 1: return lst[0] - return apply(Choice, tuple(lst)) + return Choice(*lst) def cleanup_sequence(lst): if len(lst) == 1: return lst[0] - return apply(Sequence, tuple(lst)) + return Sequence(*lst) def cleanup_rep(node, rep): if rep == 'star': return Star(node) @@ -552,16 +552,16 @@ def cleanup_rep(node, rep): def resolve_name(tokens, id, args): if id in map(lambda x: x[0], tokens): - # It's a token - if args: - print 'Warning: ignoring parameters on TOKEN %s<<%s>>' % (id, args) + # It's a token + if args: + print('Warning: ignoring parameters on TOKEN %s<<%s>>' % (id, args)) return Terminal(id) else: # It's a name, so assume it's a nonterminal return NonTerminal(id, args) -from string import * + import re from yappsrt import * @@ -717,13 +717,13 @@ def generate(inputfilename, outputfilename='', dump=0, **flags): and an output filename (defaulting to X.py).""" if not outputfilename: - if inputfilename[-2:] == '.g': + if inputfilename[-2:] == '.g': outputfilename = inputfilename[:-2]+'.py' - else: + else: raise Exception("Missing output filename") - print 'Input Grammar:', inputfilename - print 'Output File:', outputfilename + print('Input Grammar:', inputfilename) + print('Output File:', outputfilename) DIVIDER = '\n%%\n' # This pattern separates the pre/post parsers preparser, postparser = None, None # Code before and after the parser desc @@ -732,11 +732,11 @@ def generate(inputfilename, outputfilename='', dump=0, **flags): s = open(inputfilename,'r').read() # See if there's a separation between the pre-parser and parser - f = find(s, DIVIDER) + f = s.find(DIVIDER) if f >= 0: preparser, s = s[:f]+'\n\n', s[f+len(DIVIDER):] # See if there's a separation between the parser and post-parser - f = find(s, DIVIDER) + f = s.find(DIVIDER) if f >= 0: s, postparser = s[:f], '\n\n'+s[f+len(DIVIDER):] # Create the parser and scanner @@ -754,7 +754,7 @@ def generate(inputfilename, outputfilename='', dump=0, **flags): for opt,_,_ in yapps_options: if f == opt: break else: - print 'Warning: unrecognized option', f + print('Warning: unrecognized option', f) # Add command line options to the set for f in flags.keys(): t.options[f] = flags[f] @@ -769,24 +769,24 @@ def generate(inputfilename, outputfilename='', dump=0, **flags): import getopt optlist, args = getopt.getopt(sys.argv[1:], 'f:', ['dump']) if not args or len(args) > 2: - print 'Usage:' - print ' python', sys.argv[0], '[flags] input.g [output.py]' - print 'Flags:' - print (' --dump' + ' '*40)[:35] + 'Dump out grammar information' + print('Usage:') + print(' python', sys.argv[0], '[flags] input.g [output.py]') + print('Flags:') + print((' --dump' + ' '*40)[:35] + 'Dump out grammar information') for flag, _, doc in yapps_options: - print (' -f' + flag + ' '*40)[:35] + doc + print((' -f' + flag + ' '*40)[:35] + doc) else: # Read in the options and create a list of flags - flags = {} - for opt in optlist: - for flag, name, _ in yapps_options: - if opt == ('-f', flag): - flags[name] = 1 - break - else: + flags = {} + for opt in optlist: + for flag, name, _ in yapps_options: + if opt == ('-f', flag): + flags[name] = 1 + break + else: if opt == ('--dump', ''): flags['dump'] = 1 - else: - print 'Warning: unrecognized option', opt[0], opt[1] + else: + print('Warning: unrecognized option', opt[0], opt[1]) - apply(generate, tuple(args), flags) + generate(*args, **flags) diff --git a/third_party/yapps2/yappsrt.py b/third_party/yapps2/yappsrt.py index a4eba99..13e0607 100644 --- a/third_party/yapps2/yappsrt.py +++ b/third_party/yapps2/yappsrt.py @@ -2,10 +2,11 @@ # # This module is needed to run generated parsers. -from string import join, count, find, rfind +# This module is needed to run generated parsers. + import logging import re -import StringIO +import io import sys class SyntaxError(Exception): @@ -109,7 +110,7 @@ def scan(self, restrict): if best_pat == '(error)' and best_match < 0: msg = "Bad Token" if restrict: - msg = "Trying to find one of "+join(restrict,", ") + msg = "Trying to find one of "+", ".join(restrict) raise SyntaxError(self.pos, msg) # If we found something that isn't to be ignored, return it @@ -155,27 +156,27 @@ def file_position(self): def format_error(input, err, scanner): """This is a really dumb long function to print error messages nicely.""" - error_message = StringIO.StringIO() + error_message = io.StringIO() p = err.pos - print >> error_message, "error position", p + print("error position", p, file=error_message) # Figure out the line number - line = count(input[:p], '\n') - print >> error_message, err.msg, "on line", repr(line+1) + ":" + line = input[:p].count('\n') + print(err.msg, "on line", repr(line+1) + ":", file=error_message) # Now try printing part of the line text = input[max(p-80, 0):p+80] p = p - max(p-80, 0) # Strip to the left - i = rfind(text[:p], '\n') - j = rfind(text[:p], '\r') + i = text[:p].rfind('\n') + j = text[:p].rfind('\r') if i < 0 or (0 <= j < i): i = j if 0 <= i < p: p = p - i - 1 text = text[i+1:] # Strip to the right - i = find(text,'\n', p) - j = find(text,'\r', p) + i = text.find('\n', p) + j = text.find('\r', p) if i < 0 or (0 <= j < i): i = j if i >= 0: @@ -188,22 +189,22 @@ def format_error(input, err, scanner): p = p - 7 # Now print the string, along with an indicator - print >> error_message, '> ', text.replace('\t', ' ').encode(sys.getdefaultencoding()) - print >> error_message, '> ', ' '*p + '^' - print >> error_message, 'List of nearby tokens:', scanner + print('> ', text.replace('\t', ' ').encode(sys.getdefaultencoding()), file=error_message) + print('> ', ' '*p + '^', file=error_message) + print('List of nearby tokens:', scanner, file=error_message) return error_message.getvalue() def wrap_error_reporter(parser, rule): try: return getattr(parser, rule)() - except SyntaxError, e: + except SyntaxError as e: logging.exception('syntax error') input = parser._scanner.input try: error_msg = format_error(input, e, parser._scanner) except ImportError: - error_msg = 'Syntax Error %s on line\n' % (e.msg, 1 + count(input[:e.pos])) - except NoMoreTokens, e: + error_msg = 'Syntax Error %s on line\n' % (e.msg, 1 + input[:e.pos].count('\n')) + except NoMoreTokens as e: error_msg = 'Could not complete parsing; stopped around here:\n%s\n%s' % (parser._scanner, e) raise FatalParseError(error_msg)