feat: 3.12 support for the PositionNodeFinder

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev, pypy2, pypy-3.6]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.6]
 
     steps:
     - uses: actions/checkout@v2
@@ -26,15 +26,15 @@ jobs:
         pip install mypy==0.910
         python -m mypy executing --exclude=executing/_position_node_finder.py
       # fromJson because https://github.community/t/passing-an-array-literal-to-contains-function-causes-syntax-error/17213/3
-      if: ${{ !contains(fromJson('["2.7", "pypy2", "pypy-3.6", "3.11-dev"]'), matrix.python-version) }}
+      if: ${{ !contains(fromJson('["pypy-3.6", "3.11","3.12-dev"]'), matrix.python-version) }}
       # pypy < 3.8 very doesn't work
-      # 2.7 is tested separately in mypy-py2, as we need to run mypy under Python 3.x            
     - name: Mypy testing (3.11)
       run: |
         pip install mypy==0.971
         python -m mypy executing
       # fromJson because https://github.community/t/passing-an-array-literal-to-contains-function-causes-syntax-error/17213/3
-      if: ${{ contains(fromJson('["3.11-dev"]'), matrix.python-version) }}
+      # TODO: enable typechecking for 3.12
+      if: ${{ contains(fromJson('["3.11"]'), matrix.python-version) }}
       # only >=3.11 use _position_node_finder.py
     - name: Test
       env:
@@ -56,23 +56,3 @@ jobs:
         uses: AndreMiras/coveralls-python-action@v20201129
         with:
           parallel-finished: true
-
-  # Can't run mypy on Python 2.7, but can run it in Python 2 mode
-  mypy-py2:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          pip install --upgrade setuptools setuptools_scm pep517
-          pip install .[tests]
-          pip install mypy[python2]==0.910
-      - name: Mypy testing for Python 2
-        run: |
-          python -m mypy --py2 executing --exclude=executing/_position_node_finder.py

executing/_position_node_finder.py

@@ -46,8 +46,10 @@ def mangled_name(node: EnhancedAST) -> str:
     elif isinstance(node, ast.ExceptHandler):
         assert node.name
         name = node.name
+    elif sys.version_info >= (3,12) and isinstance(node,ast.TypeVar):
+        name=node.name
     else:
-        raise TypeError("no node to mangle")
+        raise TypeError("no node to mangle for type "+repr(type(node)))
 
     if name.startswith("__") and not name.endswith("__"):
 
@@ -176,21 +178,24 @@ def test_for_decorator(self, node: EnhancedAST, index: int) -> None:
 
                 # index    opname
                 # ------------------
-                # index-4  PRECALL
+                # index-4  PRECALL     (only in 3.11)
                 # index-2  CACHE
                 # index    CALL        <- the call instruction
                 # ...      CACHE       some CACHE instructions
 
                 # maybe multiple other bytecode blocks for other decorators
-                # index-4  PRECALL
+                # index-4  PRECALL     (only in 3.11)
                 # index-2  CACHE
                 # index    CALL        <- index of the next loop
                 # ...      CACHE       some CACHE instructions
 
                 # index+x  STORE_*     the ast-node of this instruction points to the decorated thing
 
-                if self.opname(index - 4) != "PRECALL" or self.opname(index) != "CALL": # pragma: no mutate
-                    break # pragma: no mutate
+                if not (
+                    (self.opname(index - 4) == "PRECALL" or sys.version_info >= (3, 12))
+                    and self.opname(index) == "CALL"
+                ):  # pragma: no mutate
+                    break  # pragma: no mutate
 
                 index += 2
 
@@ -205,7 +210,8 @@ def test_for_decorator(self, node: EnhancedAST, index: int) -> None:
                     self.decorator = node
                     return
 
-                index += 4
+                if sys.version_info < (3, 12):
+                    index += 4
 
     def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
         if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance(
@@ -259,6 +265,37 @@ def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
             # TODO: investigate
             raise KnownIssue("pattern matching ranges seems to be wrong")
 
+        if (
+            sys.version_info >= (3, 12)
+            and isinstance(node, ast.Call)
+            and isinstance(node.func, ast.Name)
+            and node.func.id == "super"
+        ):
+            # super is optimized to some instructions which do not map nicely to a Call
+
+            # find the enclosing function
+            func = node.parent
+            while hasattr(func, "parent") and not isinstance(
+                func, (ast.AsyncFunctionDef, ast.FunctionDef)
+            ):
+
+                func = func.parent
+
+            # get the first function argument (self/cls)
+            first_arg = None
+
+            if hasattr(func, "args"):
+                args = [*func.args.posonlyargs, *func.args.args]
+                if args:
+                    first_arg = args[0].arg
+
+            if (instruction.opname, instruction.argval) in [
+                ("LOAD_DEREF", "__class__"),
+                ("LOAD_FAST", first_arg),
+                ("LOAD_DEREF", first_arg),
+            ]:
+                raise KnownIssue("super optimization")
+
         if self.is_except_cleanup(instruction, node):
             raise KnownIssue("exeption cleanup does not belong to the last node in a except block")
 
@@ -279,6 +316,14 @@ def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None:
 
             raise KnownIssue("store __classcell__")
 
+        if (
+            instruction.opname == "CALL"
+            and not isinstance(node,ast.Call)
+            and any(isinstance(p, ast.Assert) for p in parents(node))
+            and sys.version_info >= (3, 11, 2)
+        ):
+            raise KnownIssue("exception generation maps to condition")
+
     @staticmethod
     def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool:
         if inst.opname not in (
@@ -392,6 +437,13 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
             # call to the generator function
             return
 
+        if (
+            sys.version_info >= (3, 12)
+            and inst_match(("LOAD_FAST_AND_CLEAR", "STORE_FAST"))
+            and node_match((ast.ListComp, ast.SetComp, ast.DictComp))
+        ):
+            return
+
         if inst_match(("CALL", "CALL_FUNCTION_EX")) and node_match(
             (ast.ClassDef, ast.Call)
         ):
@@ -410,6 +462,7 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
         if (
             (
                 inst_match("LOAD_METHOD", argval="join")
+                or inst_match("LOAD_ATTR", argval="join")  # 3.12
                 or inst_match(("CALL", "BUILD_STRING"))
             )
             and node_match(ast.BinOp, left=ast.Constant, op=ast.Mod)
@@ -495,9 +548,14 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
         ):
             return
 
-        if inst_match(("JUMP_IF_TRUE_OR_POP", "JUMP_IF_FALSE_OR_POP")) and node_match(
-            ast.BoolOp
-        ):
+        if inst_match(
+            (
+                "JUMP_IF_TRUE_OR_POP",
+                "JUMP_IF_FALSE_OR_POP",
+                "POP_JUMP_IF_TRUE",
+                "POP_JUMP_IF_FALSE",
+            )
+        ) and node_match(ast.BoolOp):
             # and/or short circuit
             return
 
@@ -511,7 +569,15 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
                 and isinstance(node.parent, ast.AugAssign)
             )
         ) and inst_match(
-            ("LOAD_NAME", "LOAD_FAST", "LOAD_GLOBAL", "LOAD_DEREF"), argval=mangled_name(node)
+            (
+                "LOAD_NAME",
+                "LOAD_FAST",
+                "LOAD_FAST_CHECK",
+                "LOAD_GLOBAL",
+                "LOAD_DEREF",
+                "LOAD_FROM_DICT_OR_DEREF",
+            ),
+            argval=mangled_name(node),
         ):
             return
 
@@ -520,6 +586,124 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
         ):
             return
 
+        if node_match(ast.Constant) and inst_match(
+            "LOAD_CONST", argval=cast(ast.Constant, node).value
+        ):
+            return
+
+        if node_match(
+            (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.For)
+        ) and inst_match(("GET_ITER", "FOR_ITER")):
+            return
+
+        if sys.version_info >= (3, 12):
+            if node_match(ast.UnaryOp, op=ast.UAdd) and inst_match(
+                "CALL_INTRINSIC_1", argrepr="INTRINSIC_UNARY_POSITIVE"
+            ):
+                return
+
+            if node_match(ast.Subscript) and inst_match("BINARY_SLICE"):
+                return
+
+            if node_match(ast.ImportFrom) and inst_match(
+                "CALL_INTRINSIC_1", argrepr="INTRINSIC_IMPORT_STAR"
+            ):
+                return
+
+            if (
+                node_match(ast.Yield) or isinstance(node.parent, ast.GeneratorExp)
+            ) and inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_ASYNC_GEN_WRAP"):
+                return
+
+            if node_match(ast.Name) and inst_match("LOAD_DEREF",argval="__classdict__"):
+                return
+
+            if node_match(ast.TypeVar) and (
+                inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVAR")
+                or inst_match(
+                    "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_BOUND"
+                )
+                or inst_match(
+                    "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_CONSTRAINTS"
+                )
+                or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=mangled_name(node))
+            ):
+                return
+
+            if node_match(ast.TypeVarTuple) and (
+                inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVARTUPLE")
+                or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)
+            ):
+                return
+
+            if node_match(ast.ParamSpec) and (
+                inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_PARAMSPEC")
+
+                or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)):
+                return
+
+
+            if node_match(ast.TypeAlias):
+                if(
+                    inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEALIAS")
+                    or inst_match(
+                        ("STORE_NAME", "STORE_FAST", "STORE_DEREF"), argrepr=node.name.id
+                    )
+                    or inst_match("CALL")
+                ):
+                    return
+
+
+            if node_match(ast.ClassDef) and node.type_params:
+                if inst_match(
+                    ("STORE_DEREF", "LOAD_DEREF", "LOAD_FROM_DICT_OR_DEREF"),
+                    argrepr=".type_params",
+                ):
+                    return
+
+                if inst_match(("STORE_FAST", "LOAD_FAST"), argrepr=".generic_base"):
+                    return
+
+                if inst_match(
+                    "CALL_INTRINSIC_1", argrepr="INTRINSIC_SUBSCRIPT_GENERIC"
+                ):
+                    return
+
+                if inst_match("LOAD_DEREF",argval="__classdict__"):
+                    return
+
+            if node_match((ast.FunctionDef,ast.AsyncFunctionDef)) and node.type_params:
+                if inst_match("CALL"):
+                    return
+
+                if inst_match(
+                    "CALL_INTRINSIC_2", argrepr="INTRINSIC_SET_FUNCTION_TYPE_PARAMS"
+                ):
+                    return
+
+                if inst_match("LOAD_FAST",argval=".defaults"):
+                    return
+
+                if inst_match("LOAD_FAST",argval=".kwdefaults"):
+                    return
+
+            if inst_match("STORE_NAME", argval="__classdictcell__"):
+                # this is a general thing
+                return
+
+
+            # f-strings
+
+            if node_match(ast.JoinedStr) and (
+                inst_match("LOAD_ATTR", argval="join")
+                or inst_match(("LIST_APPEND", "CALL"))
+            ):
+                return
+
+            if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"):
+                return
+
+
         # old verifier
 
         typ: Type = type(None)
@@ -541,7 +725,7 @@ def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool:
                 UNARY_INVERT=ast.Invert,
             )[op_name]
             extra_filter = lambda e: isinstance(cast(ast.UnaryOp, e).op, op_type)
-        elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD"):
+        elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD","LOAD_SUPER_ATTR"):
             typ = ast.Attribute
             ctx = ast.Load
             extra_filter = lambda e: mangled_name(e) == instruction.argval
@@ -593,11 +777,26 @@ def instruction(self, index: int) -> dis.Instruction:
     def opname(self, index: int) -> str:
         return self.instruction(index).opname
 
+    extra_node_types=()
+    if sys.version_info >= (3,12):
+        extra_node_types = (ast.type_param,)
+
     def find_node(
         self,
         index: int,
-        match_positions: Sequence[str]=("lineno", "end_lineno", "col_offset", "end_col_offset"),
-        typ: tuple[Type, ...]=(ast.expr, ast.stmt, ast.excepthandler, ast.pattern),
+        match_positions: Sequence[str] = (
+            "lineno",
+            "end_lineno",
+            "col_offset",
+            "end_col_offset",
+        ),
+        typ: tuple[Type, ...] = (
+            ast.expr,
+            ast.stmt,
+            ast.excepthandler,
+            ast.pattern,
+            *extra_node_types,
+        ),
     ) -> EnhancedAST:
         position = self.instruction(index).positions
         assert position is not None and position.lineno is not None