Allow overloads in source files, not just stubs (#2603)

Fixes #1136.

- The implementation must directly follow all the overloads
- The implementation is typechecked exactly according to its own declared types
- Indicates an error if the implementation's argument list is not more general than every override variant
- Also indicates an error if the implementation's return type is also not more general than the return type of every override variant

It also limits overloads to the decorators that are specifically geared towards providing overload-type functionality -- @overload, but also @Property and its cousins @funcname.setter and @funcname.deleter. All other decorators are treated purely as decorators, and now provide redefinition errors if you repeat a function definition with them, instead of errors about an overload you probably did not mean.

Author: sixolet

mypy/build.py

@@ -1401,7 +1401,8 @@ def parse_file(self) -> None:
         # this before processing imports, since this may mark some
         # import statements as unreachable.
         first = FirstPass(manager.semantic_analyzer)
-        first.visit_file(self.tree, self.xpath, self.id, self.options)
+        with self.wrap_context():
+            first.visit_file(self.tree, self.xpath, self.id, self.options)
 
         # Initialize module symbol table, which was populated by the
         # semantic analyzer.

mypy/checker.py

@@ -37,7 +37,8 @@
 from mypy import messages
 from mypy.subtypes import (
     is_subtype, is_equivalent, is_proper_subtype, is_more_precise,
-    restrict_subtype_away, is_subtype_ignoring_tvars
+    restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype,
+    unify_generic_callable,
 )
 from mypy.maptype import map_instance_to_supertype
 from mypy.typevars import fill_typevars, has_no_typevars
@@ -262,29 +263,70 @@ def accept_loop(self, body: Statement, else_body: Statement = None, *,
 
     def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
         num_abstract = 0
+        if not defn.items:
+            # In this case we have already complained about none of these being
+            # valid overloads.
+            return None
+        if len(defn.items) == 1:
+            self.fail('Single overload definition, multiple required', defn)
+
         if defn.is_property:
             # HACK: Infer the type of the property.
-            self.visit_decorator(defn.items[0])
+            self.visit_decorator(cast(Decorator, defn.items[0]))
         for fdef in defn.items:
+            assert isinstance(fdef, Decorator)
             self.check_func_item(fdef.func, name=fdef.func.name())
             if fdef.func.is_abstract:
                 num_abstract += 1
         if num_abstract not in (0, len(defn.items)):
             self.fail(messages.INCONSISTENT_ABSTRACT_OVERLOAD, defn)
+        if defn.impl:
+            defn.impl.accept(self)
         if defn.info:
             self.check_method_override(defn)
             self.check_inplace_operator_method(defn)
         self.check_overlapping_overloads(defn)
+        return None
 
     def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None:
+        # At this point we should have set the impl already, and all remaining
+        # items are decorators
         for i, item in enumerate(defn.items):
+            assert isinstance(item, Decorator)
+            sig1 = self.function_type(item.func)
             for j, item2 in enumerate(defn.items[i + 1:]):
                 # TODO overloads involving decorators
-                sig1 = self.function_type(item.func)
+                assert isinstance(item2, Decorator)
                 sig2 = self.function_type(item2.func)
                 if is_unsafe_overlapping_signatures(sig1, sig2):
                     self.msg.overloaded_signatures_overlap(i + 1, i + j + 2,
                                                            item.func)
+            if defn.impl:
+                if isinstance(defn.impl, FuncDef):
+                    impl_type = defn.impl.type
+                elif isinstance(defn.impl, Decorator):
+                    impl_type = defn.impl.var.type
+                else:
+                    assert False, "Impl isn't the right type"
+                # This can happen if we've got an overload with a different
+                # decorator too -- we gave up on the types.
+                if impl_type is None or isinstance(impl_type, AnyType) or sig1 is None:
+                    return
+
+                assert isinstance(impl_type, CallableType)
+                assert isinstance(sig1, CallableType)
+                if not is_callable_subtype(impl_type, sig1, ignore_return=True):
+                    self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl)
+                impl_type_subst = impl_type
+                if impl_type.variables:
+                    impl_type_subst = unify_generic_callable(impl_type, sig1, ignore_return=False)
+                    if impl_type_subst is None:
+                        self.fail("Type variable mismatch between " +
+                                  "overload signature {} and implementation".format(i + 1),
+                                  defn.impl)
+                        return
+                if not is_subtype(sig1.ret_type, impl_type_subst.ret_type):
+                    self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl)
 
     # Here's the scoop about generators and coroutines.
     #
@@ -2133,9 +2175,7 @@ def visit_decorator(self, e: Decorator) -> None:
         e.func.accept(self)
         sig = self.function_type(e.func)  # type: Type
         # Process decorators from the inside out.
-        for i in range(len(e.decorators)):
-            n = len(e.decorators) - 1 - i
-            d = e.decorators[n]
+        for d in reversed(e.decorators):
             if isinstance(d, NameExpr) and d.fullname == 'typing.overload':
                 self.fail('Single overload definition, multiple required', e)
                 continue
@@ -2159,7 +2199,8 @@ def check_incompatible_property_override(self, e: Decorator) -> None:
                     continue
                 if (isinstance(base_attr.node, OverloadedFuncDef) and
                         base_attr.node.is_property and
-                        base_attr.node.items[0].var.is_settable_property):
+                        cast(Decorator,
+                             base_attr.node.items[0]).var.is_settable_property):
                     self.fail(messages.READ_ONLY_PROPERTY_OVERRIDES_READ_WRITE, e)
 
     def visit_with_stmt(self, s: WithStmt) -> None:

mypy/checkmember.py

@@ -75,7 +75,8 @@ def analyze_member_access(name: str,
         if method:
             if method.is_property:
                 assert isinstance(method, OverloadedFuncDef)
-                return analyze_var(name, method.items[0].var, typ, info, node, is_lvalue, msg,
+                first_item = cast(Decorator, method.items[0])
+                return analyze_var(name, first_item.var, typ, info, node, is_lvalue, msg,
                                    original_type, not_ready_callback)
             if is_lvalue:
                 msg.cant_assign_to_method(node)

mypy/fastparse.py

@@ -2,9 +2,12 @@
 import sys
 
 from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List, Set
-from mypy.sharedparse import special_function_elide_names, argument_elide_name
+from mypy.sharedparse import (
+    special_function_elide_names, argument_elide_name,
+)
 from mypy.nodes import (
-    MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef,
+    MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef,
+    OverloadedFuncDef, OverloadPart,
     ClassDef, Decorator, Block, Var, OperatorAssignmentStmt,
     ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt,
     DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
@@ -222,11 +225,12 @@ def as_block(self, stmts: List[ast3.stmt], lineno: int) -> Block:
 
     def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]:
         ret = []  # type: List[Statement]
-        current_overload = []
+        current_overload = []  # type: List[OverloadPart]
         current_overload_name = None
-        # mypy doesn't actually check that the decorator is literally @overload
         for stmt in stmts:
-            if isinstance(stmt, Decorator) and stmt.name() == current_overload_name:
+            if (current_overload_name is not None
+                    and isinstance(stmt, (Decorator, FuncDef))
+                    and stmt.name() == current_overload_name):
                 current_overload.append(stmt)
             else:
                 if len(current_overload) == 1:

mypy/fastparse2.py

@@ -18,7 +18,9 @@
 import sys
 
 from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List, Set
-from mypy.sharedparse import special_function_elide_names, argument_elide_name
+from mypy.sharedparse import (
+    special_function_elide_names, argument_elide_name,
+)
 from mypy.nodes import (
     MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef,
     ClassDef, Decorator, Block, Var, OperatorAssignmentStmt,
@@ -31,7 +33,7 @@
     UnaryExpr, LambdaExpr, ComparisonExpr, DictionaryComprehension,
     SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument,
     Expression, Statement, BackquoteExpr, PrintStmt, ExecStmt,
-    ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2
+    ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart,
 )
 from mypy.types import (
     Type, CallableType, AnyType, UnboundType, EllipsisType
@@ -225,11 +227,12 @@ def as_block(self, stmts: List[ast27.stmt], lineno: int) -> Block:
 
     def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]:
         ret = []  # type: List[Statement]
-        current_overload = []
+        current_overload = []  # type: List[OverloadPart]
         current_overload_name = None
-        # mypy doesn't actually check that the decorator is literally @overload
         for stmt in stmts:
-            if isinstance(stmt, Decorator) and stmt.name() == current_overload_name:
+            if (current_overload_name is not None
+                    and isinstance(stmt, (Decorator, FuncDef))
+                    and stmt.name() == current_overload_name):
                 current_overload.append(stmt)
             else:
                 if len(current_overload) == 1:

mypy/fixup.py

@@ -106,6 +106,8 @@ def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None:
             o.type.accept(self.type_fixer)
         for item in o.items:
             item.accept(self)
+        if o.impl:
+            o.impl.accept(self)
 
     def visit_decorator(self, d: Decorator) -> None:
         if self.current_info is not None:

mypy/messages.py

@@ -806,6 +806,14 @@ def overloaded_signatures_overlap(self, index1: int, index2: int,
         self.fail('Overloaded function signatures {} and {} overlap with '
                   'incompatible return types'.format(index1, index2), context)
 
+    def overloaded_signatures_arg_specific(self, index1: int, context: Context) -> None:
+        self.fail('Overloaded function implementation does not accept all possible arguments '
+                  'of signature {}'.format(index1), context)
+
+    def overloaded_signatures_ret_specific(self, index1: int, context: Context) -> None:
+        self.fail('Overloaded function implementation cannot produce return type '
+                  'of signature {}'.format(index1), context)
+
     def operator_method_signatures_overlap(
             self, reverse_class: str, reverse_method: str, forward_class: str,
             forward_method: str, context: Context) -> None:

mypy/nodes.py

@@ -382,21 +382,29 @@ def fullname(self) -> str:
         return self._fullname
 
 
+OverloadPart = Union['FuncDef', 'Decorator']
+
+
 class OverloadedFuncDef(FuncBase, SymbolNode, Statement):
-    """A logical node representing all the variants of an overloaded function.
+    """A logical node representing all the variants of a multi-declaration function.
+
+    A multi-declaration function is often an @overload, but can also be a
+    @property with a setter and a/or a deleter.
 
     This node has no explicit representation in the source program.
     Overloaded variants must be consecutive in the source file.
     """
 
-    items = None  # type: List[Decorator]
+    items = None  # type: List[OverloadPart]
+    impl = None  # type: Optional[OverloadPart]
 
-    def __init__(self, items: List['Decorator']) -> None:
+    def __init__(self, items: List['OverloadPart']) -> None:
         self.items = items
+        self.impl = None
         self.set_line(items[0].line)
 
     def name(self) -> str:
-        return self.items[0].func.name()
+        return self.items[0].name()
 
     def accept(self, visitor: StatementVisitor[T]) -> T:
         return visitor.visit_overloaded_func_def(self)
@@ -407,12 +415,17 @@ def serialize(self) -> JsonDict:
                 'type': None if self.type is None else self.type.serialize(),
                 'fullname': self._fullname,
                 'is_property': self.is_property,
+                'impl': None if self.impl is None else self.impl.serialize()
                 }
 
     @classmethod
     def deserialize(cls, data: JsonDict) -> 'OverloadedFuncDef':
         assert data['.class'] == 'OverloadedFuncDef'
-        res = OverloadedFuncDef([Decorator.deserialize(d) for d in data['items']])
+        res = OverloadedFuncDef([
+            cast(OverloadPart, SymbolNode.deserialize(d))
+            for d in data['items']])
+        if data.get('impl') is not None:
+            res.impl = cast(OverloadPart, SymbolNode.deserialize(data['impl']))
         if data.get('type') is not None:
             res.type = mypy.types.deserialize_type(data['type'])
         res._fullname = data['fullname']
@@ -598,6 +611,7 @@ class Decorator(SymbolNode, Statement):
     func = None  # type: FuncDef                # Decorated function
     decorators = None  # type: List[Expression] # Decorators, at least one  # XXX Not true
     var = None  # type: Var                     # Represents the decorated function obj
+    type = None  # type: mypy.types.Type
     is_overload = False
 
     def __init__(self, func: FuncDef, decorators: List[Expression],