Add experimental hidden flag to tweak dependencies (#5367)

This flag will tweak fine grained dependencies to reflect propagation of type (im)precision instead of actual semantic dependencies. The flag should never be used to run actual type check, only to analyze type coverage in a code base.

The flag could be useful when annotating legacy code, to check which functions (decorators, base classes, variables) are most important to annotate first.

Author: ilevkivskyi

mypy/build.py

@@ -2217,7 +2217,8 @@ def compute_fine_grained_deps(self) -> None:
             return
         self.fine_grained_deps = get_dependencies(target=self.tree,
                                                   type_map=self.type_map(),
-                                                  python_version=self.options.python_version)
+                                                  python_version=self.options.python_version,
+                                                  options=self.manager.options)
 
     def valid_references(self) -> Set[str]:
         assert self.ancestors is not None
@@ -2570,7 +2571,8 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
     if manager.options.dump_deps:
         # This speeds up startup a little when not using the daemon mode.
         from mypy.server.deps import dump_all_dependencies
-        dump_all_dependencies(manager.modules, manager.all_types, manager.options.python_version)
+        dump_all_dependencies(manager.modules, manager.all_types,
+                              manager.options.python_version, manager.options)
     return graph
 
 

mypy/main.py

@@ -612,6 +612,12 @@ def add_invertible_flag(flag: str,
     # --local-partial-types disallows partial types spanning module top level and a function
     # (implicitly defined in fine-grained incremental mode)
     parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
+    # --logical-deps adds some more dependencies that are not semantically needed, but
+    # may be helpful to determine relative importance of classes and functions for overall
+    # type precision in a code base. It also _removes_ some deps, so this flag should be never
+    # used except for generating code stats. This also automatically enables --cache-fine-grained.
+    # NOTE: This is an experimental option that may be modified or removed at any time.
+    parser.add_argument('--logical-deps', action='store_true', help=argparse.SUPPRESS)
     # --bazel changes some behaviors for use with Bazel (https://bazel.build).
     parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
     # --package-root adds a directory below which directories are considered
@@ -776,6 +782,10 @@ def add_invertible_flag(flag: str,
     if options.quick_and_dirty:
         options.incremental = True
 
+    # Let logical_deps imply cache_fine_grained (otherwise the former is useless).
+    if options.logical_deps:
+        options.cache_fine_grained = True
+
     # Set target.
     if special_opts.modules + special_opts.packages:
         options.build_type = BuildType.MODULE

mypy/options.py

@@ -193,6 +193,7 @@ def __init__(self) -> None:
         self.show_column_numbers = False  # type: bool
         self.dump_graph = False
         self.dump_deps = False
+        self.logical_deps = False
         # If True, partial types can't span a module top level and a function
         self.local_partial_types = False
         # Some behaviors are changed when using Bazel (https://bazel.build).

mypy/server/deps.py

@@ -105,13 +105,15 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a
 from mypy.util import correct_relative_import
 from mypy.scope import Scope
 from mypy.typestate import TypeState
+from mypy.options import Options
 
 
 def get_dependencies(target: MypyFile,
                      type_map: Dict[Expression, Type],
-                     python_version: Tuple[int, int]) -> Dict[str, Set[str]]:
+                     python_version: Tuple[int, int],
+                     options: Options) -> Dict[str, Set[str]]:
     """Get all dependencies of a node, recursively."""
-    visitor = DependencyVisitor(type_map, python_version, target.alias_deps)
+    visitor = DependencyVisitor(type_map, python_version, target.alias_deps, options)
     target.accept(visitor)
     return visitor.map
 
@@ -148,7 +150,8 @@ class DependencyVisitor(TraverserVisitor):
     def __init__(self,
                  type_map: Dict[Expression, Type],
                  python_version: Tuple[int, int],
-                 alias_deps: 'DefaultDict[str, Set[str]]') -> None:
+                 alias_deps: 'DefaultDict[str, Set[str]]',
+                 options: Optional[Options] = None) -> None:
         self.scope = Scope()
         self.type_map = type_map
         self.python2 = python_version[0] == 2
@@ -165,6 +168,7 @@ def __init__(self,
         self.map = {}  # type: Dict[str, Set[str]]
         self.is_class = False
         self.is_package_init_file = False
+        self.options = options
 
     def visit_mypy_file(self, o: MypyFile) -> None:
         self.scope.enter_file(o.fullname())
@@ -201,6 +205,22 @@ def visit_decorator(self, o: Decorator) -> None:
         # generate dependency.
         if not o.func.is_overload and self.scope.current_function_name() is None:
             self.add_dependency(make_trigger(o.func.fullname()))
+        if self.options is not None and self.options.logical_deps:
+            # Add logical dependencies from decorators to the function. For example,
+            # if we have
+            #     @dec
+            #     def func(): ...
+            # then if `dec` is unannotated, then it will "spoil" `func` and consequently
+            # all call sites, making them all `Any`.
+            for d in o.decorators:
+                tname = None  # type: Optional[str]
+                if isinstance(d, RefExpr) and d.fullname is not None:
+                    tname = d.fullname
+                if (isinstance(d, CallExpr) and isinstance(d.callee, RefExpr) and
+                        d.callee.fullname is not None):
+                    tname = d.callee.fullname
+                if tname is not None:
+                    self.add_dependency(make_trigger(tname), make_trigger(o.func.fullname()))
         super().visit_decorator(o)
 
     def visit_class_def(self, o: ClassDef) -> None:
@@ -263,6 +283,18 @@ def process_type_info(self, info: TypeInfo) -> None:
                                         target=make_trigger(info.fullname() + '.' + name))
         for base_info in non_trivial_bases(info):
             for name, node in base_info.names.items():
+                if self.options and self.options.logical_deps:
+                    # Skip logical dependency if an attribute is not overridden. For example,
+                    # in case of:
+                    #     class Base:
+                    #         x = 1
+                    #         y = 2
+                    #     class Sub(Base):
+                    #         x = 3
+                    # we skip <Base.y> -> <Child.y>, because even if `y` is unannotated it
+                    # doesn't affect precision of Liskov checking.
+                    if name not in info.names:
+                        continue
                 self.add_dependency(make_trigger(base_info.fullname() + '.' + name),
                                     target=make_trigger(info.fullname() + '.' + name))
             self.add_dependency(make_trigger(base_info.fullname() + '.__init__'),
@@ -368,6 +400,28 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
             if o.type:
                 for trigger in get_type_triggers(o.type):
                     self.add_dependency(trigger)
+        if self.options and self.options.logical_deps and o.unanalyzed_type is None:
+            # Special case: for definitions without an explicit type like this:
+            #     x = func(...)
+            # we add a logical dependency <func> -> <x>, because if `func` is not annotated,
+            # then it will make all points of use of `x` unchecked.
+            if (isinstance(rvalue, CallExpr) and isinstance(rvalue.callee, RefExpr)
+                    and rvalue.callee.fullname is not None):
+                fname = None  # type: Optional[str]
+                if isinstance(rvalue.callee.node, TypeInfo):
+                    # use actual __init__ as a dependency source
+                    init = rvalue.callee.node.get('__init__')
+                    if init and isinstance(init.node, FuncBase):
+                        fname = init.node.fullname()
+                else:
+                    fname = rvalue.callee.fullname
+                if fname is None:
+                    return
+                for lv in o.lvalues:
+                    if isinstance(lv, RefExpr) and lv.fullname and lv.is_new_def:
+                        if lv.kind == LDEF:
+                            return  # local definitions don't generate logical deps
+                        self.add_dependency(make_trigger(fname), make_trigger(lv.fullname))
 
     def process_lvalue(self, lvalue: Expression) -> None:
         """Generate additional dependencies for an lvalue."""
@@ -815,7 +869,8 @@ def has_user_bases(info: TypeInfo) -> bool:
 
 def dump_all_dependencies(modules: Dict[str, MypyFile],
                           type_map: Dict[Expression, Type],
-                          python_version: Tuple[int, int]) -> None:
+                          python_version: Tuple[int, int],
+                          options: Options) -> None:
     """Generate dependencies for all interesting modules and print them to stdout."""
     all_deps = {}  # type: Dict[str, Set[str]]
     for id, node in modules.items():
@@ -824,7 +879,7 @@ def dump_all_dependencies(modules: Dict[str, MypyFile],
         if id in ('builtins', 'typing') or '/typeshed/' in node.path:
             continue
         assert id == node.fullname()
-        deps = get_dependencies(node, type_map, python_version)
+        deps = get_dependencies(node, type_map, python_version, options)
         for trigger, targets in deps.items():
             all_deps.setdefault(trigger, set()).update(targets)
     TypeState.add_all_protocol_deps(all_deps)

mypy/test/testdeps.py

@@ -15,7 +15,7 @@
 from mypy.server.deps import get_dependencies
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
-from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.helpers import assert_string_arrays_equal, parse_options
 from mypy.types import Type
 from mypy.typestate import TypeState
 
@@ -41,7 +41,13 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             python_version = defaults.PYTHON2_VERSION
         else:
             python_version = defaults.PYTHON3_VERSION
-        messages, files, type_map = self.build(src, python_version)
+        options = parse_options(src, testcase, incremental_step=1)
+        options.use_builtins_fixtures = True
+        options.show_traceback = True
+        options.cache_dir = os.devnull
+        options.python_version = python_version
+        options.export_types = True
+        messages, files, type_map = self.build(src, options)
         a = messages
         if files is None or type_map is None:
             if not a:
@@ -53,7 +59,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                                                                            'typing',
                                                                            'mypy_extensions',
                                                                            'enum'):
-                    new_deps = get_dependencies(files[module], type_map, python_version)
+                    new_deps = get_dependencies(files[module], type_map, python_version, options)
                     for source in new_deps:
                         deps[source].update(new_deps[source])
 
@@ -75,15 +81,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
 
     def build(self,
               source: str,
-              python_version: Tuple[int, int]) -> Tuple[List[str],
-                                                        Optional[Dict[str, MypyFile]],
-                                                        Optional[Dict[Expression, Type]]]:
-        options = Options()
-        options.use_builtins_fixtures = True
-        options.show_traceback = True
-        options.cache_dir = os.devnull
-        options.python_version = python_version
-        options.export_types = True
+              options: Options) -> Tuple[List[str],
+                                         Optional[Dict[str, MypyFile]],
+                                         Optional[Dict[Expression, Type]]]:
         try:
             result = build.build(sources=[BuildSource('main', None, source)],
                                  options=options,

test-data/unit/deps.test

@@ -1134,3 +1134,118 @@ def f(b: B) -> None:
 <m.B.__setitem__> -> m.f
 <m.B> -> <m.f>, m.B, m.f
 <m.g> -> m.f
+
+[case testLogicalDecorator]
+# flags: --logical-deps
+from mod import dec
+@dec
+def f() -> None:
+    pass
+[file mod.py]
+from typing import Callable
+def dec(f: Callable[[], None]) -> Callable[[], None]:
+    pass
+[out]
+<m.f> -> m
+<mod.dec> -> <m.f>, m
+
+[case testLogicalDecoratorWithArgs]
+# flags: --logical-deps
+from mod import dec
+@dec(str())
+def f() -> None:
+    pass
+[file mod.py]
+from typing import Callable
+def dec(arg: str) -> Callable[[Callable[[], None]], Callable[[], None]]:
+    pass
+[out]
+<m.f> -> m
+<mod.dec> -> <m.f>, m
+
+[case testLogicalDecoratorMember]
+# flags: --logical-deps
+import mod
+@mod.dec
+def f() -> None:
+    pass
+[file mod.py]
+from typing import Callable
+def dec(f: Callable[[], None]) -> Callable[[], None]:
+    pass
+[out]
+<m.f> -> m
+<mod.dec> -> <m.f>, m
+<mod> -> m
+
+[case testLogicalDefinition]
+# flags: --logical-deps
+from mod import func
+b = func()
+[file mod.py]
+def func() -> int:
+    pass
+[out]
+<m.b> -> m
+<mod.func> -> <m.b>, m
+
+[case testLogicalDefinitionIrrelevant]
+# flags: --logical-deps
+from mod import func
+def outer() -> None:
+    a = func()
+b: int = func()
+c = int()
+c = func()
+[file mod.py]
+def func() -> int:
+    pass
+[out]
+<m.b> -> m
+<m.c> -> m
+<mod.func> -> m, m.outer
+
+[case testLogicalDefinitionMember]
+# flags: --logical-deps
+import mod
+b = mod.func()
+[file mod.py]
+def func() -> int:
+    pass
+[out]
+<m.b> -> m
+<mod.func> -> <m.b>, m
+<mod> -> m
+
+[case testLogicalDefinitionClass]
+# flags: --logical-deps
+from mod import Cls
+b = Cls()
+[file mod.py]
+class Base:
+    def __init__(self) -> None: pass
+class Cls(Base): pass
+[out]
+<m.b> -> m
+<mod.Base.__init__> -> <m.b>
+<mod.Cls.__init__> -> m
+<mod.Cls.__new__> -> m
+<mod.Cls> -> <m.b>, m
+
+[case testLogicalBaseAttribute]
+# flags: --logical-deps
+from mod import C
+class D(C):
+    x: int
+[file mod.py]
+class C:
+    x: int
+    y: int
+[out]
+<m.D.x> -> m
+<m.D> -> m.D
+<mod.C.(abstract)> -> <m.D.__init__>, m
+<mod.C.__init__> -> <m.D.__init__>
+<mod.C.__new__> -> <m.D.__new__>
+<mod.C.x> -> <m.D.x>
+<mod.C> -> m, m.D