From 7a51cf24903340a4520a9b9b73967fc64210cfaf Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 13 May 2019 19:48:24 -0700 Subject: [PATCH 1/6] Stop checking at bare top-level "# type: ignore" comments. --- mypy/fastparse.py | 65 ++++++++++++++++++++++++++++++++++++---------- mypy/fastparse2.py | 34 +++++++++++++++++------- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 9d91601d5347..db377db80fb7 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2,7 +2,7 @@ import sys from typing import ( - Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List, overload + Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List, overload, Set ) MYPY = False if MYPY: @@ -258,7 +258,7 @@ def __init__(self, self.is_stub = is_stub self.errors = errors - self.extra_type_ignores = [] # type: List[int] + self.type_ignores = set() # type: Set[int] # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] @@ -294,11 +294,49 @@ def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: res.append(exp) return res - def translate_stmt_list(self, l: Sequence[AST]) -> List[Statement]: + def get_line(self, n: ast3.stmt) -> int: + if (isinstance(n, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef)) + and n.decorator_list): + return n.decorator_list[0].lineno + return n.lineno + + def translate_stmt_list(self, + l: Sequence[ast3.stmt], + module: bool = False) -> List[Statement]: + + # A "# type: ignore" comment before the first statement of a module + # ignores the whole module: + + if module and l and self.type_ignores and min(self.type_ignores) < self.get_line(l[0]): + self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores)) + b = Block(self.fix_function_overloads(self.translate_stmt_list(l))) + b.is_unreachable = True + return [b] + res = [] # type: List[Statement] - for e in l: + line = 0 + + for i, e in enumerate(l): + + # In Python 3.8, a "# type: ignore" comment between statements at + # the top level of a module skips checking for everything else: + + if module and sys.version_info >= (3, 8): + + ignores = set(range(line + 1, self.get_line(e))) & self.type_ignores + + if ignores: + self.errors.used_ignored_lines[self.errors.file].add(min(ignores)) + b = Block(self.fix_function_overloads(self.translate_stmt_list(l[i:]))) + b.is_unreachable = True + res.append(b) + return res + + line = e.end_lineno if e.end_lineno is not None else e.lineno + stmt = self.visit(e) res.append(stmt) + return res op_map = { @@ -403,13 +441,12 @@ def translate_module_id(self, id: str) -> str: return id def visit_Module(self, mod: ast3.Module) -> MypyFile: - body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - ignores = [ti.lineno for ti in mod.type_ignores] - ignores.extend(self.extra_type_ignores) + self.type_ignores = {ti.lineno for ti in mod.type_ignores} + body = self.fix_function_overloads(self.translate_stmt_list(mod.body, module=True)) return MypyFile(body, self.imports, False, - set(ignores), + set(self.type_ignores), ) # --- stmt --- @@ -615,7 +652,7 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int, elif type_comment is not None: extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(arg.lineno) + self.type_ignores.add(arg.lineno) return Argument(Var(arg.arg), arg_type, self.visit(default), kind) @@ -673,7 +710,7 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: if n.type_comment is not None: extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: typ = None s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False) @@ -707,7 +744,7 @@ def visit_For(self, n: ast3.For) -> ForStmt: if n.type_comment is not None: extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -722,7 +759,7 @@ def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt: if n.type_comment is not None: extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -753,7 +790,7 @@ def visit_With(self, n: ast3.With) -> WithStmt: if n.type_comment is not None: extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: target_type = None node = WithStmt([self.visit(i.context_expr) for i in n.items], @@ -767,7 +804,7 @@ def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt: if n.type_comment is not None: extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: target_type = None s = WithStmt([self.visit(i.context_expr) for i in n.items], diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index b4b831aa9e74..ae9d1694dfd5 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -16,7 +16,7 @@ """ import sys -from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List +from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List, Set MYPY = False if MYPY: import typing # for typing.Type, which conflicts with types.Type @@ -163,7 +163,7 @@ def __init__(self, # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] - self.extra_type_ignores = [] # type: List[int] + self.type_ignores = set() # type: Set[int] def fail(self, msg: str, line: int, column: int, blocker: bool = True) -> None: if blocker or not self.options.ignore_errors: @@ -193,7 +193,24 @@ def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: res.append(exp) return res - def translate_stmt_list(self, l: Sequence[AST]) -> List[Statement]: + def get_line(self, n: ast27.stmt) -> int: + if isinstance(n, (ast27.ClassDef, ast27.FunctionDef)) and n.decorator_list: + return n.decorator_list[0].lineno + return n.lineno + + def translate_stmt_list(self, + l: Sequence[ast27.stmt], + module: bool = False) -> List[Statement]: + + # A "# type: ignore" comment before the first statement of a module + # ignores the whole module: + + if module and l and self.type_ignores and min(self.type_ignores) < self.get_line(l[0]): + self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores)) + b = Block(self.fix_function_overloads(self.translate_stmt_list(l))) + b.is_unreachable = True + return [b] + res = [] # type: List[Statement] for e in l: stmt = self.visit(e) @@ -304,13 +321,12 @@ def translate_module_id(self, id: str) -> str: return id def visit_Module(self, mod: ast27.Module) -> MypyFile: + self.type_ignores = {ti.lineno for ti in mod.type_ignores} body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - ignores = [ti.lineno for ti in mod.type_ignores] - ignores.extend(self.extra_type_ignores) return MypyFile(body, self.imports, False, - set(ignores), + set(self.type_ignores), ) # --- stmt --- @@ -558,7 +574,7 @@ def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) stmt = AssignmentStmt(self.translate_expr_list(n.targets), self.visit(n.value), @@ -578,7 +594,7 @@ def visit_For(self, n: ast27.For) -> ForStmt: extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: typ = None stmt = ForStmt(self.visit(n.target), @@ -608,7 +624,7 @@ def visit_With(self, n: ast27.With) -> WithStmt: extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, assume_str_is_unicode=self.unicode_literals) if extra_ignore: - self.extra_type_ignores.append(n.lineno) + self.type_ignores.add(n.lineno) else: typ = None stmt = WithStmt([self.visit(n.context_expr)], From aa11089c20fad5eb242d0af07a0d80d99a2793ac Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 13 May 2019 19:48:44 -0700 Subject: [PATCH 2/6] Test new "# type: ignore" behavior. --- test-data/unit/check-38.test | 5 ++++ test-data/unit/check-ignore.test | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/test-data/unit/check-38.test b/test-data/unit/check-38.test index 16f279545bc8..376fea612386 100644 --- a/test-data/unit/check-38.test +++ b/test-data/unit/check-38.test @@ -106,3 +106,8 @@ def g(x: int): ... / 0 # type: ignore ) # type: ignore # E: unused 'type: ignore' comment + +[case testIgnoreRestOfModule] +ERROR # E: Name 'ERROR' is not defined +# type: ignore +IGNORE diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test index c17befa6e0f4..3992819a4648 100644 --- a/test-data/unit/check-ignore.test +++ b/test-data/unit/check-ignore.test @@ -218,3 +218,43 @@ def f() -> None: pass [case testCannotIgnoreBlockingError] yield # type: ignore # E: 'yield' outside function + +[case testIgnoreWholeModule1] +# flags: --warn-unused-ignores +# type: ignore +IGNORE # type: ignore # E: unused 'type: ignore' comment + +[case testIgnoreWholeModule2] +# type: ignore +if True: + IGNORE + +[case testIgnoreWholeModule3] +# type: ignore +@d +class C: ... +IGNORE + +[case testIgnoreWholeModule4] +# type: ignore +@d + +def f(): ... +IGNORE + +[case testDontIgnoreWholeModule1] +if True: + # type: ignore + ERROR # E: Name 'ERROR' is not defined +ERROR # E: Name 'ERROR' is not defined + +[case testDontIgnoreWholeModule2] +@d # type: ignore +class C: ... +ERROR # E: Name 'ERROR' is not defined + +[case testDontIgnoreWholeModule3] +@d # type: ignore + +def f(): ... +ERROR # E: Name 'ERROR' is not defined From ce54ab48d145bdc6d0e0c9bcffe6cc1631d6ef55 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 13 May 2019 20:29:42 -0700 Subject: [PATCH 3/6] Appease mypy. --- mypy/fastparse.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index db377db80fb7..cc4c7e6cafaa 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -318,21 +318,22 @@ def translate_stmt_list(self, for i, e in enumerate(l): - # In Python 3.8, a "# type: ignore" comment between statements at - # the top level of a module skips checking for everything else: + if module: # and... + if sys.version_info >= (3, 8): - if module and sys.version_info >= (3, 8): + # In Python 3.8, a "# type: ignore" comment between statements at + # the top level of a module skips checking for everything else: - ignores = set(range(line + 1, self.get_line(e))) & self.type_ignores + ignores = set(range(line + 1, self.get_line(e))) & self.type_ignores - if ignores: - self.errors.used_ignored_lines[self.errors.file].add(min(ignores)) - b = Block(self.fix_function_overloads(self.translate_stmt_list(l[i:]))) - b.is_unreachable = True - res.append(b) - return res + if ignores: + self.errors.used_ignored_lines[self.errors.file].add(min(ignores)) + b = Block(self.fix_function_overloads(self.translate_stmt_list(l[i:]))) + b.is_unreachable = True + res.append(b) + return res - line = e.end_lineno if e.end_lineno is not None else e.lineno + line = e.end_lineno if e.end_lineno is not None else e.lineno stmt = self.visit(e) res.append(stmt) From 45676615883991d7ebded58b2ddd135565527261 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 14 May 2019 13:19:57 -0700 Subject: [PATCH 4/6] Add docs for new # type: ignore usage. --- docs/source/common_issues.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index a4da8ec13546..e3e42c6102b1 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -131,6 +131,30 @@ The second line is now fine, since the ignore comment causes the name if we did have a stub available for ``frobnicate`` then mypy would ignore the ``# type: ignore`` comment and typecheck the stub as usual. +A ``# type: ignore`` comment at the top of a module (before any statements, +including imports or docstrings) has the effect of ignoring the *entire* module. + +.. code-block:: python + + # type: ignore + import frobnicate + frobnicate.start() + +When running mypy with Python 3.8 or later, a ``# type: ignore`` comment +anywhere at the top indentation level of a module will skip type checking for +all remaining lines in the file. + +.. code-block:: python + + """Docstring.""" + + import spam # These imports are still checked! + import eggs + + # type: ignore + import frobnicate + frobnicate.start() + Another option is to explicitly annotate values with type ``Any`` -- mypy will let you perform arbitrary operations on ``Any`` values. Sometimes there is no more precise type you can use for a From c95b98ddf69d1f8b6ef2de0e2a67f73bbd7a7cca Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 14 May 2019 13:21:02 -0700 Subject: [PATCH 5/6] Code cleanup from GvR's first review. --- mypy/fastparse.py | 55 +++++++++++++++++++++++----------------------- mypy/fastparse2.py | 31 +++++++++++++------------- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index cc4c7e6cafaa..788667105167 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -294,49 +294,48 @@ def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: res.append(exp) return res - def get_line(self, n: ast3.stmt) -> int: - if (isinstance(n, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef)) - and n.decorator_list): - return n.decorator_list[0].lineno - return n.lineno + def get_lineno(self, node: Union[ast3.expr, ast3.stmt]) -> int: + if (isinstance(node, (ast3.AsyncFunctionDef, ast3.ClassDef, ast3.FunctionDef)) + and node.decorator_list): + return node.decorator_list[0].lineno + return node.lineno def translate_stmt_list(self, - l: Sequence[ast3.stmt], - module: bool = False) -> List[Statement]: - + stmts: Sequence[ast3.stmt], + ismodule: bool = False) -> List[Statement]: # A "# type: ignore" comment before the first statement of a module # ignores the whole module: - - if module and l and self.type_ignores and min(self.type_ignores) < self.get_line(l[0]): + if (ismodule and stmts and self.type_ignores + and min(self.type_ignores) < self.get_lineno(stmts[0])): self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores)) - b = Block(self.fix_function_overloads(self.translate_stmt_list(l))) - b.is_unreachable = True - return [b] + block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + block.is_unreachable = True + return [block] res = [] # type: List[Statement] line = 0 - for i, e in enumerate(l): + for i, stmt in enumerate(stmts): - if module: # and... + if ismodule: # This line needs to be split for mypy to branch on version: if sys.version_info >= (3, 8): - - # In Python 3.8, a "# type: ignore" comment between statements at - # the top level of a module skips checking for everything else: - - ignores = set(range(line + 1, self.get_line(e))) & self.type_ignores + # In Python 3.8+ (we need end_lineno), a "# type: ignore" comment + # between statements at the top level of a module skips checking + # for everything else: + ignores = set(range(line + 1, self.get_lineno(stmt))) & self.type_ignores if ignores: self.errors.used_ignored_lines[self.errors.file].add(min(ignores)) - b = Block(self.fix_function_overloads(self.translate_stmt_list(l[i:]))) - b.is_unreachable = True - res.append(b) + rest = self.fix_function_overloads(self.translate_stmt_list(stmts[i:])) + block = Block(rest) + block.is_unreachable = True + res.append(block) return res - line = e.end_lineno if e.end_lineno is not None else e.lineno + line = stmt.end_lineno if stmt.end_lineno is not None else stmt.lineno - stmt = self.visit(e) - res.append(stmt) + node = self.visit(stmt) + res.append(node) return res @@ -443,11 +442,11 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast3.Module) -> MypyFile: self.type_ignores = {ti.lineno for ti in mod.type_ignores} - body = self.fix_function_overloads(self.translate_stmt_list(mod.body, module=True)) + body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True)) return MypyFile(body, self.imports, False, - set(self.type_ignores), + self.type_ignores, ) # --- stmt --- diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index ae9d1694dfd5..0cea7706dacb 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -193,29 +193,28 @@ def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: res.append(exp) return res - def get_line(self, n: ast27.stmt) -> int: - if isinstance(n, (ast27.ClassDef, ast27.FunctionDef)) and n.decorator_list: - return n.decorator_list[0].lineno - return n.lineno + def get_lineno(self, node: Union[ast27.expr, ast27.stmt]) -> int: + if isinstance(node, (ast27.ClassDef, ast27.FunctionDef)) and node.decorator_list: + return node.decorator_list[0].lineno + return node.lineno def translate_stmt_list(self, - l: Sequence[ast27.stmt], + stmts: Sequence[ast27.stmt], module: bool = False) -> List[Statement]: - # A "# type: ignore" comment before the first statement of a module # ignores the whole module: - - if module and l and self.type_ignores and min(self.type_ignores) < self.get_line(l[0]): + if (module and stmts and self.type_ignores + and min(self.type_ignores) < self.get_lineno(stmts[0])): self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores)) - b = Block(self.fix_function_overloads(self.translate_stmt_list(l))) - b.is_unreachable = True - return [b] + block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + block.is_unreachable = True + return [block] res = [] # type: List[Statement] - for e in l: - stmt = self.visit(e) - assert isinstance(stmt, Statement) - res.append(stmt) + for stmt in stmts: + node = self.visit(stmt) + assert isinstance(node, Statement) + res.append(node) return res op_map = { @@ -326,7 +325,7 @@ def visit_Module(self, mod: ast27.Module) -> MypyFile: return MypyFile(body, self.imports, False, - set(self.type_ignores), + self.type_ignores, ) # --- stmt --- From 07cc3152f4b4dd9a3e5000264ac58785b3486f6f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 15 May 2019 16:13:44 -0700 Subject: [PATCH 6/6] Break out a new section for this feature. --- docs/source/common_issues.rst | 55 ++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index e3e42c6102b1..c67c6cb728d3 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -131,14 +131,37 @@ The second line is now fine, since the ignore comment causes the name if we did have a stub available for ``frobnicate`` then mypy would ignore the ``# type: ignore`` comment and typecheck the stub as usual. +Another option is to explicitly annotate values with type ``Any`` -- +mypy will let you perform arbitrary operations on ``Any`` +values. Sometimes there is no more precise type you can use for a +particular value, especially if you use dynamic Python features +such as ``__getattr__``: + +.. code-block:: python + + class Wrapper: + ... + def __getattr__(self, a: str) -> Any: + return getattr(self._wrapped, a) + +Finally, you can create a stub file (``.pyi``) for a file that +generates spurious errors. Mypy will only look at the stub file +and ignore the implementation, since stub files take precedence +over ``.py`` files. + +Ignoring a whole file +--------------------- + A ``# type: ignore`` comment at the top of a module (before any statements, including imports or docstrings) has the effect of ignoring the *entire* module. .. code-block:: python # type: ignore - import frobnicate - frobnicate.start() + + import foo + + foo.bar() When running mypy with Python 3.8 or later, a ``# type: ignore`` comment anywhere at the top indentation level of a module will skip type checking for @@ -148,30 +171,16 @@ all remaining lines in the file. """Docstring.""" - import spam # These imports are still checked! + import spam import eggs - - # type: ignore - import frobnicate - frobnicate.start() + import foo -Another option is to explicitly annotate values with type ``Any`` -- -mypy will let you perform arbitrary operations on ``Any`` -values. Sometimes there is no more precise type you can use for a -particular value, especially if you use dynamic Python features -such as ``__getattr__``: + eggs.fry() # This code is still checked! -.. code-block:: python - - class Wrapper: - ... - def __getattr__(self, a: str) -> Any: - return getattr(self._wrapped, a) - -Finally, you can create a stub file (``.pyi``) for a file that -generates spurious errors. Mypy will only look at the stub file -and ignore the implementation, since stub files take precedence -over ``.py`` files. + # type: ignore + + foo.bar() # Mypy skips over everything here. + foo.baz() Unexpected errors about 'None' and/or 'Optional' types ------------------------------------------------------