Enforces having unique test ids in a single namespace (#11687)

I found lots of duplicate test names. It might hide important errors, be hard to reproduce 
or rerun.

Refs #11662

Co-authored-by: Shantanu <[email protected]>

Author: sobolevn

mypy/test/data.py

@@ -578,8 +578,13 @@ def split_test_cases(parent: 'DataFileCollector', suite: 'DataSuite',
                      data,
                      flags=re.DOTALL | re.MULTILINE)
     line_no = cases[0].count('\n') + 1
+    test_names = set()
     for i in range(1, len(cases), NUM_GROUPS):
         name, writescache, only_when, platform_flag, skip, xfail, data = cases[i:i + NUM_GROUPS]
+        if name in test_names:
+            raise RuntimeError('Found a duplicate test name "{}" in {} on line {}'.format(
+                name, parent.name, line_no,
+            ))
         platform = platform_flag[1:] if platform_flag else None
         yield DataDrivenTestCase.from_parent(
             parent=parent,
@@ -596,6 +601,9 @@ def split_test_cases(parent: 'DataFileCollector', suite: 'DataSuite',
         )
         line_no += data.count('\n') + 1
 
+        # Record existing tests to prevent duplicates:
+        test_names.update({name})
+
 
 class DataSuiteCollector(pytest.Class):
     def collect(self) -> Iterator['DataFileCollector']:

test-data/unit/check-abstract.test

@@ -600,7 +600,7 @@ a.f(B()) # E: No overload variant of "f" of "A" matches argument type "B" \
          # N:     def f(self, x: int) -> int \
          # N:     def f(self, x: str) -> str
 
-[case testOverloadedAbstractMethodVariantMissingDecorator1]
+[case testOverloadedAbstractMethodVariantMissingDecorator0]
 from foo import *
 [file foo.pyi]
 from abc import abstractmethod, ABCMeta

test-data/unit/check-attr.test

@@ -502,7 +502,7 @@ reveal_type(sub_str.attr)  # N: Revealed type is "builtins.str*"
 [builtins fixtures/bool.pyi]
 
 
-[case testAttrsGenericInheritance]
+[case testAttrsGenericInheritance2]
 from typing import Generic, TypeVar
 import attr
 

test-data/unit/check-class-namedtuple.test

@@ -270,7 +270,7 @@ a = A(B())
 a = A(1)  # E: Argument 1 to "A" has incompatible type "int"; expected "B"
 [builtins fixtures/tuple.pyi]
 
-[case testNewNamedTupleProperty]
+[case testNewNamedTupleProperty36]
 # flags: --python-version 3.6
 from typing import NamedTuple
 

test-data/unit/check-classes.test

@@ -6463,7 +6463,7 @@ class C(B):
 [out]
 main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testIgnorePrivateMethodsTypeCheck]
+[case testIgnorePrivateMethodsTypeCheck2]
 class A:
     def __foo_(self) -> int: ...
 class B:

test-data/unit/check-functions.test

@@ -1485,7 +1485,7 @@ x = None # type: Any
 if x:
     def f(x: int) -> None: pass # E: All conditional function variants must have identical signatures
 
-[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition1]
+[case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition2]
 from typing import Any
 def f(x: int) -> None: pass # N: "f" defined here
 x = None # type: Any
@@ -2212,7 +2212,7 @@ from typing import Callable
 class A:
     def f(self) -> None:
         # In particular, test that the error message contains "g" of "A".
-        self.g() # E: Too few arguments for "g" of "A"   
+        self.g() # E: Too few arguments for "g" of "A"
         self.g(1)
     @dec
     def g(self, x: str) -> None: pass

test-data/unit/check-generics.test

@@ -52,7 +52,7 @@ class C: pass
 [out]
 main:8: error: Incompatible types in assignment (expression has type "C", variable has type "B")
 
-[case testGenericMemberVariable]
+[case testGenericMemberVariable2]
 from typing import TypeVar, Generic
 T = TypeVar('T')
 a, b, c = None, None, None # type: (A[B], B, C)
@@ -2118,7 +2118,7 @@ class B(A[T], Generic[T, S]):
 reveal_type(B.foo)  # N: Revealed type is "def [T, S] () -> Tuple[T`1, __main__.B[T`1, S`2]]"
 [builtins fixtures/classmethod.pyi]
 
-[case testGenericClassAlternativeConstructorPrecise]
+[case testGenericClassAlternativeConstructorPrecise2]
 from typing import Generic, TypeVar, Type, Tuple, Any
 
 T = TypeVar('T')

test-data/unit/check-inference.test

@@ -2614,7 +2614,7 @@ class C(B):
 reveal_type(B.x)  # N: Revealed type is "None"
 reveal_type(C.x)  # N: Revealed type is "None"
 
-[case testLocalPartialTypesWithInheritance2]
+[case testLocalPartialTypesWithInheritance3]
 # flags: --local-partial-types
 from typing import Optional
 
@@ -2652,7 +2652,7 @@ class C:
     def f(self, x) -> None:
         C.a.y  # E: Item "None" of "Optional[Any]" has no attribute "y"
 
-[case testLocalPartialTypesAccessPartialNoneAttribute]
+[case testLocalPartialTypesAccessPartialNoneAttribute2]
 # flags: --local-partial-types
 class C:
     a = None  # E: Need type annotation for "a"

test-data/unit/check-modules.test

@@ -1,7 +1,7 @@
 -- Type checker test cases dealing with modules and imports.
 -- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages).
 
-[case testAccessImportedDefinitions]
+[case testAccessImportedDefinitions0]
 import m
 import typing
 m.f()           # E: Missing positional argument "a" in call to "f"
@@ -14,7 +14,7 @@ class A: pass
 def f(a: A) -> None: pass
 x = A()
 
-[case testAccessImportedDefinitions]
+[case testAccessImportedDefinitions1]
 import m
 import typing
 m.f(object()) # E: Argument 1 to "f" has incompatible type "object"; expected "A"

test-data/unit/check-multiple-inheritance.test

@@ -502,7 +502,7 @@ class A(Base1, Base2):
 [out]
 main:10: error: Incompatible types in assignment (expression has type "GenericBase[Base2]", base class "Base1" defined the type as "GenericBase[Base1]")
 
-[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType]
+[case testMultipleInheritance_NestedVariableOverriddenWithCompatibleType2]
 from typing import TypeVar, Generic
 T = TypeVar('T', covariant=True)
 class GenericBase(Generic[T]):