Use package_data to ship typeshed and xml files (#5517)

Data files have caused many issues where mypy cannot find typeshed.
Using package_data allows us to find typeshed relative to the mypy
package itself, which avoids a lot of fragile path computation.

Fixes #5307
Fixes #5319

Author: emmatyping

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "typeshed"]
-	path = typeshed
+	path = mypy/typeshed
 	url = https://github.com/python/typeshed

MANIFEST.in

@@ -2,5 +2,7 @@ recursive-include scripts *
 recursive-include test-data *
 recursive-include extensions *
 recursive-include docs *
+recursive-include mypy/typeshed *.py *.pyi
+recursive-include mypy/xml *.xsd *.xslt *.css
 include mypy_self_check.ini
 include LICENSE

appveyor.yml

@@ -9,10 +9,7 @@ environment:
       PYTHON_ARCH: "64"
 
 install:
-    - "git config core.symlinks true"
-    - "git reset --hard"
-    - "git submodule update --init typeshed"
-    - "cd typeshed && git config core.symlinks true && git reset --hard && cd .."
+    - "git submodule update --init mypy/typeshed"
     - "%PYTHON%\\python.exe -m pip install -U setuptools tox"
     - "%PYTHON%\\python.exe -m tox -e py37 --notest"
 

mypy/build.py

@@ -20,6 +20,7 @@
 import hashlib
 import json
 import os.path
+import pathlib
 import re
 import site
 import stat
@@ -326,7 +327,7 @@ def _build(sources: List[BuildSource],
     # This seems the most reasonable place to tune garbage collection.
     gc.set_threshold(50000)
 
-    data_dir = default_data_dir(bin_dir)
+    data_dir = default_data_dir()
     fscache = fscache or FileSystemCache()
 
     search_paths = compute_search_paths(sources, options, data_dir, fscache, alt_lib_path)
@@ -365,68 +366,9 @@ def _build(sources: List[BuildSource],
         reports.finish()
 
 
-def default_data_dir(bin_dir: Optional[str]) -> str:
-    """Returns directory containing typeshed directory
-
-    Args:
-      bin_dir: directory containing the mypy script
-    """
-    if not bin_dir:
-        if os.name == 'nt':
-            prefixes = [os.path.join(sys.prefix, 'Lib')]
-            try:
-                prefixes.append(os.path.join(site.getuserbase(), 'lib'))
-            except AttributeError:
-                # getuserbase in not available in virtualenvs
-                prefixes.append(os.path.join(get_python_lib(), 'lib'))
-            for parent in prefixes:
-                    data_dir = os.path.join(parent, 'mypy')
-                    if os.path.exists(data_dir):
-                        return data_dir
-        mypy_package = os.path.dirname(__file__)
-        parent = os.path.dirname(mypy_package)
-        if (os.path.basename(parent) == 'site-packages' or
-                os.path.basename(parent) == 'dist-packages'):
-            # Installed in site-packages or dist-packages, but invoked with python3 -m mypy;
-            # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py
-            # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian)
-            # or .../blah/lib64/python3.N/dist-packages/mypy/build.py (Gentoo)
-            # or .../blah/lib/site-packages/mypy/build.py (Windows)
-            # blah may be a virtualenv or /usr/local.  We want .../blah/lib/mypy.
-            lib = parent
-            for i in range(2):
-                lib = os.path.dirname(lib)
-                if os.path.basename(lib) in ('lib', 'lib32', 'lib64'):
-                    return os.path.join(os.path.dirname(lib), 'lib/mypy')
-        subdir = os.path.join(parent, 'lib', 'mypy')
-        if os.path.isdir(subdir):
-            # If installed via buildout, the __file__ is
-            # somewhere/mypy/__init__.py and what we want is
-            # somewhere/lib/mypy.
-            return subdir
-        # Default to directory containing this file's parent.
-        return parent
-    base = os.path.basename(bin_dir)
-    dir = os.path.dirname(bin_dir)
-    if (sys.platform == 'win32' and base.lower() == 'scripts'
-            and not os.path.isdir(os.path.join(dir, 'typeshed'))):
-        # Installed, on Windows.
-        return os.path.join(dir, 'Lib', 'mypy')
-    elif base == 'scripts':
-        # Assume that we have a repo check out or unpacked source tarball.
-        return dir
-    elif base == 'bin':
-        # Installed to somewhere (can be under /usr/local or anywhere).
-        return os.path.join(dir, 'lib', 'mypy')
-    elif base == 'python3':
-        # Assume we installed python3 with brew on os x
-        return os.path.join(os.path.dirname(dir), 'lib', 'mypy')
-    elif dir.endswith('python-exec'):
-        # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink.
-        return os.path.join(os.path.dirname(dir), 'mypy')
-    else:
-        # Don't know where to find the data files!
-        raise RuntimeError("Broken installation: can't determine base dir")
+def default_data_dir() -> str:
+    """Returns directory containing typeshed directory."""
+    return os.path.dirname(__file__)
 
 
 def mypy_path() -> List[str]:

typeshed renamed to mypy/typeshed

@@ -22,7 +22,7 @@ exclude =
   # fixtures have their own .pyi-specific configuration
   test-data/*,
   # typeshed has its own .pyi-specific configuration
-  typeshed/*,
+  mypy/typeshed/*,
   # flake8 might be started when there's still examples in the temp dir
   tmp-test-dirs/*
   .tox