From 204887da03f1683d6338e5f79257f9892506f79f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 19 Dec 2019 15:54:58 +0800 Subject: [PATCH 1/6] Apply changes from bundled appdirs to vendored * Convert Windows app data directory values to bytes on Python 2, so the output type is consistent across platforms (pypa/pip#4000) * Also look in ~/.config for user config on macOS (pypa/pip#4100) * Remove pywin32 dependency, only use ctypes and winreg for directory lookup on Windows (pypa/pip#2467) * Always use os.path.join() instead of os.sep.join() so cross-platform tests work as expected (pypa/pip#3275) --- ...0cda98-2240-11ea-9951-00e04c3600d8.trivial | 0 src/pip/_internal/utils/appdirs.py | 265 +----------------- src/pip/_vendor/appdirs.py | 36 ++- tests/unit/test_appdirs.py | 57 ++-- .../vendoring/patches/appdirs.patch | 88 +++++- 5 files changed, 160 insertions(+), 286 deletions(-) create mode 100644 news/050cda98-2240-11ea-9951-00e04c3600d8.trivial diff --git a/news/050cda98-2240-11ea-9951-00e04c3600d8.trivial b/news/050cda98-2240-11ea-9951-00e04c3600d8.trivial new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pip/_internal/utils/appdirs.py b/src/pip/_internal/utils/appdirs.py index 06cd8314a5c..cce1e293c00 100644 --- a/src/pip/_internal/utils/appdirs.py +++ b/src/pip/_internal/utils/appdirs.py @@ -1,19 +1,17 @@ """ -This code was taken from https://github.com/ActiveState/appdirs and modified -to suit our purposes. -""" +This code wraps the vendored appdirs module to so the return values are +compatible for the current pip code base. -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False +The intention is to rewrite current usages guradually, keeping the tests pass, +and eventually drop this after all usages are changed. +""" from __future__ import absolute_import import os -import sys -from pip._vendor.six import PY2, text_type +from pip._vendor import appdirs as _appdirs -from pip._internal.utils.compat import WINDOWS, expanduser from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -22,255 +20,22 @@ def user_cache_dir(appname): # type: (str) -> str - r""" - Return full path to the user-specific cache dir for this application. - - "appname" is the name of application. - - Typical user cache directories are: - macOS: ~/Library/Caches/ - Unix: ~/.cache/ (XDG default) - Windows: C:\Users\\AppData\Local\\Cache - - On Windows the only suggestion in the MSDN docs is that local settings go - in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the - non-roaming app data dir (the default returned by `user_data_dir`). Apps - typically put cache data somewhere *under* the given dir here. Some - examples: - ...\Mozilla\Firefox\Profiles\\Cache - ...\Acme\SuperApp\Cache\1.0 - - OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. - """ - if WINDOWS: - # Get the base path - path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) - - # When using Python 2, return paths as bytes on Windows like we do on - # other operating systems. See helper function docs for more details. - if PY2 and isinstance(path, text_type): - path = _win_path_to_bytes(path) - - # Add our app name and Cache directory to it - path = os.path.join(path, appname, "Cache") - elif sys.platform == "darwin": - # Get the base path - path = expanduser("~/Library/Caches") - - # Add our app name to it - path = os.path.join(path, appname) - else: - # Get the base path - path = os.getenv("XDG_CACHE_HOME", expanduser("~/.cache")) - - # Add our app name to it - path = os.path.join(path, appname) - - return path - - -def user_data_dir(appname, roaming=False): - # type: (str, bool) -> str - r""" - Return full path to the user-specific data dir for this application. - - "appname" is the name of application. - If None, just the system directory is returned. - "roaming" (boolean, default False) can be set True to use the Windows - roaming appdata directory. That means that for users on a Windows - network setup for roaming profiles, this user data will be - sync'd on login. See - - for a discussion of issues. - - Typical user data directories are: - macOS: ~/Library/Application Support/ - if it exists, else ~/.config/ - Unix: ~/.local/share/ # or in - $XDG_DATA_HOME, if defined - Win XP (not roaming): C:\Documents and Settings\\ ... - ...Application Data\ - Win XP (roaming): C:\Documents and Settings\\Local ... - ...Settings\Application Data\ - Win 7 (not roaming): C:\\Users\\AppData\Local\ - Win 7 (roaming): C:\\Users\\AppData\Roaming\ - - For Unix, we follow the XDG spec and support $XDG_DATA_HOME. - That means, by default "~/.local/share/". - """ - if WINDOWS: - const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" - path = os.path.join(os.path.normpath(_get_win_folder(const)), appname) - elif sys.platform == "darwin": - path = os.path.join( - expanduser('~/Library/Application Support/'), - appname, - ) if os.path.isdir(os.path.join( - expanduser('~/Library/Application Support/'), - appname, - ) - ) else os.path.join( - expanduser('~/.config/'), - appname, - ) - else: - path = os.path.join( - os.getenv('XDG_DATA_HOME', expanduser("~/.local/share")), - appname, - ) - - return path + return _appdirs.user_cache_dir(appname, appauthor=False) def user_config_dir(appname, roaming=True): # type: (str, bool) -> str - """Return full path to the user-specific config dir for this application. - - "appname" is the name of application. - If None, just the system directory is returned. - "roaming" (boolean, default True) can be set False to not use the - Windows roaming appdata directory. That means that for users on a - Windows network setup for roaming profiles, this user data will be - sync'd on login. See - - for a discussion of issues. - - Typical user data directories are: - macOS: same as user_data_dir - Unix: ~/.config/ - Win *: same as user_data_dir + return _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming) - For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. - That means, by default "~/.config/". - """ - if WINDOWS: - path = user_data_dir(appname, roaming=roaming) - elif sys.platform == "darwin": - path = user_data_dir(appname) - else: - path = os.getenv('XDG_CONFIG_HOME', expanduser("~/.config")) - path = os.path.join(path, appname) - return path +def user_data_dir(appname, roaming=False): + # type: (str, bool) -> str + return _appdirs.user_data_dir(appname, appauthor=False, roaming=roaming) -# for the discussion regarding site_config_dirs locations -# see def site_config_dirs(appname): # type: (str) -> List[str] - r"""Return a list of potential user-shared config dirs for this application. - - "appname" is the name of application. - - Typical user config directories are: - macOS: /Library/Application Support// - Unix: /etc or $XDG_CONFIG_DIRS[i]// for each value in - $XDG_CONFIG_DIRS - Win XP: C:\Documents and Settings\All Users\Application ... - ...Data\\ - Vista: (Fail! "C:\ProgramData" is a hidden *system* directory - on Vista.) - Win 7: Hidden, but writeable on Win 7: - C:\ProgramData\\ - """ - if WINDOWS: - path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) - pathlist = [os.path.join(path, appname)] - elif sys.platform == 'darwin': - pathlist = [os.path.join('/Library/Application Support', appname)] - else: - # try looking in $XDG_CONFIG_DIRS - xdg_config_dirs = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') - if xdg_config_dirs: - pathlist = [ - os.path.join(expanduser(x), appname) - for x in xdg_config_dirs.split(os.pathsep) - ] - else: - pathlist = [] - - # always look in /etc directly as well - pathlist.append('/etc') - - return pathlist - - -# -- Windows support functions -- - -def _get_win_folder_from_registry(csidl_name): - # type: (str) -> str - """ - This is a fallback technique at best. I'm not sure if using the - registry for this guarantees us the correct answer for all CSIDL_* - names. - """ - import _winreg - - shell_folder_name = { - "CSIDL_APPDATA": "AppData", - "CSIDL_COMMON_APPDATA": "Common AppData", - "CSIDL_LOCAL_APPDATA": "Local AppData", - }[csidl_name] - - key = _winreg.OpenKey( - _winreg.HKEY_CURRENT_USER, - r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" - ) - directory, _type = _winreg.QueryValueEx(key, shell_folder_name) - return directory - - -def _get_win_folder_with_ctypes(csidl_name): - # type: (str) -> str - # On Python 2, ctypes.create_unicode_buffer().value returns "unicode", - # which isn't the same as str in the annotation above. - csidl_const = { - "CSIDL_APPDATA": 26, - "CSIDL_COMMON_APPDATA": 35, - "CSIDL_LOCAL_APPDATA": 28, - }[csidl_name] - - buf = ctypes.create_unicode_buffer(1024) - windll = ctypes.windll # type: ignore - windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) - - # Downgrade to short path name if have highbit chars. See - # . - has_high_char = False - for c in buf: - if ord(c) > 255: - has_high_char = True - break - if has_high_char: - buf2 = ctypes.create_unicode_buffer(1024) - if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): - buf = buf2 - - # The type: ignore is explained under the type annotation for this function - return buf.value # type: ignore - - -if WINDOWS: - try: - import ctypes - _get_win_folder = _get_win_folder_with_ctypes - except ImportError: - _get_win_folder = _get_win_folder_from_registry - - -def _win_path_to_bytes(path): - """Encode Windows paths to bytes. Only used on Python 2. - - Motivation is to be consistent with other operating systems where paths - are also returned as bytes. This avoids problems mixing bytes and Unicode - elsewhere in the codebase. For more details and discussion see - . - - If encoding using ASCII and MBCS fails, return the original Unicode path. - """ - for encoding in ('ASCII', 'MBCS'): - try: - return path.encode(encoding) - except (UnicodeEncodeError, LookupError): - pass - return path + dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True) + if _appdirs.system == "linux2": + return dirval.split(os.pathsep) + return [dirval] diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py index 2bd39110281..92b80251e14 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -64,7 +64,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): for a discussion of issues. Typical user data directories are: - Mac OS X: ~/Library/Application Support/ + Mac OS X: ~/Library/Application Support/ # or ~/.config/, if the other does not exist Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined Win XP (not roaming): C:\Documents and Settings\\Application Data\\ Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ @@ -88,6 +88,10 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): path = os.path.expanduser('~/Library/Application Support/') if appname: path = os.path.join(path, appname) + if not os.path.isdir(path): + path = os.path.expanduser('~/.config/') + if appname: + path = os.path.join(path, appname) else: path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) if appname: @@ -150,7 +154,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): if appname: if version: appname = os.path.join(appname, version) - pathlist = [os.sep.join([x, appname]) for x in pathlist] + pathlist = [os.path.join(x, appname) for x in pathlist] if multipath: path = os.pathsep.join(pathlist) @@ -203,6 +207,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): return path +# for the discussion regarding site_config_dir locations +# see def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): r"""Return full path to the user-shared data dir for this application. @@ -245,7 +251,9 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) if appname: if version: appname = os.path.join(appname, version) - pathlist = [os.sep.join([x, appname]) for x in pathlist] + pathlist = [os.path.join(x, appname) for x in pathlist] + # always look in /etc directly as well + pathlist.append('/etc') if multipath: path = os.pathsep.join(pathlist) @@ -291,6 +299,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): if appauthor is None: appauthor = appname path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + # When using Python 2, return paths as bytes on Windows like we do on + # other operating systems. See helper function docs for more details. + if not PY3 and isinstance(path, unicode): + path = _win_path_to_bytes(path) if appname: if appauthor is not False: path = os.path.join(path, appauthor, appname) @@ -567,6 +579,24 @@ def _get_win_folder_with_jna(csidl_name): _get_win_folder = _get_win_folder_from_registry +def _win_path_to_bytes(path): + """Encode Windows paths to bytes. Only used on Python 2. + + Motivation is to be consistent with other operating systems where paths + are also returned as bytes. This avoids problems mixing bytes and Unicode + elsewhere in the codebase. For more details and discussion see + . + + If encoding using ASCII and MBCS fails, return the original Unicode path. + """ + for encoding in ('ASCII', 'MBCS'): + try: + return path.encode(encoding) + except (UnicodeEncodeError, LookupError): + pass + return path + + #---- self test code if __name__ == "__main__": diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py index 1ee68ef2f72..1a01464174f 100644 --- a/tests/unit/test_appdirs.py +++ b/tests/unit/test_appdirs.py @@ -4,6 +4,7 @@ import sys import pretend +from pip._vendor import appdirs as _appdirs from pip._internal.utils import appdirs @@ -16,12 +17,12 @@ def _get_win_folder(base): return "C:\\Users\\test\\AppData\\Local" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert (appdirs.user_cache_dir("pip") == @@ -29,7 +30,7 @@ def _get_win_folder(base): assert _get_win_folder.calls == [pretend.call("CSIDL_LOCAL_APPDATA")] def test_user_cache_dir_osx(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "darwin") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("HOME", "/home/test") monkeypatch.setattr(sys, "platform", "darwin") @@ -37,7 +38,7 @@ def test_user_cache_dir_osx(self, monkeypatch): assert appdirs.user_cache_dir("pip") == "/home/test/Library/Caches/pip" def test_user_cache_dir_linux(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.setenv("HOME", "/home/test") @@ -46,7 +47,7 @@ def test_user_cache_dir_linux(self, monkeypatch): assert appdirs.user_cache_dir("pip") == "/home/test/.cache/pip" def test_user_cache_dir_linux_override(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("XDG_CACHE_HOME", "/home/test/.other-cache") monkeypatch.setenv("HOME", "/home/test") @@ -55,7 +56,7 @@ def test_user_cache_dir_linux_override(self, monkeypatch): assert appdirs.user_cache_dir("pip") == "/home/test/.other-cache/pip" def test_user_cache_dir_linux_home_slash(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) # Verify that we are not affected by https://bugs.python.org/issue14768 monkeypatch.delenv("XDG_CACHE_HOME", raising=False) @@ -71,7 +72,7 @@ def test_user_cache_dir_unicode(self, monkeypatch): def my_get_win_folder(csidl_name): return u"\u00DF\u00E4\u03B1\u20AC" - monkeypatch.setattr(appdirs, "_get_win_folder", my_get_win_folder) + monkeypatch.setattr(_appdirs, "_get_win_folder", my_get_win_folder) # Do not use the isinstance expression directly in the # assert statement, as the Unicode characters in the result @@ -92,19 +93,19 @@ def _get_win_folder(base): return "C:\\ProgramData" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert appdirs.site_config_dirs("pip") == ["C:\\ProgramData\\pip"] assert _get_win_folder.calls == [pretend.call("CSIDL_COMMON_APPDATA")] def test_site_config_dirs_osx(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "darwin") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("HOME", "/home/test") monkeypatch.setattr(sys, "platform", "darwin") @@ -113,7 +114,7 @@ def test_site_config_dirs_osx(self, monkeypatch): ["/Library/Application Support/pip"] def test_site_config_dirs_linux(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.delenv("XDG_CONFIG_DIRS", raising=False) monkeypatch.setattr(sys, "platform", "linux2") @@ -124,7 +125,7 @@ def test_site_config_dirs_linux(self, monkeypatch): ] def test_site_config_dirs_linux_override(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setattr(os, "pathsep", ':') monkeypatch.setenv("XDG_CONFIG_DIRS", "/spam:/etc:/etc/xdg") @@ -146,12 +147,12 @@ def _get_win_folder(base): return "C:\\Users\\test\\AppData\\Local" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert (appdirs.user_data_dir("pip") == @@ -164,12 +165,12 @@ def _get_win_folder(base): return "C:\\Users\\test\\AppData\\Roaming" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert ( @@ -179,7 +180,7 @@ def _get_win_folder(base): assert _get_win_folder.calls == [pretend.call("CSIDL_APPDATA")] def test_user_data_dir_osx(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "darwin") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("HOME", "/home/test") monkeypatch.setattr(sys, "platform", "darwin") @@ -192,7 +193,7 @@ def test_user_data_dir_osx(self, monkeypatch): "/home/test/.config/pip") def test_user_data_dir_linux(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.delenv("XDG_DATA_HOME", raising=False) monkeypatch.setenv("HOME", "/home/test") @@ -201,7 +202,7 @@ def test_user_data_dir_linux(self, monkeypatch): assert appdirs.user_data_dir("pip") == "/home/test/.local/share/pip" def test_user_data_dir_linux_override(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("XDG_DATA_HOME", "/home/test/.other-share") monkeypatch.setenv("HOME", "/home/test") @@ -210,7 +211,7 @@ def test_user_data_dir_linux_override(self, monkeypatch): assert appdirs.user_data_dir("pip") == "/home/test/.other-share/pip" def test_user_data_dir_linux_home_slash(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) # Verify that we are not affected by https://bugs.python.org/issue14768 monkeypatch.delenv("XDG_DATA_HOME", raising=False) @@ -228,12 +229,12 @@ def _get_win_folder(base): return "C:\\Users\\test\\AppData\\Local" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert ( @@ -248,12 +249,12 @@ def _get_win_folder(base): return "C:\\Users\\test\\AppData\\Roaming" monkeypatch.setattr( - appdirs, + _appdirs, "_get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(appdirs, "WINDOWS", True) + monkeypatch.setattr(_appdirs, "system", "win32") monkeypatch.setattr(os, "path", ntpath) assert (appdirs.user_config_dir("pip") == @@ -261,7 +262,7 @@ def _get_win_folder(base): assert _get_win_folder.calls == [pretend.call("CSIDL_APPDATA")] def test_user_config_dir_osx(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "darwin") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("HOME", "/home/test") monkeypatch.setattr(sys, "platform", "darwin") @@ -274,7 +275,7 @@ def test_user_config_dir_osx(self, monkeypatch): "/home/test/.config/pip") def test_user_config_dir_linux(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) monkeypatch.setenv("HOME", "/home/test") @@ -283,7 +284,7 @@ def test_user_config_dir_linux(self, monkeypatch): assert appdirs.user_config_dir("pip") == "/home/test/.config/pip" def test_user_config_dir_linux_override(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) monkeypatch.setenv("XDG_CONFIG_HOME", "/home/test/.other-config") monkeypatch.setenv("HOME", "/home/test") @@ -292,7 +293,7 @@ def test_user_config_dir_linux_override(self, monkeypatch): assert appdirs.user_config_dir("pip") == "/home/test/.other-config/pip" def test_user_config_dir_linux_home_slash(self, monkeypatch): - monkeypatch.setattr(appdirs, "WINDOWS", False) + monkeypatch.setattr(_appdirs, "system", "linux2") monkeypatch.setattr(os, "path", posixpath) # Verify that we are not affected by https://bugs.python.org/issue14768 monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) diff --git a/tools/automation/vendoring/patches/appdirs.patch b/tools/automation/vendoring/patches/appdirs.patch index 73f9f2b743a..136f34b4898 100644 --- a/tools/automation/vendoring/patches/appdirs.patch +++ b/tools/automation/vendoring/patches/appdirs.patch @@ -1,9 +1,69 @@ diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py -index ae67001a..2bd39110 100644 +index ae67001a..92b80251 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py -@@ -557,18 +557,14 @@ def _get_win_folder_with_jna(csidl_name): - +@@ -64,7 +64,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + for a discussion of issues. + + Typical user data directories are: +- Mac OS X: ~/Library/Application Support/ ++ Mac OS X: ~/Library/Application Support/ # or ~/.config/, if the other does not exist + Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\\Application Data\\ + Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ +@@ -88,6 +88,10 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) ++ if not os.path.isdir(path): ++ path = os.path.expanduser('~/.config/') ++ if appname: ++ path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: +@@ -150,7 +154,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + if appname: + if version: + appname = os.path.join(appname, version) +- pathlist = [os.sep.join([x, appname]) for x in pathlist] ++ pathlist = [os.path.join(x, appname) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) +@@ -203,6 +207,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + return path + + ++# for the discussion regarding site_config_dir locations ++# see + def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + r"""Return full path to the user-shared data dir for this application. + +@@ -245,7 +251,9 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) + if appname: + if version: + appname = os.path.join(appname, version) +- pathlist = [os.sep.join([x, appname]) for x in pathlist] ++ pathlist = [os.path.join(x, appname) for x in pathlist] ++ # always look in /etc directly as well ++ pathlist.append('/etc') + + if multipath: + path = os.pathsep.join(pathlist) +@@ -291,6 +299,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) ++ # When using Python 2, return paths as bytes on Windows like we do on ++ # other operating systems. See helper function docs for more details. ++ if not PY3 and isinstance(path, unicode): ++ path = _win_path_to_bytes(path) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) +@@ -557,18 +569,32 @@ def _get_win_folder_with_jna(csidl_name): + if system == "win32": try: - import win32com.shell @@ -23,6 +83,24 @@ index ae67001a..2bd39110 100644 - except ImportError: - _get_win_folder = _get_win_folder_from_registry + _get_win_folder = _get_win_folder_from_registry - - ++ ++ ++def _win_path_to_bytes(path): ++ """Encode Windows paths to bytes. Only used on Python 2. ++ ++ Motivation is to be consistent with other operating systems where paths ++ are also returned as bytes. This avoids problems mixing bytes and Unicode ++ elsewhere in the codebase. For more details and discussion see ++ . ++ ++ If encoding using ASCII and MBCS fails, return the original Unicode path. ++ """ ++ for encoding in ('ASCII', 'MBCS'): ++ try: ++ return path.encode(encoding) ++ except (UnicodeEncodeError, LookupError): ++ pass ++ return path + + #---- self test code From 82b456e043397e212959ec5ac3e13c93e867c1cb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 20 Dec 2019 14:15:52 +0800 Subject: [PATCH 2/6] Fix typo in docstring Co-Authored-By: Christopher Hunt --- src/pip/_internal/utils/appdirs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/appdirs.py b/src/pip/_internal/utils/appdirs.py index cce1e293c00..56850d38629 100644 --- a/src/pip/_internal/utils/appdirs.py +++ b/src/pip/_internal/utils/appdirs.py @@ -2,7 +2,7 @@ This code wraps the vendored appdirs module to so the return values are compatible for the current pip code base. -The intention is to rewrite current usages guradually, keeping the tests pass, +The intention is to rewrite current usages gradually, keeping the tests pass, and eventually drop this after all usages are changed. """ From 368c811467ed851d317ca89559c1c2799f45ff43 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 20 Dec 2019 14:13:42 +0800 Subject: [PATCH 3/6] Treat Windows an macOS as special case in appdirs --- src/pip/_internal/utils/appdirs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/appdirs.py b/src/pip/_internal/utils/appdirs.py index 56850d38629..251c5fd59a5 100644 --- a/src/pip/_internal/utils/appdirs.py +++ b/src/pip/_internal/utils/appdirs.py @@ -36,6 +36,6 @@ def user_data_dir(appname, roaming=False): def site_config_dirs(appname): # type: (str) -> List[str] dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True) - if _appdirs.system == "linux2": + if _appdirs.system not in ["win32", "darwin"]: return dirval.split(os.pathsep) return [dirval] From 2ccc5c055db7247b4529569e9ddf3185b538060d Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 20 Dec 2019 14:36:30 +0800 Subject: [PATCH 4/6] Match site_config_dirs for empty XDG_CONFIG_DIRS --- src/pip/_vendor/appdirs.py | 2 +- tests/unit/test_appdirs.py | 8 ++++++++ tools/automation/vendoring/patches/appdirs.patch | 9 +++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py index 92b80251e14..e9ff1aa4f0f 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -247,7 +247,7 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) # XDG default for $XDG_CONFIG_DIRS # only first, if multipath is False path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') - pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x] if appname: if version: appname = os.path.join(appname, version) diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py index 1a01464174f..402d0db311b 100644 --- a/tests/unit/test_appdirs.py +++ b/tests/unit/test_appdirs.py @@ -138,6 +138,14 @@ def test_site_config_dirs_linux_override(self, monkeypatch): '/etc' ] + def test_site_config_dirs_linux_empty(self, monkeypatch): + monkeypatch.setattr(_appdirs, "system", "linux2") + monkeypatch.setattr(os, "path", posixpath) + monkeypatch.setattr(os, "pathsep", ':') + monkeypatch.setenv("XDG_CONFIG_DIRS", "") + monkeypatch.setattr(sys, "platform", "linux2") + assert appdirs.site_config_dirs("pip") == ['/etc'] + class TestUserDataDir: diff --git a/tools/automation/vendoring/patches/appdirs.patch b/tools/automation/vendoring/patches/appdirs.patch index 136f34b4898..6c6f37b45c5 100644 --- a/tools/automation/vendoring/patches/appdirs.patch +++ b/tools/automation/vendoring/patches/appdirs.patch @@ -1,5 +1,5 @@ diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py -index ae67001a..92b80251 100644 +index ae67001a..e9ff1aa4 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -64,7 +64,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): @@ -40,7 +40,12 @@ index ae67001a..92b80251 100644 def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): r"""Return full path to the user-shared data dir for this application. -@@ -245,7 +251,9 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) +@@ -241,11 +247,13 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') +- pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] ++ pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x] if appname: if version: appname = os.path.join(appname, version) From f6afa1a15423566c7e16ce8de97e32bdc9b8843d Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 20 Dec 2019 15:21:34 +0800 Subject: [PATCH 5/6] Make appdirs detect IronPython Windows --- src/pip/_vendor/appdirs.py | 4 ++ .../vendoring/patches/appdirs.patch | 45 ++++++++++++------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py index e9ff1aa4f0f..c3db25de30d 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -37,6 +37,10 @@ # are actually checked for and the rest of the module expects # *sys.platform* style strings. system = 'linux2' +elif sys.platform == 'cli' and os.name == 'nt': + # Detect Windows in IronPython to match pip._internal.utils.compat.WINDOWS + # Discussion: + system = 'win32' else: system = sys.platform diff --git a/tools/automation/vendoring/patches/appdirs.patch b/tools/automation/vendoring/patches/appdirs.patch index 6c6f37b45c5..1c8d087af70 100644 --- a/tools/automation/vendoring/patches/appdirs.patch +++ b/tools/automation/vendoring/patches/appdirs.patch @@ -1,17 +1,28 @@ diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py -index ae67001a..e9ff1aa4 100644 +index ae67001a..87a1e0a6 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py -@@ -64,7 +64,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): +@@ -37,6 +37,10 @@ if sys.platform.startswith('java'): + # are actually checked for and the rest of the module expects + # *sys.platform* style strings. + system = 'linux2' ++elif sys.platform == 'cli' and os.name == 'nt': ++ # Detect Windows in IronPython to match pip._internal.utils.compat.WINDOWS ++ # Discussion: ++ system = 'win32' + else: + system = sys.platform + +@@ -64,7 +68,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): for a discussion of issues. - + Typical user data directories are: - Mac OS X: ~/Library/Application Support/ + Mac OS X: ~/Library/Application Support/ # or ~/.config/, if the other does not exist Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined Win XP (not roaming): C:\Documents and Settings\\Application Data\\ Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ -@@ -88,6 +88,10 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): +@@ -88,6 +92,10 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): path = os.path.expanduser('~/Library/Application Support/') if appname: path = os.path.join(path, appname) @@ -22,25 +33,25 @@ index ae67001a..e9ff1aa4 100644 else: path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) if appname: -@@ -150,7 +154,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): +@@ -150,7 +158,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): if appname: if version: appname = os.path.join(appname, version) - pathlist = [os.sep.join([x, appname]) for x in pathlist] + pathlist = [os.path.join(x, appname) for x in pathlist] - + if multipath: path = os.pathsep.join(pathlist) -@@ -203,6 +207,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): +@@ -203,6 +211,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): return path - - + + +# for the discussion regarding site_config_dir locations +# see def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): r"""Return full path to the user-shared data dir for this application. - -@@ -241,11 +247,13 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) + +@@ -241,11 +251,13 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) # XDG default for $XDG_CONFIG_DIRS # only first, if multipath is False path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') @@ -53,10 +64,10 @@ index ae67001a..e9ff1aa4 100644 + pathlist = [os.path.join(x, appname) for x in pathlist] + # always look in /etc directly as well + pathlist.append('/etc') - + if multipath: path = os.pathsep.join(pathlist) -@@ -291,6 +299,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): +@@ -291,6 +303,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): if appauthor is None: appauthor = appname path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) @@ -67,8 +78,8 @@ index ae67001a..e9ff1aa4 100644 if appname: if appauthor is not False: path = os.path.join(path, appauthor, appname) -@@ -557,18 +569,32 @@ def _get_win_folder_with_jna(csidl_name): - +@@ -557,18 +573,32 @@ def _get_win_folder_with_jna(csidl_name): + if system == "win32": try: - import win32com.shell @@ -106,6 +117,6 @@ index ae67001a..e9ff1aa4 100644 + except (UnicodeEncodeError, LookupError): + pass + return path - - + + #---- self test code From c98c0ad79c5f4530cb0bbb33358d414884f0a27a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 23 Dec 2019 14:27:44 +0800 Subject: [PATCH 6/6] Default to /etc/xdg if XDG_CONFIG_DIRS if empty --- src/pip/_vendor/appdirs.py | 5 ++- tests/unit/test_appdirs.py | 2 +- .../vendoring/patches/appdirs.patch | 38 +++++++++++-------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py index c3db25de30d..3a52b75846b 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -248,9 +248,10 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) if appname and version: path = os.path.join(path, version) else: - # XDG default for $XDG_CONFIG_DIRS + # XDG default for $XDG_CONFIG_DIRS (missing or empty) + # see # only first, if multipath is False - path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + path = os.getenv('XDG_CONFIG_DIRS') or '/etc/xdg' pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x] if appname: if version: diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py index 402d0db311b..6b2ca8ff196 100644 --- a/tests/unit/test_appdirs.py +++ b/tests/unit/test_appdirs.py @@ -144,7 +144,7 @@ def test_site_config_dirs_linux_empty(self, monkeypatch): monkeypatch.setattr(os, "pathsep", ':') monkeypatch.setenv("XDG_CONFIG_DIRS", "") monkeypatch.setattr(sys, "platform", "linux2") - assert appdirs.site_config_dirs("pip") == ['/etc'] + assert appdirs.site_config_dirs("pip") == ['/etc/xdg/pip', '/etc'] class TestUserDataDir: diff --git a/tools/automation/vendoring/patches/appdirs.patch b/tools/automation/vendoring/patches/appdirs.patch index 1c8d087af70..a6135c35f9e 100644 --- a/tools/automation/vendoring/patches/appdirs.patch +++ b/tools/automation/vendoring/patches/appdirs.patch @@ -1,5 +1,5 @@ diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py -index ae67001a..87a1e0a6 100644 +index ae67001a..3a52b758 100644 --- a/src/pip/_vendor/appdirs.py +++ b/src/pip/_vendor/appdirs.py @@ -37,6 +37,10 @@ if sys.platform.startswith('java'): @@ -12,10 +12,10 @@ index ae67001a..87a1e0a6 100644 + system = 'win32' else: system = sys.platform - + @@ -64,7 +68,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): for a discussion of issues. - + Typical user data directories are: - Mac OS X: ~/Library/Application Support/ + Mac OS X: ~/Library/Application Support/ # or ~/.config/, if the other does not exist @@ -39,23 +39,29 @@ index ae67001a..87a1e0a6 100644 appname = os.path.join(appname, version) - pathlist = [os.sep.join([x, appname]) for x in pathlist] + pathlist = [os.path.join(x, appname) for x in pathlist] - + if multipath: path = os.pathsep.join(pathlist) @@ -203,6 +211,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): return path - - + + +# for the discussion regarding site_config_dir locations +# see def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): r"""Return full path to the user-shared data dir for this application. - -@@ -241,11 +251,13 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) - # XDG default for $XDG_CONFIG_DIRS + +@@ -238,14 +248,17 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) + if appname and version: + path = os.path.join(path, version) + else: +- # XDG default for $XDG_CONFIG_DIRS ++ # XDG default for $XDG_CONFIG_DIRS (missing or empty) ++ # see # only first, if multipath is False - path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') +- path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') - pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] ++ path = os.getenv('XDG_CONFIG_DIRS') or '/etc/xdg' + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x] if appname: if version: @@ -64,10 +70,10 @@ index ae67001a..87a1e0a6 100644 + pathlist = [os.path.join(x, appname) for x in pathlist] + # always look in /etc directly as well + pathlist.append('/etc') - + if multipath: path = os.pathsep.join(pathlist) -@@ -291,6 +303,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): +@@ -291,6 +304,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): if appauthor is None: appauthor = appname path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) @@ -78,8 +84,8 @@ index ae67001a..87a1e0a6 100644 if appname: if appauthor is not False: path = os.path.join(path, appauthor, appname) -@@ -557,18 +573,32 @@ def _get_win_folder_with_jna(csidl_name): - +@@ -557,18 +574,32 @@ def _get_win_folder_with_jna(csidl_name): + if system == "win32": try: - import win32com.shell @@ -117,6 +123,6 @@ index ae67001a..87a1e0a6 100644 + except (UnicodeEncodeError, LookupError): + pass + return path - - + + #---- self test code