From 6f61ef35ab5c7b4cea204df33837246b311d28c8 Mon Sep 17 00:00:00 2001 From: Leonardo Uieda Date: Wed, 19 Jun 2019 18:22:30 -1000 Subject: [PATCH 1/9] Separate out the libgmt loading tests --- pygmt/clib/loading.py | 2 +- pygmt/tests/test_clib.py | 51 ---------------------------- pygmt/tests/test_clib_loading.py | 58 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 52 deletions(-) create mode 100644 pygmt/tests/test_clib_loading.py diff --git a/pygmt/clib/loading.py b/pygmt/clib/loading.py index 58b33196e85..d6a35470cbe 100644 --- a/pygmt/clib/loading.py +++ b/pygmt/clib/loading.py @@ -52,7 +52,7 @@ def load_libgmt(env=None): return libgmt -def get_clib_path(env): +def get_clib_path(env=None): """ Get the path to the libgmt shared library. diff --git a/pygmt/tests/test_clib.py b/pygmt/tests/test_clib.py index 77a26de0a06..bd76643c712 100644 --- a/pygmt/tests/test_clib.py +++ b/pygmt/tests/test_clib.py @@ -14,12 +14,9 @@ from .. import clib from ..clib.session import FAMILIES, VIAS -from ..clib.loading import clib_name, load_libgmt, check_libgmt, get_clib_path from ..clib.conversion import dataarray_to_matrix from ..exceptions import ( GMTCLibError, - GMTOSError, - GMTCLibNotFoundError, GMTCLibNoSessionError, GMTInvalidInput, GMTVersionError, @@ -70,54 +67,6 @@ def mock_get_libgmt_func(name, argtypes=None, restype=None): setattr(session, "get_libgmt_func", get_libgmt_func) -def test_load_libgmt(): - "Test that loading libgmt works and doesn't crash." - load_libgmt() - - -def test_load_libgmt_fail(): - "Test that loading fails when given a bad library path." - env = {"GMT_LIBRARY_PATH": "not/a/real/path"} - with pytest.raises(GMTCLibNotFoundError): - load_libgmt(env=env) - - -def test_get_clib_path(): - "Test that the correct path is found when setting GMT_LIBRARY_PATH." - # Get the real path to the library first - with clib.Session() as lib: - libpath = lib.info["library path"] - libdir = os.path.dirname(libpath) - # Assign it to the environment variable but keep a backup value to restore - # later - env = {"GMT_LIBRARY_PATH": libdir} - - # Check that the path is determined correctly - path_used = get_clib_path(env=env) - assert os.path.samefile(path_used, libpath) - assert os.path.dirname(path_used) == libdir - - # Check that loading libgmt works - load_libgmt(env=env) - - -def test_check_libgmt(): - "Make sure check_libgmt fails when given a bogus library" - with pytest.raises(GMTCLibError): - check_libgmt(dict()) - - -def test_clib_name(): - "Make sure we get the correct library name for different OS names" - for linux in ["linux", "linux2", "linux3"]: - assert clib_name(linux) == "libgmt.so" - assert clib_name("darwin") == "libgmt.dylib" - assert clib_name("win32", is_64bit=True) == "gmt_w64.dll" - assert clib_name("win32", is_64bit=False) == "gmt_w32.dll" - with pytest.raises(GMTOSError): - clib_name("meh") - - def test_getitem(): "Test that I can get correct constants from the C lib" ses = clib.Session() diff --git a/pygmt/tests/test_clib_loading.py b/pygmt/tests/test_clib_loading.py new file mode 100644 index 00000000000..a62c1d69e6c --- /dev/null +++ b/pygmt/tests/test_clib_loading.py @@ -0,0 +1,58 @@ +""" +Test the functions that load libgmt +""" +import os + +import pytest + +from ..clib.loading import clib_name, load_libgmt, check_libgmt, get_clib_path +from .. import clib +from ..exceptions import GMTCLibError, GMTOSError, GMTCLibNotFoundError + + +def test_load_libgmt(): + "Test that loading libgmt works and doesn't crash." + load_libgmt() + + +def test_load_libgmt_fail(): + "Test that loading fails when given a bad library path." + env = {"GMT_LIBRARY_PATH": "not/a/real/path"} + with pytest.raises(GMTCLibNotFoundError): + load_libgmt(env=env) + + +def test_get_clib_path(): + "Test that the correct path is found when setting GMT_LIBRARY_PATH." + # Get the real path to the library first + with clib.Session() as lib: + libpath = lib.info["library path"] + libdir = os.path.dirname(libpath) + # Assign it to the environment variable but keep a backup value to restore + # later + env = {"GMT_LIBRARY_PATH": libdir} + + # Check that the path is determined correctly + path_used = get_clib_path(env=env) + assert os.path.samefile(path_used, libpath) + assert os.path.dirname(path_used) == libdir + + # Check that loading libgmt works + load_libgmt(env=env) + + +def test_check_libgmt(): + "Make sure check_libgmt fails when given a bogus library" + with pytest.raises(GMTCLibError): + check_libgmt(dict()) + + +def test_clib_name(): + "Make sure we get the correct library name for different OS names" + for linux in ["linux", "linux2", "linux3"]: + assert clib_name(linux) == "libgmt.so" + assert clib_name("darwin") == "libgmt.dylib" + assert clib_name("win32", is_64bit=True) == "gmt_w64.dll" + assert clib_name("win32", is_64bit=False) == "gmt_w32.dll" + with pytest.raises(GMTOSError): + clib_name("meh") From 553c09ad78bdf12e10eefa365db8a8fe708c5495 Mon Sep 17 00:00:00 2001 From: Leonardo Uieda Date: Wed, 19 Jun 2019 18:43:40 -1000 Subject: [PATCH 2/9] Try to find the Windows library --- pygmt/clib/loading.py | 83 +++++++++----------------------- pygmt/tests/test_clib_loading.py | 45 +++++------------ 2 files changed, 33 insertions(+), 95 deletions(-) diff --git a/pygmt/clib/loading.py b/pygmt/clib/loading.py index d6a35470cbe..71157972883 100644 --- a/pygmt/clib/loading.py +++ b/pygmt/clib/loading.py @@ -37,85 +37,46 @@ def load_libgmt(env=None): couldn't access the functions). """ - libpath = get_clib_path(env) - try: - libgmt = ctypes.CDLL(libpath) - check_libgmt(libgmt) - except OSError as err: - msg = "\n".join( - [ - "Error loading the GMT shared library '{}':".format(libpath), - "{}".format(str(err)), - ] - ) - raise GMTCLibNotFoundError(msg) - return libgmt - - -def get_clib_path(env=None): - """ - Get the path to the libgmt shared library. - - Determine the file name and extension and append to the path set by - ``GMT_LIBRARY_PATH``, if any. - - Parameters - ---------- - env : dict or None - A dictionary containing the environment variables. If ``None``, will - default to ``os.environ``. - - Returns - ------- - libpath : str - The path to the libgmt shared library. - - """ - libname = clib_name() if env is None: env = os.environ - if "GMT_LIBRARY_PATH" in env: - libpath = os.path.join(env["GMT_LIBRARY_PATH"], libname) - else: - libpath = libname - return libpath + libnames = clib_name(os_name=sys.platform) + libpath = env.get("GMT_LIBRARY_PATH", "") + error = False + for libname in libnames: + try: + libgmt = ctypes.CDLL(os.path.join(libpath, libname)) + check_libgmt(libgmt) + except OSError as err: + error = err + if error: + raise GMTCLibNotFoundError( + "Error loading the GMT shared library '{}':".format(libname) + ) + return libgmt -def clib_name(os_name=None, is_64bit=None): +def clib_name(os_name): """ Return the name of GMT's shared library for the current OS. Parameters ---------- - os_name : str or None - The operating system name as given by ``sys.platform`` - (the default if None). - is_64bit : bool or None - Whether or not the OS is 64bit. Only used if the OS is ``win32``. If - None, will determine automatically. + os_name : str + The operating system name as given by ``sys.platform``. Returns ------- - libname : str - The name of GMT's shared library. + libname : list of str + List of possible names of GMT's shared library. """ - if os_name is None: - os_name = sys.platform - - if is_64bit is None: - is_64bit = sys.maxsize > 2 ** 32 - if os_name.startswith("linux"): - libname = "libgmt.so" + libname = ["libgmt.so"] elif os_name == "darwin": # Darwin is macOS - libname = "libgmt.dylib" + libname = ["libgmt.dylib"] elif os_name == "win32": - if is_64bit: - libname = "gmt_w64.dll" - else: - libname = "gmt_w32.dll" + libname = ["gmt.lib", "gmt_w64.dll", "gmt_w32.dll"] else: raise GMTOSError('Operating system "{}" not supported.'.format(sys.platform)) return libname diff --git a/pygmt/tests/test_clib_loading.py b/pygmt/tests/test_clib_loading.py index a62c1d69e6c..a3c4f1140ce 100644 --- a/pygmt/tests/test_clib_loading.py +++ b/pygmt/tests/test_clib_loading.py @@ -1,18 +1,21 @@ """ Test the functions that load libgmt """ -import os - import pytest -from ..clib.loading import clib_name, load_libgmt, check_libgmt, get_clib_path -from .. import clib +from ..clib.loading import clib_name, load_libgmt, check_libgmt from ..exceptions import GMTCLibError, GMTOSError, GMTCLibNotFoundError +def test_check_libgmt(): + "Make sure check_libgmt fails when given a bogus library" + with pytest.raises(GMTCLibError): + check_libgmt(dict()) + + def test_load_libgmt(): "Test that loading libgmt works and doesn't crash." - load_libgmt() + check_libgmt(load_libgmt()) def test_load_libgmt_fail(): @@ -22,37 +25,11 @@ def test_load_libgmt_fail(): load_libgmt(env=env) -def test_get_clib_path(): - "Test that the correct path is found when setting GMT_LIBRARY_PATH." - # Get the real path to the library first - with clib.Session() as lib: - libpath = lib.info["library path"] - libdir = os.path.dirname(libpath) - # Assign it to the environment variable but keep a backup value to restore - # later - env = {"GMT_LIBRARY_PATH": libdir} - - # Check that the path is determined correctly - path_used = get_clib_path(env=env) - assert os.path.samefile(path_used, libpath) - assert os.path.dirname(path_used) == libdir - - # Check that loading libgmt works - load_libgmt(env=env) - - -def test_check_libgmt(): - "Make sure check_libgmt fails when given a bogus library" - with pytest.raises(GMTCLibError): - check_libgmt(dict()) - - def test_clib_name(): "Make sure we get the correct library name for different OS names" for linux in ["linux", "linux2", "linux3"]: - assert clib_name(linux) == "libgmt.so" - assert clib_name("darwin") == "libgmt.dylib" - assert clib_name("win32", is_64bit=True) == "gmt_w64.dll" - assert clib_name("win32", is_64bit=False) == "gmt_w32.dll" + assert clib_name(linux) == ["libgmt.so"] + assert clib_name("darwin") == ["libgmt.dylib"] + assert clib_name("win32") == ["gmt.lib", "gmt_w64.dll", "gmt_w32.dll"] with pytest.raises(GMTOSError): clib_name("meh") From c96898600361fc1ac879675a47910df478309b42 Mon Sep 17 00:00:00 2001 From: Leonardo Uieda Date: Wed, 19 Jun 2019 19:02:07 -1000 Subject: [PATCH 3/9] Use .dll instead of .lib --- pygmt/clib/loading.py | 2 +- pygmt/tests/test_clib_loading.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygmt/clib/loading.py b/pygmt/clib/loading.py index 71157972883..93ed1d488cc 100644 --- a/pygmt/clib/loading.py +++ b/pygmt/clib/loading.py @@ -76,7 +76,7 @@ def clib_name(os_name): # Darwin is macOS libname = ["libgmt.dylib"] elif os_name == "win32": - libname = ["gmt.lib", "gmt_w64.dll", "gmt_w32.dll"] + libname = ["gmt.dll", "gmt_w64.dll", "gmt_w32.dll"] else: raise GMTOSError('Operating system "{}" not supported.'.format(sys.platform)) return libname diff --git a/pygmt/tests/test_clib_loading.py b/pygmt/tests/test_clib_loading.py index a3c4f1140ce..1cc6dc4f4df 100644 --- a/pygmt/tests/test_clib_loading.py +++ b/pygmt/tests/test_clib_loading.py @@ -30,6 +30,6 @@ def test_clib_name(): for linux in ["linux", "linux2", "linux3"]: assert clib_name(linux) == ["libgmt.so"] assert clib_name("darwin") == ["libgmt.dylib"] - assert clib_name("win32") == ["gmt.lib", "gmt_w64.dll", "gmt_w32.dll"] + assert clib_name("win32") == ["gmt.dll", "gmt_w64.dll", "gmt_w32.dll"] with pytest.raises(GMTOSError): clib_name("meh") From 4409772235f82e14118a14e89ca6ee008c62ee80 Mon Sep 17 00:00:00 2001 From: Leonardo Uieda Date: Thu, 20 Jun 2019 09:39:14 -1000 Subject: [PATCH 4/9] Update pygmt/clib/loading.py Co-Authored-By: Dongdong Tian --- pygmt/clib/loading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygmt/clib/loading.py b/pygmt/clib/loading.py index 93ed1d488cc..96817d654b6 100644 --- a/pygmt/clib/loading.py +++ b/pygmt/clib/loading.py @@ -46,6 +46,7 @@ def load_libgmt(env=None): try: libgmt = ctypes.CDLL(os.path.join(libpath, libname)) check_libgmt(libgmt) + break except OSError as err: error = err if error: From f21bf74a61898c6300a56c8f03bcb51b928dbaa8 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Thu, 22 Aug 2019 01:29:10 -0400 Subject: [PATCH 5/9] Re-enable Windows CI in current PR --- .azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 483586e081d..e907353be7d 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -155,7 +155,6 @@ jobs: ######################################################################################## - job: displayName: 'Windows' - condition: False pool: vmImage: 'vs2017-win2016' From 298de3ba4f9123ce9c923e68a9a4698e84afc26b Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 18 May 2020 23:15:31 -0400 Subject: [PATCH 6/9] Fix how DLLs are found Here we need to set `error` to True first, so if libnames is empty, than we raise the GMTCLibNotFoundError error. On Windows, libnames is a list of possible DLL names, "gmt.dll", "gmt_w64.dll", and "gmt_w32.dll". If any one of them is found, then we set `error` to False and break to for loop. If none of them are found, then we raise the GMTCLibNotFoundError error: ``` Error loading the GMT shared library 'gmt.dll, gmt_w64.dll, gmt_w32.dll'. ``` The error message above can be improved, but it's not a big issue. --- pygmt/clib/loading.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pygmt/clib/loading.py b/pygmt/clib/loading.py index 96817d654b6..a0b9b18c043 100644 --- a/pygmt/clib/loading.py +++ b/pygmt/clib/loading.py @@ -41,17 +41,18 @@ def load_libgmt(env=None): env = os.environ libnames = clib_name(os_name=sys.platform) libpath = env.get("GMT_LIBRARY_PATH", "") - error = False + error = True for libname in libnames: try: libgmt = ctypes.CDLL(os.path.join(libpath, libname)) check_libgmt(libgmt) + error = False break except OSError as err: error = err if error: raise GMTCLibNotFoundError( - "Error loading the GMT shared library '{}':".format(libname) + "Error loading the GMT shared library '{}':".format(", ".join(libnames)) ) return libgmt From 3f55de771e56569a97f86003143521e741166156 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 18 May 2020 23:23:19 -0400 Subject: [PATCH 7/9] Set two variables to let pygmt work with conda GMT To make PyGMT work with conda's GMT package on Windows, we have to set two environmental variables: - GMT_LIBRARY_PATH: `C:\Miniconda\envs\testing\Library\bin` - GMT_SHAREDIR: `C:\Miniconda\envs\testing\Library\share\gmt` --- .azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index e907353be7d..adb4593a017 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -162,7 +162,11 @@ jobs: variables: CONDA_REQUIREMENTS: requirements.txt CONDA_REQUIREMENTS_DEV: requirements-dev.txt - CONDA_INSTALL_EXTRA: "codecov" + CONDA_INSTALL_EXTRA: "codecov gmt=6.0.0" + # ctypes.CDLL cannot find conda's libraries + GMT_LIBRARY_PATH: 'C:\Miniconda\envs\testing\Library\bin' + # Due to potential bug in GMT, we have to set GMT_SHAREDIR manually + GMT_SHAREDIR: 'C:\Miniconda\envs\testing\Library\share\gmt' strategy: matrix: From 0514bf0c316bf562e9214c37fa48323616dcac4b Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 19 May 2020 13:10:53 -0400 Subject: [PATCH 8/9] Use the ghostscript from conda-forge --- .azure-pipelines.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index adb4593a017..6c1f63ba394 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -182,12 +182,6 @@ jobs: steps: - # Install ghostscript separately since there is no Windows conda-forge package for it. - - bash: | - set -x -e - choco install ghostscript - displayName: Install ghostscript via chocolatey - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" displayName: Add conda to PATH From 81ed8f40090126f344bb6122e62aa4ef85dbba5b Mon Sep 17 00:00:00 2001 From: Wei Ji <23487320+weiji14@users.noreply.github.com> Date: Wed, 20 May 2020 17:01:44 +1200 Subject: [PATCH 9/9] Open netcdf grid using with statement So that the file is closed properly for deletion later. --- pygmt/tests/test_surface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygmt/tests/test_surface.py b/pygmt/tests/test_surface.py index 13cf09d028d..1a6c3af9aae 100644 --- a/pygmt/tests/test_surface.py +++ b/pygmt/tests/test_surface.py @@ -90,8 +90,8 @@ def test_surface_with_outfile_param(): ) assert output is None # check that output is None since outfile is set assert os.path.exists(path=TEMP_GRID) # check that outfile exists at path - grid = xr.open_dataarray(TEMP_GRID) - assert isinstance(grid, xr.DataArray) # check that netcdf grid loaded properly + with xr.open_dataarray(TEMP_GRID) as grid: + assert isinstance(grid, xr.DataArray) # ensure netcdf grid loads ok finally: os.remove(path=TEMP_GRID) return output @@ -108,8 +108,8 @@ def test_surface_short_aliases(): output = surface(data=data, I="5m", R=[245, 255, 20, 30], G=TEMP_GRID) assert output is None # check that output is None since outfile is set assert os.path.exists(path=TEMP_GRID) # check that outfile exists at path - grid = xr.open_dataarray(TEMP_GRID) - assert isinstance(grid, xr.DataArray) # check that netcdf grid loaded properly + with xr.open_dataarray(TEMP_GRID) as grid: + assert isinstance(grid, xr.DataArray) # ensure netcdf grid loads ok finally: os.remove(path=TEMP_GRID) return output