From 416cf8f0319607a19dfa526b7581af8facf5f34d Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Fri, 26 Feb 2021 17:20:05 -0500 Subject: [PATCH 1/7] Initial support for passing strings to GMT via GMT_Put_Vectors --- pygmt/clib/session.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index 93734645e90..45f79d06e39 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -59,6 +59,7 @@ np.uint64: "GMT_ULONG", np.uint32: "GMT_UINT", np.datetime64: "GMT_DATETIME", + np.str_: "GMT_TEXT", } @@ -718,9 +719,7 @@ def _check_dtype_and_dim(self, array, ndim): """ # check the array has the given dimension if array.ndim != ndim: - raise GMTInvalidInput( - "Expected a numpy 1d array, got {}d.".format(array.ndim) - ) + raise GMTInvalidInput(f"Expected a numpy 1d array, got {array.ndim}d.") # check the array has a valid/known data type if array.dtype.type not in DTYPES: @@ -744,7 +743,7 @@ def put_vector(self, dataset, column, vector): first. Use ``family='GMT_IS_DATASET|GMT_VIA_VECTOR'``. Not at all numpy dtypes are supported, only: float64, float32, int64, - int32, uint64, and uint32. + int32, uint64, uint32, datetime64 and str_. .. warning:: The numpy array must be C contiguous in memory. If it comes from a @@ -776,11 +775,14 @@ def put_vector(self, dataset, column, vector): ) gmt_type = self._check_dtype_and_dim(vector, ndim=1) - if gmt_type == self["GMT_DATETIME"]: + if gmt_type in (self["GMT_TEXT"], self["GMT_DATETIME"]): vector_pointer = (ctp.c_char_p * len(vector))() - vector_pointer[:] = np.char.encode( - np.datetime_as_string(array_to_datetime(vector)) - ) + if gmt_type == self["GMT_DATETIME"]: + vector_pointer[:] = np.char.encode( + np.datetime_as_string(array_to_datetime(vector)) + ) + else: + vector_pointer[:] = np.char.encode(vector) else: vector_pointer = vector.ctypes.data_as(ctp.c_void_p) status = c_put_vector( @@ -788,11 +790,9 @@ def put_vector(self, dataset, column, vector): ) if status != 0: raise GMTCLibError( - " ".join( - [ - "Failed to put vector of type {}".format(vector.dtype), - "in column {} of dataset.".format(column), - ] + ( + f"Failed to put vector of type {vector.dtype} " + f"in column {column} of dataset." ) ) From f0584650ef7356021bca07e539eedbd666fa840c Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Sun, 28 Feb 2021 01:46:14 -0500 Subject: [PATCH 2/7] Add more tests --- pygmt/tests/test_clib_put_vector.py | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/pygmt/tests/test_clib_put_vector.py b/pygmt/tests/test_clib_put_vector.py index 72a4c858898..bb0b7779b24 100644 --- a/pygmt/tests/test_clib_put_vector.py +++ b/pygmt/tests/test_clib_put_vector.py @@ -90,6 +90,62 @@ def test_put_vector_mixed_dtypes(): npt.assert_allclose(newy, y) +def test_put_vector_string_dtype(): + """ + Passing string type data to a dataset. + """ + vectors = np.array( + [ + ["10", "20.0", "-30.0", "3.5e1"], # numbers + ["10W", "30.50E", "30:30W", "40:30:30.500E"], # longitudes + ["10N", "30.50S", "30:30N", "40:30:30.500S"], # latitudes + # datetimes + ["2021-02-03", "2021-02-03T04", "2021-02-03T04:05:06.700", "T04:50:06.700"], + ] + ) + + expected_vectors = [ + [10.0, 20.0, -30.0, 35], # numbers + [-10, 30.5, -30.5, 40.508472], # longitudes + [10, -30.50, 30.5, -40.508472], # latitudes + # datetimes + ["2021-02-03", "2021-02-03T04", "2021-02-03T04:05:06.700", "T04:50:06.700"], + ] + + expected_dtypes = [np.double, np.double, np.double, np.str_] + + for i, j in itertools.combinations_with_replacement(range(3), r=2): + # TODO: Change range(3) to range(4) + with clib.Session() as lib: + dataset = lib.create_data( + family="GMT_IS_DATASET|GMT_VIA_VECTOR", + geometry="GMT_IS_POINT", + mode="GMT_CONTAINER_ONLY", + dim=[2, 4, 1, 0], # columns, rows, layers, dtype + ) + lib.put_vector(dataset, column=lib["GMT_X"], vector=vectors[i]) + lib.put_vector(dataset, column=lib["GMT_Y"], vector=vectors[j]) + # Turns out wesn doesn't matter for Datasets + wesn = [0] * 6 + # Save the data to a file to see if it's being accessed correctly + with GMTTempFile() as tmp_file: + lib.write_data( + "GMT_IS_VECTOR", + "GMT_IS_POINT", + "GMT_WRITE_SET", + wesn, + tmp_file.name, + dataset, + ) + # Load the data and check that it's correct + newx, newy = tmp_file.loadtxt( + unpack=True, + dtype=[("x", expected_dtypes[i]), ("y", expected_dtypes[j])], + ) + npt.assert_allclose(newx, expected_vectors[i]) + npt.assert_allclose(newy, expected_vectors[j]) + + def test_put_vector_invalid_dtype(): """ Check that it fails with an exception for invalid data types. From 24ad0afd69e0d958e4a9c469b48354178b18debb Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 1 Mar 2021 22:08:42 -0500 Subject: [PATCH 3/7] Add clib.Session.put_strings to API docs --- doc/api/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/index.rst b/doc/api/index.rst index 0ef0b359b39..38b74202bb3 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -212,6 +212,7 @@ Low level access (these are mostly used by the :mod:`pygmt.clib` package): clib.Session.get_default clib.Session.create_data clib.Session.put_matrix + clib.Session.put_strings clib.Session.put_vector clib.Session.write_data clib.Session.open_virtual_file From bf7587cb2ca1b34ade13f0239fc078ca4279c332 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 2 Mar 2021 02:08:10 -0500 Subject: [PATCH 4/7] Improve the tests for datetime strings --- pygmt/tests/test_clib_put_vector.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pygmt/tests/test_clib_put_vector.py b/pygmt/tests/test_clib_put_vector.py index bb0b7779b24..3c1e1b4eda7 100644 --- a/pygmt/tests/test_clib_put_vector.py +++ b/pygmt/tests/test_clib_put_vector.py @@ -109,13 +109,17 @@ def test_put_vector_string_dtype(): [-10, 30.5, -30.5, 40.508472], # longitudes [10, -30.50, 30.5, -40.508472], # latitudes # datetimes - ["2021-02-03", "2021-02-03T04", "2021-02-03T04:05:06.700", "T04:50:06.700"], + [ + "2021-02-03T00:00:00", + "2021-02-03T04:00:00", + "2021-02-03T04:05:06", + "2021-03-02T04:50:06", + ], ] expected_dtypes = [np.double, np.double, np.double, np.str_] - for i, j in itertools.combinations_with_replacement(range(3), r=2): - # TODO: Change range(3) to range(4) + for i, j in itertools.combinations_with_replacement(range(4), r=2): with clib.Session() as lib: dataset = lib.create_data( family="GMT_IS_DATASET|GMT_VIA_VECTOR", @@ -138,12 +142,17 @@ def test_put_vector_string_dtype(): dataset, ) # Load the data and check that it's correct - newx, newy = tmp_file.loadtxt( - unpack=True, - dtype=[("x", expected_dtypes[i]), ("y", expected_dtypes[j])], + output = np.genfromtxt( + tmp_file.name, dtype=None, names=("x", "y"), encoding=None ) - npt.assert_allclose(newx, expected_vectors[i]) - npt.assert_allclose(newy, expected_vectors[j]) + if i != 3: + npt.assert_allclose(output["x"], expected_vectors[i]) + else: + npt.assert_array_equal(output["x"], expected_vectors[i]) + if j != 3: + npt.assert_allclose(output["y"], expected_vectors[j]) + else: + npt.assert_array_equal(output["y"], expected_vectors[j]) def test_put_vector_invalid_dtype(): From c1585a90d9e3d6e4780edafaadce519140d431cf Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 2 Mar 2021 02:12:15 -0500 Subject: [PATCH 5/7] Remove unused expected_dtypes --- pygmt/tests/test_clib_put_vector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pygmt/tests/test_clib_put_vector.py b/pygmt/tests/test_clib_put_vector.py index 3c1e1b4eda7..1873623a100 100644 --- a/pygmt/tests/test_clib_put_vector.py +++ b/pygmt/tests/test_clib_put_vector.py @@ -117,8 +117,6 @@ def test_put_vector_string_dtype(): ], ] - expected_dtypes = [np.double, np.double, np.double, np.str_] - for i, j in itertools.combinations_with_replacement(range(4), r=2): with clib.Session() as lib: dataset = lib.create_data( From 6ba4d5c0ef5a6ccf49004f7c44ae7bb0a553f71a Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 2 Mar 2021 02:37:17 -0500 Subject: [PATCH 6/7] Add more comments to the test --- pygmt/tests/test_clib_put_vector.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/pygmt/tests/test_clib_put_vector.py b/pygmt/tests/test_clib_put_vector.py index 1873623a100..f5626449a58 100644 --- a/pygmt/tests/test_clib_put_vector.py +++ b/pygmt/tests/test_clib_put_vector.py @@ -92,23 +92,25 @@ def test_put_vector_mixed_dtypes(): def test_put_vector_string_dtype(): """ - Passing string type data to a dataset. + Passing string type vectors to a dataset. """ + # input string vectors: numbers, longitudes, latitudes, and datetimes vectors = np.array( [ - ["10", "20.0", "-30.0", "3.5e1"], # numbers - ["10W", "30.50E", "30:30W", "40:30:30.500E"], # longitudes - ["10N", "30.50S", "30:30N", "40:30:30.500S"], # latitudes - # datetimes + ["10", "20.0", "-30.0", "3.5e1"], + ["10W", "30.50E", "30:30W", "40:30:30.500E"], + ["10N", "30.50S", "30:30N", "40:30:30.500S"], ["2021-02-03", "2021-02-03T04", "2021-02-03T04:05:06.700", "T04:50:06.700"], ] ) - + # output vectors in double or string type + # Notes: + # 1. longitudes and latitudes are stored in double in GMT + # 2. The default output format for datetime is YYYY-mm-ddTHH:MM:SS expected_vectors = [ - [10.0, 20.0, -30.0, 35], # numbers - [-10, 30.5, -30.5, 40.508472], # longitudes - [10, -30.50, 30.5, -40.508472], # latitudes - # datetimes + [10.0, 20.0, -30.0, 35], + [-10, 30.5, -30.5, 40.508472], + [10, -30.50, 30.5, -40.508472], [ "2021-02-03T00:00:00", "2021-02-03T04:00:00", @@ -117,6 +119,7 @@ def test_put_vector_string_dtype(): ], ] + # loop over all possible combinations of input types for i, j in itertools.combinations_with_replacement(range(4), r=2): with clib.Session() as lib: dataset = lib.create_data( @@ -139,10 +142,13 @@ def test_put_vector_string_dtype(): tmp_file.name, dataset, ) - # Load the data and check that it's correct + # Load the data output = np.genfromtxt( tmp_file.name, dtype=None, names=("x", "y"), encoding=None ) + # check that the output is correct + # Use npt.assert_allclose for numeric arrays + # and npt.assert_array_equal for string arrays if i != 3: npt.assert_allclose(output["x"], expected_vectors[i]) else: From cadfec035dcfe878760fd10288e205ba7c0cbeb4 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Fri, 23 Apr 2021 17:06:48 -0400 Subject: [PATCH 7/7] Fix a broken test --- pygmt/tests/test_clib_put_vector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygmt/tests/test_clib_put_vector.py b/pygmt/tests/test_clib_put_vector.py index f5626449a58..1d085eb92ca 100644 --- a/pygmt/tests/test_clib_put_vector.py +++ b/pygmt/tests/test_clib_put_vector.py @@ -2,6 +2,7 @@ Test the functions that put vector data into GMT. """ import itertools +from datetime import datetime import numpy as np import numpy.testing as npt @@ -115,7 +116,7 @@ def test_put_vector_string_dtype(): "2021-02-03T00:00:00", "2021-02-03T04:00:00", "2021-02-03T04:05:06", - "2021-03-02T04:50:06", + f"{datetime.utcnow().strftime('%Y-%m-%d')}T04:50:06", ], ]