Merge pull request #296 from onalant/shape

Add support for shapes in dtype definitions

Author: alimanfoo

docs/release.rst

@@ -6,12 +6,23 @@ Release notes
 2.3.0 (Work in Progress)
 ------------------------
 
+Enhancements
+~~~~~~~~~~~~
+
+* Support has been added for structured arrays with sub-array shape and/or nested fields. By
+  :user:`Tarik Onalan <onalant>`, :issue:`111`, :issue:`296`.
+
 Maintenance
 ~~~~~~~~~~~
 
 * CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and
-  upgrade all package requirements. :issue:`308`.
+  upgrade all pinned package requirements. :issue:`308`.
 
+* Failing tests related to pickling/unpickling have been fixed. By :user:`Ryan Williams <ryan-williams>`,
+  :issue:`273`, :issue:`308`.
+
+Acknowledgments
+~~~~~~~~~~~~~~~
 
 .. _release_2.2.0:
 

docs/spec/v2.rst

@@ -140,13 +140,31 @@ the `NumPy documentation on Datetimes and Timedeltas
 <https://docs.scipy.org/doc/numpy/reference/arrays.datetime.html#datetime-units>`_.
 For example, ``"<M8[ns]"`` specifies a datetime64 data type with nanosecond time units.
 
-Structured data types (i.e., with multiple named fields) are encoded as a list
-of two-element lists, following `NumPy array protocol type descriptions (descr)
-<http://docs.scipy.org/doc/numpy/reference/arrays.interface.html#>`_. For
-example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", "|u1"]]`` defines a
-data type composed of three single-byte unsigned integers labelled "r", "g" and
-"b".
-
+Structured data types (i.e., with multiple named fields) are encoded
+as a list of lists, following `NumPy array protocol type descriptions
+(descr)
+<http://docs.scipy.org/doc/numpy/reference/arrays.interface.html#>`_. Each
+sub-list has the form ``[fieldname, datatype, shape]`` where ``shape``
+is optional. ``fieldname`` is a string, ``datatype`` is a string
+specifying a simple data type (see above), and ``shape`` is a list of
+integers specifying subarray shape. For example, the JSON list below
+defines a data type composed of three single-byte unsigned integer
+fields named "r", "g" and "b"::
+
+    [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]]
+
+For example, the JSON list below defines a data type composed of three
+fields named "x", "y" and "z", where "x" and "y" each contain 32-bit
+floats, and each item in "z" is a 2 by 2 array of floats::
+
+    [["x", "<f4"], ["y", "<f4"], ["z", "<f4", [2, 2]]]
+
+Structured data types may also be nested, e.g., the following JSON
+list defines a data type with two fields "foo" and "bar", where "bar"
+has two sub-fields "baz" and "qux"::
+
+    [["foo", "<f4"], ["bar", [["baz", "<f4"], ["qux", "<i4"]]]] 
+    
 .. _spec_v2_array_fill_value:
 
 Fill value encoding
@@ -512,6 +530,10 @@ initially published to clarify ambiguities and add some missing information.
   either arrays or groups, and if absent then custom attributes should be treated as
   empty.
 
+* The specification now describes how structured datatypes with
+  subarray shapes and/or with nested structured data types are encoded
+  in array metadata (:issue:`111`, :issue:`296`).
+
 
 Changes from version 1 to version 2
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

zarr/core.py

@@ -1568,7 +1568,11 @@ def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection,
         except KeyError:
             # chunk not initialized
             if self._fill_value is not None:
-                out[out_selection] = self._fill_value
+                if fields:
+                    fill_value = self._fill_value[fields]
+                else:
+                    fill_value = self._fill_value
+                out[out_selection] = fill_value
 
         else:
 

zarr/meta.py

@@ -54,9 +54,12 @@ def decode_array_metadata(s):
 
 def encode_array_metadata(meta):
     dtype = meta['dtype']
+    sdshape = ()
+    if dtype.subdtype is not None:
+        dtype, sdshape = dtype.subdtype
     meta = dict(
         zarr_format=ZARR_FORMAT,
-        shape=meta['shape'],
+        shape=meta['shape'] + sdshape,
         chunks=meta['chunks'],
         dtype=encode_dtype(dtype),
         compressor=meta['compressor'],
@@ -83,10 +86,9 @@ def _decode_dtype_descr(d):
         # recurse to handle nested structures
         if PY2:  # pragma: py3 no cover
             # under PY2 numpy rejects unicode field names
-            d = [(f.encode('ascii'), _decode_dtype_descr(v))
-                 for f, v in d]
+            d = [(k[0].encode("ascii"), _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d]
         else:  # pragma: py2 no cover
-            d = [(f, _decode_dtype_descr(v)) for f, v in d]
+            d = [(k[0], _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d]
     return d
 
 

zarr/storage.py

@@ -329,8 +329,9 @@ def _init_array_metadata(store, shape, chunks=None, dtype=None, compressor='defa
         err_contains_group(path)
 
     # normalize metadata
-    shape = normalize_shape(shape)
     dtype, object_codec = normalize_dtype(dtype, object_codec)
+    shape = normalize_shape(shape) + dtype.shape
+    dtype = dtype.base
     chunks = normalize_chunks(chunks, shape, dtype.itemsize)
     order = normalize_order(order)
     fill_value = normalize_fill_value(fill_value, dtype)

zarr/tests/test_core.py

@@ -846,34 +846,90 @@ def test_nchunks_initialized(self):
         z[:] = 42
         assert 10 == z.nchunks_initialized
 
-    def test_structured_array(self):
+    def test_array_dtype_shape(self):
 
+        dt = "(2, 2)f4"
         # setup some data
-        d = np.array([(b'aaa', 1, 4.2),
-                      (b'bbb', 2, 8.4),
-                      (b'ccc', 3, 12.6)],
-                     dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')])
+        d = np.array([((0, 1),
+                       (1, 2)),
+                      ((1, 2),
+                       (2, 3)),
+                      ((2, 3),
+                       (3, 4))],
+                     dtype=dt)
+
         for a in (d, d[:0]):
-            for fill_value in None, b'', (b'zzz', 42, 16.8):
+            for fill_value in None, 0:
+                z = self.create_array(shape=a.shape[:-2], chunks=2, dtype=dt, fill_value=fill_value)
+                assert len(a) == len(z)
+                if fill_value is not None:
+                    assert fill_value == z.fill_value
+                z[...] = a
+                assert_array_equal(a, z[...])
+
+    def check_structured_array(self, d, fill_values):
+        for a in (d, d[:0]):
+            for fill_value in fill_values:
                 z = self.create_array(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=fill_value)
                 assert len(a) == len(z)
+                assert a.shape == z.shape
+                assert a.dtype == z.dtype
+
+                # check use of fill value before array is initialised with data
                 if fill_value is not None:
                     if fill_value == b'':
                         # numpy 1.14 compatibility
                         np_fill_value = np.array(fill_value, dtype=a.dtype.str).view(a.dtype)[()]
                     else:
                         np_fill_value = np.array(fill_value, dtype=a.dtype)[()]
                     assert np_fill_value == z.fill_value
-                    if len(z):
+                    if len(a):
                         assert np_fill_value == z[0]
                         assert np_fill_value == z[-1]
+                        empty = np.empty_like(a)
+                        empty[:] = np_fill_value
+                        assert empty[0] == z[0]
+                        assert_array_equal(empty[0:2], z[0:2])
+                        assert_array_equal(empty, z[...])
+                        for f in a.dtype.names:
+                            assert_array_equal(empty[f], z[f])
+
+                # store data in array
                 z[...] = a
+
+                # check stored data
                 if len(a):
                     assert a[0] == z[0]
-                assert_array_equal(a, z[...])
-                assert_array_equal(a['foo'], z['foo'])
-                assert_array_equal(a['bar'], z['bar'])
-                assert_array_equal(a['baz'], z['baz'])
+                    assert a[-1] == z[-1]
+                    assert_array_equal(a[0:2], z[0:2])
+                    assert_array_equal(a, z[...])
+                    for f in a.dtype.names:
+                        assert_array_equal(a[f], z[f])
+
+    def test_structured_array(self):
+        d = np.array([(b'aaa', 1, 4.2),
+                      (b'bbb', 2, 8.4),
+                      (b'ccc', 3, 12.6)],
+                     dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')])
+        fill_values = None, b'', (b'zzz', 42, 16.8)
+        self.check_structured_array(d, fill_values)
+
+    def test_structured_array_subshapes(self):
+        d = np.array([(0, ((0, 1, 2), (1, 2, 3)), b'aaa'),
+                      (1, ((1, 2, 3), (2, 3, 4)), b'bbb'),
+                      (2, ((2, 3, 4), (3, 4, 5)), b'ccc')],
+                     dtype=[('foo', 'i8'), ('bar', '(2, 3)f4'), ('baz', 'S3')])
+        fill_values = None, b'', (0, ((0, 0, 0), (1, 1, 1)), b'zzz')
+        self.check_structured_array(d, fill_values)
+
+    def test_structured_array_nested(self):
+        d = np.array([(0, (0, ((0, 1), (1, 2), (2, 3)), 0), b'aaa'),
+                      (1, (1, ((1, 2), (2, 3), (3, 4)), 1), b'bbb'),
+                      (2, (2, ((2, 3), (3, 4), (4, 5)), 2), b'ccc')],
+                     dtype=[('foo', 'i8'), ('bar', [('foo', 'i4'), ('bar', '(3, 2)f4'),
+                            ('baz', 'u1')]), ('baz', 'S3')])
+        fill_values = None, b'', (0, (0, ((0, 0), (1, 1), (2, 2)), 0), b'zzz')
+        self.check_structured_array(d, fill_values)
 
     def test_dtypes(self):
 
@@ -1559,10 +1615,22 @@ def test_astype(self):
         expected = data.astype(astype)
         assert_array_equal(expected, z2)
 
+    def test_array_dtype_shape(self):
+        # skip this one, cannot do delta on unstructured array
+        pass
+
     def test_structured_array(self):
         # skip this one, cannot do delta on structured array
         pass
 
+    def test_structured_array_subshapes(self):
+        # skip this one, cannot do delta on structured array
+        pass
+
+    def test_structured_array_nested(self):
+        # skip this one, cannot do delta on structured array
+        pass
+
     def test_dtypes(self):
         # skip this one, delta messes up floats
         pass

zarr/tests/test_meta.py

@@ -116,6 +116,88 @@ def test_encode_decode_array_2():
     assert [df.get_config()] == meta_dec['filters']
 
 
+def test_encode_decode_array_dtype_shape():
+
+    meta = dict(
+        shape=(100,),
+        chunks=(10,),
+        dtype=np.dtype('(10, 10)f8'),
+        compressor=Zlib(1).get_config(),
+        fill_value=None,
+        filters=None,
+        order='C'
+    )
+
+    meta_json = '''{
+        "chunks": [10],
+        "compressor": {"id": "zlib", "level": 1},
+        "dtype": "<f8",
+        "fill_value": null,
+        "filters": null,
+        "order": "C",
+        "shape": [100, 10, 10],
+        "zarr_format": %s
+    }''' % ZARR_FORMAT
+
+    # test encoding
+    meta_enc = encode_array_metadata(meta)
+    assert_json_equal(meta_json, meta_enc)
+
+    # test decoding
+    meta_dec = decode_array_metadata(meta_enc)
+    assert ZARR_FORMAT == meta_dec['zarr_format']
+    # to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
+    assert meta['shape'] + meta['dtype'].shape == meta_dec['shape']
+    assert meta['chunks'] == meta_dec['chunks']
+    # to maintain consistency with numpy unstructured arrays, unpack dtypes
+    assert meta['dtype'].base == meta_dec['dtype']
+    assert meta['compressor'] == meta_dec['compressor']
+    assert meta['order'] == meta_dec['order']
+    assert meta_dec['fill_value'] is None
+    assert meta_dec['filters'] is None
+
+
+def test_encode_decode_array_structured():
+
+    meta = dict(
+        shape=(100,),
+        chunks=(10,),
+        dtype=np.dtype('i8, (10, 10)f8, (5, 10, 15)u1'),
+        compressor=Zlib(1).get_config(),
+        fill_value=None,
+        filters=None,
+        order='C'
+    )
+
+    meta_json = '''{
+        "chunks": [10],
+        "compressor": {"id": "zlib", "level": 1},
+        "dtype": [["f0", "<i8"], ["f1", "<f8", [10, 10]], ["f2", "|u1", [5, 10, 15]]],
+        "fill_value": null,
+        "filters": null,
+        "order": "C",
+        "shape": [100],
+        "zarr_format": %s
+    }''' % ZARR_FORMAT
+
+    # test encoding
+    meta_enc = encode_array_metadata(meta)
+    assert_json_equal(meta_json, meta_enc)
+
+    # test decoding
+    meta_dec = decode_array_metadata(meta_enc)
+    assert ZARR_FORMAT == meta_dec['zarr_format']
+    # to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
+    assert meta['shape'] + meta['dtype'].shape == meta_dec['shape']
+    assert meta['chunks'] == meta_dec['chunks']
+    # to maintain consistency with numpy unstructured arrays, unpack dimensions into shape
+    assert meta['dtype'].base == meta_dec['dtype']
+    assert meta['compressor'] == meta_dec['compressor']
+    assert meta['order'] == meta_dec['order']
+    assert meta_dec['fill_value'] is None
+    assert meta_dec['filters'] is None
+
+
 def test_encode_decode_fill_values_nan():
 
     fills = (