Merge branch 'master' into default-write-empty-chunks-false

Author: joshmoore

.github/workflows/python-package.yml

@@ -32,7 +32,7 @@ jobs:
         ports:
           - 6379:6379
       mongodb:
-        image: mongo:3.4.23
+        image: mongo:4.4.11
         ports:
           - 27017:27017
     steps:

requirements_dev_minimal.txt

@@ -1,8 +1,8 @@
 # library requirements
 asciitree==0.3.3
-fasteners==0.16.3
+fasteners==0.17.3
 numcodecs==0.9.1
 msgpack-python==0.5.6
-setuptools-scm==6.3.2
+setuptools-scm==6.4.2
 # test requirements
 pytest==6.2.5

requirements_dev_numpy.txt

@@ -1,4 +1,4 @@
 # Break this out into a separate file to allow testing against
 # different versions of numpy. This file should pin to the latest
 # numpy version.
-numpy==1.22.0
+numpy==1.22.1

requirements_dev_optional.txt

@@ -7,18 +7,18 @@ ipytree==0.2.1
 # don't let pyup change pinning for azure-storage-blob, need to pin to older
 # version to get compatibility with azure storage emulator on appveyor (FIXME)
 azure-storage-blob==12.8.1 # pyup: ignore
-redis==4.1.0
+redis==4.1.1
 types-redis
 types-setuptools
-pymongo==3.12.1
+pymongo==4.0.1
 # optional test requirements
 tox==3.24.5
 coverage
 flake8==4.0.1
 pytest-cov==3.0.0
 pytest-doctestplus==0.11.2
-pytest-timeout==2.0.2
+pytest-timeout==2.1.0
 h5py==3.6.0
-fsspec==2021.11.1
-s3fs==2021.11.1
+fsspec==2022.1.0
+s3fs==2022.1.0
 moto[server]>=1.3.14

zarr/convenience.py

@@ -1117,7 +1117,7 @@ def consolidate_metadata(store: StoreLike, metadata_key=".zmetadata"):
     open_consolidated
 
     """
-    store = normalize_store_arg(store)
+    store = normalize_store_arg(store, clobber=True)
 
     def is_zarr_key(key):
         return (key.endswith('.zarray') or key.endswith('.zgroup') or

zarr/creation.py

@@ -69,7 +69,9 @@ def create(shape, chunks=True, dtype=None, compressor='default',
         A codec to encode object arrays, only needed if dtype=object.
     dimension_separator : {'.', '/'}, optional
         Separator placed between the dimensions of a chunk.
+
         .. versionadded:: 2.8
+
     write_empty_chunks : bool, optional
         If True, all chunks will be stored regardless of their contents. If
         False (default), each chunk is compared to the array's fill value prior

zarr/storage.py

@@ -1551,7 +1551,7 @@ def __getitem__(self, key):
     def __setitem__(self, key, value):
         if self.mode == 'r':
             raise ReadOnlyError()
-        value = ensure_contiguous_ndarray(value)
+        value = ensure_contiguous_ndarray(value).view("u1")
         with self.mutex:
             # writestr(key, value) writes with default permissions from
             # zipfile (600) that are too restrictive, build ZipInfo for

zarr/tests/test_storage.py

@@ -19,6 +19,7 @@
 
 import zarr
 from zarr.codecs import BZ2, AsType, Blosc, Zlib
+from zarr.convenience import consolidate_metadata
 from zarr.errors import MetadataError
 from zarr.hierarchy import group
 from zarr.meta import ZARR_FORMAT, decode_array_metadata
@@ -1009,6 +1010,10 @@ def test_create(self):
         assert "data" in os.listdir(path1)
         assert ".zgroup" in os.listdir(path1)
 
+        # consolidated metadata (GH#915)
+        consolidate_metadata("file://" + path1)
+        assert ".zmetadata" in os.listdir(path1)
+
         g = zarr.open_group("simplecache::file://" + path1, mode='r',
                             storage_options={"cache_storage": path2,
                                              "same_names": True})
@@ -1549,6 +1554,13 @@ def test_permissions(self):
             assert perm == '0o40775'
         z.close()
 
+    def test_store_and_retrieve_ndarray(self):
+        store = ZipStore('data/store.zip')
+        x = np.array([[1, 2], [3, 4]])
+        store['foo'] = x
+        y = np.frombuffer(store['foo'], dtype=x.dtype).reshape(x.shape)
+        assert np.array_equiv(y, x)
+
 
 class TestDBMStore(StoreTests):