Add test for copying across filesystems.

tiledb/tests/common.py

@@ -70,6 +70,19 @@ def create_vfs_dir(path):
     vfs.create_dir(path)
 
 
+def vfs_path(scheme: str) -> str:
+    prefix = "tiledb-"
+    if scheme == "s3":
+        prefix += str(random.randint(0, 10000000000))
+    else:
+        prefix += str(uuid.uuid4())
+
+    if scheme == "file":
+        return f"{scheme}://{tempfile.mktemp(prefix=prefix)}"
+    else:
+        return f"{scheme}://{prefix}"
+
+
 class DiskTestCase:
     """Helper class to store paths and associated allocation frames. This is both
     a cleanup step and a test of resource management. Some platforms will

tiledb/tests/conftest.py

@@ -72,3 +72,36 @@ def original_os_fork():
     """Provides the original unpatched os.fork."""
     if sys.platform != "win32":
         return os.fork
+
+
+@pytest.fixture
+def vfs_config() -> dict[str, str]:
+    config: dict[str, str] = {}
+    # Configure S3
+    if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
+        config["vfs.s3.aws_access_key_id"] = os.getenv("AWS_ACCESS_KEY_ID")
+        config["vfs.s3.aws_secret_access_key"] = os.getenv("AWS_SECRET_ACCESS_KEY")
+        if os.getenv("VFS_S3_USE_MINIO"):
+            config["vfs.s3.endpoint_override"] = "localhost:9999"
+            config["vfs.s3.scheme"] = "https"
+            config["vfs.s3.use_virtual_addressing"] = "false"
+            config["vfs.s3.verify_ssl"] = "false"
+
+    #   Configure Azure
+    if os.getenv("AZURE_BLOB_ENDPOINT"):
+        config["vfs.azure.blob_endpoint"] = os.getenv("AZURE_BLOB_ENDPOINT")
+    if os.getenv("AZURE_STORAGE_ACCOUNT_TOKEN"):
+        config["vfs.azure.storage_sas_token"] = os.getenv("AZURE_STORAGE_ACCOUNT_TOKEN")
+    elif os.getenv("AZURE_STORAGE_ACCOUNT_NAME") and os.getenv(
+        "AZURE_STORAGE_ACCOUNT_KEY"
+    ):
+        config["vfs.azure.storage_account_name"] = os.getenv(
+            "AZURE_STORAGE_ACCOUNT_NAME"
+        )
+        config["vfs.azure.storage_account_key"] = os.getenv("AZURE_STORAGE_ACCOUNT_KEY")
+
+    #   Configure Google Cloud
+    if os.getenv("TILEDB_TEST_GCS_ENDPOINT"):
+        config["vfs.gcs.endpoint"] = os.getenv("TILEDB_TEST_GCS_ENDPOINT")
+
+    return config

tiledb/tests/test_vfs.py

@@ -4,13 +4,16 @@
 import pickle
 import random
 import sys
+import tempfile
 
 import numpy as np
 import pytest
 
 import tiledb
 
-from .common import DiskTestCase, rand_utf8
+from .common import DiskTestCase, rand_utf8, vfs_path
+
+s3_bucket = os.getenv("S3_BUCKET")
 
 
 class TestVFS(DiskTestCase):
@@ -115,6 +118,78 @@ def test_copy(self):
             with self.assertRaises(tiledb.TileDBError):
                 vfs.copy_dir(self.path("foo/baz"), self.path("do_not_exist/baz"))
 
+    # Note: Azure tests are intermittently failing.
+    # Seemingly mostly azure->azure, but also test teardown.
+    # I think it's an Azurite bucket storage thing, actually.
+    @pytest.mark.skipif(
+        sys.platform == "win32",
+        reason="Windows paths are difficult; Posix is sufficient for testing.",
+    )
+    @pytest.mark.parametrize("src", ["file", "s3", "azure", "gcs"])
+    @pytest.mark.parametrize("dst", ["file", "s3", "azure", "gcs"])
+    def test_copy_across(self, src: str, dst: str, vfs_config):
+        # Currently, skip if both FSes are the same. This is covered above.
+        # However, this test might ought to replace the above.
+        if src == dst:
+            return
+
+        # Set configuration options
+        if (src == "s3" or dst == "s3") and not vfs_config.get(
+            "vfs.s3.aws_access_key_id"
+        ):
+            return
+        if (src == "azure" or dst == "azure") and not any(
+            x.startswith("vfs.azure") for x in vfs_config.keys()
+        ):
+            return
+        if (src == "gcs" or dst == "gcs") and not vfs_config.get("vfs.gcs.endpoint"):
+            return
+        vfs = tiledb.VFS(vfs_config)
+
+        # Return if neither filesystem is supported.
+        if not vfs.supports(src) or not vfs.supports(dst):
+            return
+
+        # Create source file, and write to it.
+        srcdir: str = vfs_path(src)
+        if src == "file":
+            vfs.create_dir(srcdir)
+            self.assertTrue(vfs.is_dir(srcdir))
+        else:
+            vfs.create_bucket(srcdir)
+            self.assertTrue(vfs.is_bucket(srcdir))
+        srcfile: str = f"{srcdir}/testfile"
+        vfs.touch(srcfile)
+        self.assertTrue(vfs.isfile(srcfile))
+        contents: bytes = b"TileDB test copying across filesystems."
+        with vfs.open(srcfile, "wb") as handle:
+            handle.write(contents)
+        with vfs.open(srcfile) as handle:
+            self.assertEqual(handle.read(), contents)
+
+        # Copy src -> dst and assert the file contents are unchanged.
+        dstdir: str = vfs_path(dst)
+        if dst == "file":
+            vfs.create_dir(dstdir)
+            self.assertTrue(vfs.is_dir(dstdir))
+        else:
+            vfs.create_bucket(dstdir)
+            self.assertTrue(vfs.is_bucket(dstdir))
+        dstfile: str = f"{dstdir}/testfile"
+        vfs.copy_dir(srcdir, dstdir)
+        with vfs.open(dstfile) as handle:
+            self.assertEqual(handle.read(), contents)
+
+        # Clean up
+        if src == "file":
+            vfs.remove_dir(srcdir)
+        else:
+            vfs.remove_bucket(srcdir)
+        if dst == "file":
+            vfs.remove_dir(dstdir)
+        else:
+            vfs.remove_bucket(dstdir)
+
     def test_write_read(self):
         vfs = tiledb.VFS()