S3 region fix and improve hidden file check for Windows (#64)

* specify region when initializing s3 client

* improve hidden file check for windows

* param change

* bump version

* import lint

Author: nathanthorpe

cirro/api/clients/s3.py

@@ -43,9 +43,9 @@ def __call__(self, bytes_amount):
 
 
 class S3Client:
-    def __init__(self, creds_getter: Callable[[], Creds]):
+    def __init__(self, creds_getter: Callable[[], Creds], region_name: str):
         self._creds_getter = creds_getter
-        self._client = self._build_session_client()
+        self._client = self._build_session_client(region_name)
 
     def upload_file(self, local_path: Path, bucket: str, key: str):
         file_size = local_path.stat().st_size
@@ -88,7 +88,7 @@ def get_file(self, bucket: str, key: str) -> bytes:
     def get_file_stats(self, bucket: str, key: str):
         return self._client.head_object(Bucket=bucket, Key=key)
 
-    def _build_session_client(self):
+    def _build_session_client(self, region_name: str):
         creds = self._creds_getter()
 
         if creds['Expiration']:
@@ -105,7 +105,7 @@ def _build_session_client(self):
                 aws_secret_access_key=creds['SecretAccessKey'],
                 aws_session_token=creds['SessionToken']
             )
-        return session.client('s3')
+        return session.client('s3', region_name=region_name)
 
     def _refresh_credentials(self):
         new_creds = self._creds_getter()

cirro/api/services/file.py

@@ -30,7 +30,7 @@ def get_file_from_path(self, access_context: FileAccessContext, file_path: str)
         """
         Gets the contents of a file by providing the path, used internally
         """
-        s3_client = S3Client(partial(self.get_access_credentials, access_context))
+        s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region)
         full_path = f'{access_context.path_prefix}/{file_path}'.lstrip('/')
         return s3_client.get_file(access_context.bucket, full_path)
 
@@ -39,7 +39,7 @@ def create_file(self, access_context: FileAccessContext, key: str,
         """
         Creates a file at the specified path
         """
-        s3_client = S3Client(partial(self.get_access_credentials, access_context))
+        s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region)
         s3_client.create_object(key=key,
                                 contents=contents,
                                 content_type=content_type,
@@ -53,7 +53,7 @@ def upload_files(self, access_context: FileAccessContext, directory: str, files:
         :param files: relative path of files to upload
         :return:
         """
-        s3_client = S3Client(partial(self.get_access_credentials, access_context))
+        s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region)
         upload_directory(directory, files, s3_client, access_context.bucket, access_context.path_prefix)
 
     def download_files(self, access_context: FileAccessContext, directory: str, files: List[str]):
@@ -63,7 +63,7 @@ def download_files(self, access_context: FileAccessContext, directory: str, file
         :param directory: download location
         :param files: relative path of files to download
         """
-        s3_client = S3Client(partial(self.get_access_credentials, access_context))
+        s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region)
         download_directory(directory, files, s3_client, access_context.bucket, access_context.path_prefix)
 
     def get_file_listing(self, access_context: FileAccessContext) -> List[File]:

cirro/file_utils.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path, PurePath
 from typing import List, Union
 
@@ -6,6 +7,10 @@
 from cirro.api.clients import S3Client
 from cirro.api.models.file import DirectoryStatistics, File
 
+if os.name == 'nt':
+    import win32api
+    import win32con
+
 DEFAULT_TRANSFER_SPEED = 160
 
 
@@ -22,6 +27,15 @@ def matches_glob(file: Union[File, str]):
     ]
 
 
+def _is_hidden_file(file_path: Path):
+    # Remove hidden files from listing, desktop.ini .DS_Store, etc.
+    if os.name == 'nt':
+        attributes = win32api.GetFileAttributes(str(file_path))
+        return attributes & (win32con.FILE_ATTRIBUTE_HIDDEN | win32con.FILE_ATTRIBUTE_SYSTEM)
+    else:
+        return file_path.name.startswith('.')
+
+
 def get_files_in_directory(directory) -> List[str]:
     path = Path(directory)
     path_posix = str(path.as_posix())
@@ -31,7 +45,8 @@ def get_files_in_directory(directory) -> List[str]:
     for file_path in path.rglob("*"):
         if file_path.is_dir():
             continue
-        if file_path.name.startswith('.'):
+
+        if _is_hidden_file(file_path):
             continue
 
         str_file_path = str(file_path.as_posix())

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cirro"
-version = "0.6.7"
+version = "0.6.8"
 description = "CLI tool and SDK for interacting with the Cirro platform"
 authors = ["Fred Hutch <[email protected]>"]
 license = "MIT"