diff --git a/cirro/api/clients/s3.py b/cirro/api/clients/s3.py index b14f27b7..5e86672f 100644 --- a/cirro/api/clients/s3.py +++ b/cirro/api/clients/s3.py @@ -43,9 +43,9 @@ def __call__(self, bytes_amount): class S3Client: - def __init__(self, creds_getter: Callable[[], Creds]): + def __init__(self, creds_getter: Callable[[], Creds], region_name: str): self._creds_getter = creds_getter - self._client = self._build_session_client() + self._client = self._build_session_client(region_name) def upload_file(self, local_path: Path, bucket: str, key: str): file_size = local_path.stat().st_size @@ -88,7 +88,7 @@ def get_file(self, bucket: str, key: str) -> bytes: def get_file_stats(self, bucket: str, key: str): return self._client.head_object(Bucket=bucket, Key=key) - def _build_session_client(self): + def _build_session_client(self, region_name: str): creds = self._creds_getter() if creds['Expiration']: @@ -105,7 +105,7 @@ def _build_session_client(self): aws_secret_access_key=creds['SecretAccessKey'], aws_session_token=creds['SessionToken'] ) - return session.client('s3') + return session.client('s3', region_name=region_name) def _refresh_credentials(self): new_creds = self._creds_getter() diff --git a/cirro/api/services/file.py b/cirro/api/services/file.py index a2af14ec..14f818fa 100644 --- a/cirro/api/services/file.py +++ b/cirro/api/services/file.py @@ -30,7 +30,7 @@ def get_file_from_path(self, access_context: FileAccessContext, file_path: str) """ Gets the contents of a file by providing the path, used internally """ - s3_client = S3Client(partial(self.get_access_credentials, access_context)) + s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region) full_path = f'{access_context.path_prefix}/{file_path}'.lstrip('/') return s3_client.get_file(access_context.bucket, full_path) @@ -39,7 +39,7 @@ def create_file(self, access_context: FileAccessContext, key: str, """ Creates a file at the specified path """ - s3_client = S3Client(partial(self.get_access_credentials, access_context)) + s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region) s3_client.create_object(key=key, contents=contents, content_type=content_type, @@ -53,7 +53,7 @@ def upload_files(self, access_context: FileAccessContext, directory: str, files: :param files: relative path of files to upload :return: """ - s3_client = S3Client(partial(self.get_access_credentials, access_context)) + s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region) upload_directory(directory, files, s3_client, access_context.bucket, access_context.path_prefix) def download_files(self, access_context: FileAccessContext, directory: str, files: List[str]): @@ -63,7 +63,7 @@ def download_files(self, access_context: FileAccessContext, directory: str, file :param directory: download location :param files: relative path of files to download """ - s3_client = S3Client(partial(self.get_access_credentials, access_context)) + s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region) download_directory(directory, files, s3_client, access_context.bucket, access_context.path_prefix) def get_file_listing(self, access_context: FileAccessContext) -> List[File]: diff --git a/cirro/file_utils.py b/cirro/file_utils.py index c5f4a7a8..149d209e 100644 --- a/cirro/file_utils.py +++ b/cirro/file_utils.py @@ -1,3 +1,4 @@ +import os from pathlib import Path, PurePath from typing import List, Union @@ -6,6 +7,10 @@ from cirro.api.clients import S3Client from cirro.api.models.file import DirectoryStatistics, File +if os.name == 'nt': + import win32api + import win32con + DEFAULT_TRANSFER_SPEED = 160 @@ -22,6 +27,15 @@ def matches_glob(file: Union[File, str]): ] +def _is_hidden_file(file_path: Path): + # Remove hidden files from listing, desktop.ini .DS_Store, etc. + if os.name == 'nt': + attributes = win32api.GetFileAttributes(str(file_path)) + return attributes & (win32con.FILE_ATTRIBUTE_HIDDEN | win32con.FILE_ATTRIBUTE_SYSTEM) + else: + return file_path.name.startswith('.') + + def get_files_in_directory(directory) -> List[str]: path = Path(directory) path_posix = str(path.as_posix()) @@ -31,7 +45,8 @@ def get_files_in_directory(directory) -> List[str]: for file_path in path.rglob("*"): if file_path.is_dir(): continue - if file_path.name.startswith('.'): + + if _is_hidden_file(file_path): continue str_file_path = str(file_path.as_posix()) diff --git a/pyproject.toml b/pyproject.toml index 245658b9..1555eac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cirro" -version = "0.6.7" +version = "0.6.8" description = "CLI tool and SDK for interacting with the Cirro platform" authors = ["Fred Hutch "] license = "MIT"