Exclude hidden files when uploading, and remove option to use third party client from CLI (#63)

* ignore hidden files

* remove third party tool option

* lint and bump version

Author: nathanthorpe

README.md

@@ -36,7 +36,7 @@ Options:
   --project TEXT         Name or ID of the project
   --dataset TEXT         ID of the dataset
   --data-directory TEXT  Directory to store the files
-  --interactive          Gather arguments interactively
+  -i, --interactive      Gather arguments interactively
   --help                 Show this message and exit.
 ```
 
@@ -52,8 +52,7 @@ Options:
   --project TEXT          Name or ID of the project
   --process TEXT          Name or ID of the ingest process
   --data-directory TEXT   Directory you wish to upload
-  --interactive           Gather arguments interactively
-  --use-third-party-tool  Use third party tool for upload (Generate manifest and one-time upload authentication token)
+  -i, --interactive       Gather arguments interactively
   --help                  Show this message and exit.
 ```
 
@@ -65,7 +64,7 @@ Usage: cirro-cli list-datasets [OPTIONS]
 
 Options:
   --project TEXT         ID of the project
-  --interactive          Gather arguments interactively
+  -i, --interactive      Gather arguments interactively
   --help                 Show this message and exit.
 ```
 
@@ -83,7 +82,6 @@ $ cirro-cli upload --interactive
 ? What type of files?  Illumina Sequencing Run
 ? What is the name of this dataset?  test
 ? Enter a description of the dataset (optional)
-? How would you like to upload or download your data? Cirro CLI
 ```
 
 ## Python Usage

cirro/cli/cli.py

@@ -59,9 +59,6 @@ def download(**kwargs):
 @click.option('-i', '--interactive',
               help='Gather arguments interactively',
               is_flag=True, default=False)
-@click.option('--use-third-party-tool',
-              help='Use third party tool for upload (Generate manifest and one-time upload authentication token)',
-              is_flag=True, default=False)
 def upload(**kwargs):
     check_required_args(kwargs)
     run_ingest(kwargs, interactive=kwargs.get('interactive'))

cirro/cli/controller.py

@@ -1,7 +1,6 @@
 from cirro.api.clients.portal import DataPortalClient
 from cirro.api.config import UserConfig, save_user_config
 from cirro.api.models.dataset import CreateIngestDatasetInput
-from cirro.api.models.file import FileAccessContext
 from cirro.api.models.process import Executor
 from cirro.cli.interactive.auth_args import gather_auth_config
 from cirro.cli.interactive.download_args import gather_download_arguments, ask_dataset_files
@@ -16,7 +15,6 @@
 from cirro.cli.models import ListArguments, UploadArguments, DownloadArguments
 from cirro.file_utils import get_files_in_directory
 from cirro.helpers import WorkflowConfigBuilder
-from cirro.utils import print_credentials
 
 
 def run_list_datasets(input_params: ListArguments, interactive=False):
@@ -78,24 +76,10 @@ def run_ingest(input_params: UploadArguments, interactive=False):
 
     create_resp = cirro.dataset.create(create_request)
 
-    if input_params['use_third_party_tool']:
-        token_lifetime = 1  # TODO: Max token lifetime is 1 hour?
-        access_context = FileAccessContext.upload_dataset(project_id=create_request.project_id,
-                                                          dataset_id=create_resp['datasetId'],
-                                                          token_lifetime_override=token_lifetime)
-        creds = cirro.file.get_access_credentials(access_context)
-        print()
-        print("Please use the following information in your tool:")
-        print(f"Bucket: {access_context.bucket}")
-        print(f"Data path: {create_resp['dataPath']}")
-        print()
-        print_credentials(creds)
-
-    else:
-        cirro.dataset.upload_files(dataset_id=create_resp['datasetId'],
-                                   project_id=create_request.project_id,
-                                   directory=directory,
-                                   files=files)
+    cirro.dataset.upload_files(dataset_id=create_resp['datasetId'],
+                               project_id=create_request.project_id,
+                               directory=directory,
+                               files=files)
 
 
 def run_download(input_params: DownloadArguments, interactive=False):

cirro/cli/interactive/common_args.py

@@ -15,17 +15,3 @@ def ask_project(projects: List[Project], input_value: str) -> str:
     }
     answers = prompt_wrapper(project_prompt)
     return answers['project']
-
-
-def ask_use_third_party_tool():
-    answers = prompt_wrapper({
-        'type': 'list',
-        'message': 'How would you like to upload or download your data?',
-        'name': 'use_tool',
-        'choices': [
-            "Cirro CLI",
-            "Third-party tool (using AWS credentials temporarily issued for download)"
-        ]
-    })
-
-    return answers['use_tool'] != 'Cirro CLI'

cirro/cli/interactive/upload_args.py

@@ -7,7 +7,7 @@
 
 from cirro.api.models.process import Process
 from cirro.api.models.project import Project
-from cirro.cli.interactive.common_args import ask_project, ask_use_third_party_tool
+from cirro.cli.interactive.common_args import ask_project
 from cirro.cli.interactive.utils import prompt_wrapper
 from cirro.cli.models import UploadArguments
 from cirro.file_utils import get_directory_stats
@@ -101,6 +101,4 @@ def gather_upload_arguments(input_params: UploadArguments, projects: List[Projec
     default_name = input_params.get('name') or data_directory_name
     input_params['name'] = ask_name(default_name)
     input_params['description'] = ask_description(input_params.get('description'))
-
-    input_params['use_third_party_tool'] = ask_use_third_party_tool()
     return input_params

cirro/cli/models.py

@@ -15,7 +15,6 @@ class UploadArguments(TypedDict):
     process: str
     data_directory: str
     interactive: bool
-    use_third_party_tool: bool
 
 
 class ListArguments(TypedDict):

cirro/file_utils.py

@@ -31,6 +31,9 @@ def get_files_in_directory(directory) -> List[str]:
     for file_path in path.rglob("*"):
         if file_path.is_dir():
             continue
+        if file_path.name.startswith('.'):
+            continue
+
         str_file_path = str(file_path.as_posix())
         str_file_path = str_file_path.replace(f'{path_posix}/', "")
         paths.append(str_file_path)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cirro"
-version = "0.6.6"
+version = "0.6.7"
 description = "CLI tool and SDK for interacting with the Cirro platform"
 authors = ["Fred Hutch <[email protected]>"]
 license = "MIT"