From 9a7afdcb4c741e5c01d6ab49739e7c19c0f36d93 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 22 Mar 2023 10:16:27 -0700 Subject: [PATCH 1/3] ignore hidden files --- cirro/file_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cirro/file_utils.py b/cirro/file_utils.py index 3dc7c786..c5f4a7a8 100644 --- a/cirro/file_utils.py +++ b/cirro/file_utils.py @@ -31,6 +31,9 @@ def get_files_in_directory(directory) -> List[str]: for file_path in path.rglob("*"): if file_path.is_dir(): continue + if file_path.name.startswith('.'): + continue + str_file_path = str(file_path.as_posix()) str_file_path = str_file_path.replace(f'{path_posix}/', "") paths.append(str_file_path) From 2c700da8032969a8014ae27f7af2cc494791f91c Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 22 Mar 2023 10:21:32 -0700 Subject: [PATCH 2/3] remove third party tool option --- README.md | 8 +++----- cirro/cli/cli.py | 3 --- cirro/cli/controller.py | 22 ++++------------------ cirro/cli/interactive/common_args.py | 14 -------------- cirro/cli/interactive/upload_args.py | 4 +--- cirro/cli/models.py | 1 - 6 files changed, 8 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 55ae0aa8..2a9b47e3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Options: --project TEXT Name or ID of the project --dataset TEXT ID of the dataset --data-directory TEXT Directory to store the files - --interactive Gather arguments interactively + -i, --interactive Gather arguments interactively --help Show this message and exit. ``` @@ -52,8 +52,7 @@ Options: --project TEXT Name or ID of the project --process TEXT Name or ID of the ingest process --data-directory TEXT Directory you wish to upload - --interactive Gather arguments interactively - --use-third-party-tool Use third party tool for upload (Generate manifest and one-time upload authentication token) + -i, --interactive Gather arguments interactively --help Show this message and exit. ``` @@ -65,7 +64,7 @@ Usage: cirro-cli list-datasets [OPTIONS] Options: --project TEXT ID of the project - --interactive Gather arguments interactively + -i, --interactive Gather arguments interactively --help Show this message and exit. ``` @@ -83,7 +82,6 @@ $ cirro-cli upload --interactive ? What type of files? Illumina Sequencing Run ? What is the name of this dataset? test ? Enter a description of the dataset (optional) -? How would you like to upload or download your data? Cirro CLI ``` ## Python Usage diff --git a/cirro/cli/cli.py b/cirro/cli/cli.py index 08f26b78..95bccc3f 100644 --- a/cirro/cli/cli.py +++ b/cirro/cli/cli.py @@ -59,9 +59,6 @@ def download(**kwargs): @click.option('-i', '--interactive', help='Gather arguments interactively', is_flag=True, default=False) -@click.option('--use-third-party-tool', - help='Use third party tool for upload (Generate manifest and one-time upload authentication token)', - is_flag=True, default=False) def upload(**kwargs): check_required_args(kwargs) run_ingest(kwargs, interactive=kwargs.get('interactive')) diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py index c6af68e4..031cec28 100644 --- a/cirro/cli/controller.py +++ b/cirro/cli/controller.py @@ -78,24 +78,10 @@ def run_ingest(input_params: UploadArguments, interactive=False): create_resp = cirro.dataset.create(create_request) - if input_params['use_third_party_tool']: - token_lifetime = 1 # TODO: Max token lifetime is 1 hour? - access_context = FileAccessContext.upload_dataset(project_id=create_request.project_id, - dataset_id=create_resp['datasetId'], - token_lifetime_override=token_lifetime) - creds = cirro.file.get_access_credentials(access_context) - print() - print("Please use the following information in your tool:") - print(f"Bucket: {access_context.bucket}") - print(f"Data path: {create_resp['dataPath']}") - print() - print_credentials(creds) - - else: - cirro.dataset.upload_files(dataset_id=create_resp['datasetId'], - project_id=create_request.project_id, - directory=directory, - files=files) + cirro.dataset.upload_files(dataset_id=create_resp['datasetId'], + project_id=create_request.project_id, + directory=directory, + files=files) def run_download(input_params: DownloadArguments, interactive=False): diff --git a/cirro/cli/interactive/common_args.py b/cirro/cli/interactive/common_args.py index 64a43d6e..27bc1ec8 100644 --- a/cirro/cli/interactive/common_args.py +++ b/cirro/cli/interactive/common_args.py @@ -15,17 +15,3 @@ def ask_project(projects: List[Project], input_value: str) -> str: } answers = prompt_wrapper(project_prompt) return answers['project'] - - -def ask_use_third_party_tool(): - answers = prompt_wrapper({ - 'type': 'list', - 'message': 'How would you like to upload or download your data?', - 'name': 'use_tool', - 'choices': [ - "Cirro CLI", - "Third-party tool (using AWS credentials temporarily issued for download)" - ] - }) - - return answers['use_tool'] != 'Cirro CLI' diff --git a/cirro/cli/interactive/upload_args.py b/cirro/cli/interactive/upload_args.py index 45c7fe12..fb8b785c 100644 --- a/cirro/cli/interactive/upload_args.py +++ b/cirro/cli/interactive/upload_args.py @@ -7,7 +7,7 @@ from cirro.api.models.process import Process from cirro.api.models.project import Project -from cirro.cli.interactive.common_args import ask_project, ask_use_third_party_tool +from cirro.cli.interactive.common_args import ask_project from cirro.cli.interactive.utils import prompt_wrapper from cirro.cli.models import UploadArguments from cirro.file_utils import get_directory_stats @@ -101,6 +101,4 @@ def gather_upload_arguments(input_params: UploadArguments, projects: List[Projec default_name = input_params.get('name') or data_directory_name input_params['name'] = ask_name(default_name) input_params['description'] = ask_description(input_params.get('description')) - - input_params['use_third_party_tool'] = ask_use_third_party_tool() return input_params diff --git a/cirro/cli/models.py b/cirro/cli/models.py index 8af9cf08..ab46f96e 100644 --- a/cirro/cli/models.py +++ b/cirro/cli/models.py @@ -15,7 +15,6 @@ class UploadArguments(TypedDict): process: str data_directory: str interactive: bool - use_third_party_tool: bool class ListArguments(TypedDict): From d581772f5371c9df8043914e0f8e71ff3a4b3387 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 22 Mar 2023 10:24:21 -0700 Subject: [PATCH 3/3] lint and bump version --- cirro/cli/controller.py | 2 -- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py index 031cec28..310e52b1 100644 --- a/cirro/cli/controller.py +++ b/cirro/cli/controller.py @@ -1,7 +1,6 @@ from cirro.api.clients.portal import DataPortalClient from cirro.api.config import UserConfig, save_user_config from cirro.api.models.dataset import CreateIngestDatasetInput -from cirro.api.models.file import FileAccessContext from cirro.api.models.process import Executor from cirro.cli.interactive.auth_args import gather_auth_config from cirro.cli.interactive.download_args import gather_download_arguments, ask_dataset_files @@ -16,7 +15,6 @@ from cirro.cli.models import ListArguments, UploadArguments, DownloadArguments from cirro.file_utils import get_files_in_directory from cirro.helpers import WorkflowConfigBuilder -from cirro.utils import print_credentials def run_list_datasets(input_params: ListArguments, interactive=False): diff --git a/pyproject.toml b/pyproject.toml index 3dd380ff..245658b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cirro" -version = "0.6.6" +version = "0.6.7" description = "CLI tool and SDK for interacting with the Cirro platform" authors = ["Fred Hutch "] license = "MIT"