From 56d3c0550af247b76ca9ab8e1b95738d8a9be5cb Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 07:33:27 -0700 Subject: [PATCH 01/11] remove config options and fix transfer retries issue --- cirro/api/config.py | 1 + cirro/cli/controller.py | 3 ++- cirro/cli/interactive/auth_args.py | 29 +++-------------------------- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/cirro/api/config.py b/cirro/api/config.py index 83a373ee..0fe09b34 100644 --- a/cirro/api/config.py +++ b/cirro/api/config.py @@ -31,6 +31,7 @@ def save_user_config(user_config: UserConfig): } if original_user_config: ini_config['General']['base_url'] = original_user_config.base_url + ini_config['General']['transfer_max_retries'] = str(original_user_config.transfer_max_retries) ini_config[user_config.auth_method] = user_config.auth_method_config Constants.config_path.parent.mkdir(exist_ok=True) diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py index 310e52b1..6cbf2816 100644 --- a/cirro/cli/controller.py +++ b/cirro/cli/controller.py @@ -171,4 +171,5 @@ def run_configure(): auth_method, auth_method_config = gather_auth_config() save_user_config(UserConfig(auth_method=auth_method, auth_method_config=auth_method_config, - base_url=None)) + base_url=None, + transfer_max_retries=None)) diff --git a/cirro/cli/interactive/auth_args.py b/cirro/cli/interactive/auth_args.py index cb6033a0..1885666b 100644 --- a/cirro/cli/interactive/auth_args.py +++ b/cirro/cli/interactive/auth_args.py @@ -5,31 +5,8 @@ def gather_auth_config() -> Tuple[str, Dict]: - auth_methods_map = { - 'Default authentication (recommended)': ClientAuth, - 'Username/password authentication (non-institutional)': UsernameAndPasswordAuth, - 'AWS IAM Credentials': IAMAuth + auth_method_config = { + 'enable_cache': ask_yes_no("Would you like to cache your login?") } - auth_method_answer = ask('select', 'Please select and authentication method', - choices=auth_methods_map.keys()) - - auth_method = auth_methods_map[auth_method_answer] - auth_method_config = {} - - if auth_method == UsernameAndPasswordAuth: - auth_method_config['username'] = ask('text', 'Please enter your username', required=True) - auth_method_config['password'] = ask('password', 'Please enter your password', required=True) - - if auth_method == ClientAuth: - auth_method_config['enable_cache'] = ask_yes_no("Would you like to cache your login?") - - if auth_method == IAMAuth: - if ask_yes_no("Would you like to manually specify AWS credentials?"): - auth_method_config['access_key'] = ask('text', 'Please enter your access key ID', required=True) - auth_method_config['secret_key'] = ask('text', 'Please enter your secret access key', required=True) - auth_method_config['token'] = ask('text', 'Please enter your session token (optional)') - else: - auth_method_config['load_current'] = True - - return auth_method.__name__, auth_method_config + return 'ClientAuth', auth_method_config From df15a3e25b2dcc6365f4d68f4c607688f3da2e73 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 07:45:04 -0700 Subject: [PATCH 02/11] readme change --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8be88f14..f4008eda 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,16 @@ or by cloning the repository and running: `python setup.py install` ## Set Up -Run a one-time configuration of your login credentials in the command line using: -`cirro-cli configure` +The Cirro client can save your login tokens securely, so you don't have to re-authenticate every time you use the CLI. - This will ask you to select an authentication method. If you are a member of Fred Hutch or the University of Washington, select the default method which will give you a link to use to log through the browser. If you are not a member of those institutions, select the non-institutional authentication method and enter your Data Portal username and password into the command line when prompted. +To set this up, run: + +``` +cirro-cli configure + +? Would you like to cache your login? Yes +``` ## Command Line Usage From b158df530dbb495569030d1e069bb362cfe52421 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 07:46:34 -0700 Subject: [PATCH 03/11] unused import --- cirro/cli/interactive/auth_args.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cirro/cli/interactive/auth_args.py b/cirro/cli/interactive/auth_args.py index 1885666b..3178c1df 100644 --- a/cirro/cli/interactive/auth_args.py +++ b/cirro/cli/interactive/auth_args.py @@ -1,7 +1,6 @@ from typing import Tuple, Dict -from cirro.api.auth import ClientAuth, UsernameAndPasswordAuth, IAMAuth -from cirro.cli.interactive.utils import ask, ask_yes_no +from cirro.cli.interactive.utils import ask_yes_no def gather_auth_config() -> Tuple[str, Dict]: From 705c3a2cc3fa0ab420f94e4906e46b1ace4aebd7 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 07:47:31 -0700 Subject: [PATCH 04/11] readme change --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f4008eda..d12ff4b5 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ cirro-cli configure ? Would you like to cache your login? Yes ``` +Upon first use, it will give you a link to authenticate through the web browser. ## Command Line Usage From 21974d402ffcb8cdb5636f643d2deb06a30c22f8 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 07:55:26 -0700 Subject: [PATCH 05/11] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 72a66eae..f5dea305 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cirro" -version = "0.6.10" +version = "0.6.11" description = "CLI tool and SDK for interacting with the Cirro platform" authors = ["Fred Hutch "] license = "MIT" From 753ee5740fed26df309d1e1cd823c5b2852e93c8 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 08:12:51 -0700 Subject: [PATCH 06/11] read in time with time zone information --- cirro/api/auth/oauth_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirro/api/auth/oauth_client.py b/cirro/api/auth/oauth_client.py index 29053661..c956afbf 100644 --- a/cirro/api/auth/oauth_client.py +++ b/cirro/api/auth/oauth_client.py @@ -59,7 +59,7 @@ def _authenticate(client_id: str, auth_endpoint: str): auth_status = 'authorization_pending' while auth_status == 'authorization_pending': time.sleep(flow['interval']) - if device_expiry < datetime.now(): + if device_expiry < datetime.now().astimezone(): raise RuntimeError('Authentication timed out') resp = requests.post(f'{auth_endpoint}/token', params=params) From 4d0d69eb52c5fc8f4f8e034d94e064e174f315a7 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 08:52:25 -0700 Subject: [PATCH 07/11] prompt for configuration if configuration not found --- cirro/api/config.py | 2 ++ cirro/api/services/file.py | 2 +- cirro/cli/controller.py | 19 +++++++++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cirro/api/config.py b/cirro/api/config.py index 0fe09b34..dba5dc6c 100644 --- a/cirro/api/config.py +++ b/cirro/api/config.py @@ -73,6 +73,8 @@ def __init__(self, base_url: str = None): os.environ.get('PW_BASE_URL') or (self.user_config.base_url if self.user_config else None) or Constants.default_base_url) + self.transfer_max_retries = self.user_config.transfer_max_retries\ + if self.user_config else Constants.default_max_retries self._init_config() def _init_config(self): diff --git a/cirro/api/services/file.py b/cirro/api/services/file.py index a62aac95..37b07003 100644 --- a/cirro/api/services/file.py +++ b/cirro/api/services/file.py @@ -55,7 +55,7 @@ def upload_files(self, access_context: FileAccessContext, directory: str, files: """ s3_client = S3Client(partial(self.get_access_credentials, access_context), self._configuration.region) upload_directory(directory, files, s3_client, access_context.bucket, access_context.path_prefix, - max_retries=self._configuration.user_config.transfer_max_retries) + max_retries=self._configuration.transfer_max_retries) def download_files(self, access_context: FileAccessContext, directory: str, files: List[str]): """ diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py index 6cbf2816..a7123913 100644 --- a/cirro/cli/controller.py +++ b/cirro/cli/controller.py @@ -1,5 +1,5 @@ from cirro.api.clients.portal import DataPortalClient -from cirro.api.config import UserConfig, save_user_config +from cirro.api.config import UserConfig, save_user_config, load_user_config from cirro.api.models.dataset import CreateIngestDatasetInput from cirro.api.models.process import Executor from cirro.cli.interactive.auth_args import gather_auth_config @@ -19,8 +19,7 @@ def run_list_datasets(input_params: ListArguments, interactive=False): """List the datasets available in a particular project.""" - - # Instantiate the Cirro Data Portal client + _check_configure() cirro = DataPortalClient() # If the user provided the --interactive flag @@ -47,6 +46,7 @@ def run_list_datasets(input_params: ListArguments, interactive=False): def run_ingest(input_params: UploadArguments, interactive=False): + _check_configure() cirro = DataPortalClient() projects = cirro.project.list() processes = cirro.process.list(process_type=Executor.INGEST) @@ -83,6 +83,7 @@ def run_ingest(input_params: UploadArguments, interactive=False): def run_download(input_params: DownloadArguments, interactive=False): + _check_configure() cirro = DataPortalClient() projects = cirro.project.list() @@ -114,7 +115,7 @@ def run_download(input_params: DownloadArguments, interactive=False): def run_configure_workflow(): """Configure a workflow to be run in the Data Portal as a process.""" - + _check_configure() cirro = DataPortalClient() process_options = cirro.process.list(process_type=Executor.NEXTFLOW) resources_folder, repo_prefix = get_output_resources_path() @@ -173,3 +174,13 @@ def run_configure(): auth_method_config=auth_method_config, base_url=None, transfer_max_retries=None)) + + +def _check_configure(): + """ + Prompts the user to do initial configuration if needed + """ + if load_user_config() is not None: + return + + run_configure() From 30079295be1b3859b39f916a9236b55140c06161 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 08:59:46 -0700 Subject: [PATCH 08/11] add timezone info to refresh expiry as well --- cirro/api/auth/oauth_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirro/api/auth/oauth_client.py b/cirro/api/auth/oauth_client.py index c956afbf..95c5e54f 100644 --- a/cirro/api/auth/oauth_client.py +++ b/cirro/api/auth/oauth_client.py @@ -102,7 +102,7 @@ def __init__(self, client_id: str, region: str, auth_endpoint: str, enable_cache if self._token_info and self._token_info.get('refresh_expires_in'): refresh_expiry_threshold = datetime.fromtimestamp(self._token_info.get('refresh_expires_in'))\ - timedelta(hours=12) - if refresh_expiry_threshold < datetime.now(): + if refresh_expiry_threshold < datetime.now().astimezone(): logger.debug('Refresh token expiry is too soon, re-authenticating') self._clear_token_info() From 8b069321a552800bef6688dbf08a4f03e1698b03 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 09:17:18 -0700 Subject: [PATCH 09/11] remove tz info since it's a timestamp --- cirro/api/auth/oauth_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirro/api/auth/oauth_client.py b/cirro/api/auth/oauth_client.py index 95c5e54f..c956afbf 100644 --- a/cirro/api/auth/oauth_client.py +++ b/cirro/api/auth/oauth_client.py @@ -102,7 +102,7 @@ def __init__(self, client_id: str, region: str, auth_endpoint: str, enable_cache if self._token_info and self._token_info.get('refresh_expires_in'): refresh_expiry_threshold = datetime.fromtimestamp(self._token_info.get('refresh_expires_in'))\ - timedelta(hours=12) - if refresh_expiry_threshold < datetime.now().astimezone(): + if refresh_expiry_threshold < datetime.now(): logger.debug('Refresh token expiry is too soon, re-authenticating') self._clear_token_info() From fb5ee0f67163f70a51246835a64c46ad7365ffae Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 09:47:25 -0700 Subject: [PATCH 10/11] change save message and readme --- README.md | 21 +++++++++------------ cirro/cli/interactive/auth_args.py | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d12ff4b5..e469da59 100644 --- a/README.md +++ b/README.md @@ -16,19 +16,11 @@ or by cloning the repository and running: `python setup.py install` -## Set Up +## Authentication -The Cirro client can save your login tokens securely, so you don't have to re-authenticate every time you use the CLI. +Upon first use, the Cirro client will ask if you would like to save your login information and give you a link to authenticate through the web browser. -To set this up, run: - -``` -cirro-cli configure - -? Would you like to cache your login? Yes -``` - -Upon first use, it will give you a link to authenticate through the web browser. +If you need to change your credentials after this point, and you've opted to save your login, please see the [clearing saved login](#clearing-saved-login) section. ## Command Line Usage @@ -132,4 +124,9 @@ It will pause for an increasing amount of time for each retry attempt. [General] base_url = data-portal.io transfer_max_retries = 15 -``` \ No newline at end of file +``` + +### Clearing saved login + +You can clear your saved login information by removing the `~/.cirro/token.dat` file from your system or +by running `cirro-cli configure` and selecting **No** when it asks if you'd like to save your login information. diff --git a/cirro/cli/interactive/auth_args.py b/cirro/cli/interactive/auth_args.py index 3178c1df..eba4f57d 100644 --- a/cirro/cli/interactive/auth_args.py +++ b/cirro/cli/interactive/auth_args.py @@ -5,7 +5,7 @@ def gather_auth_config() -> Tuple[str, Dict]: auth_method_config = { - 'enable_cache': ask_yes_no("Would you like to cache your login?") + 'enable_cache': ask_yes_no("Would you like to cache your login? (do not use this on shared devices)") } return 'ClientAuth', auth_method_config From 8406a88403d1005f1faaf0d4b5b83def2984c439 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Wed, 17 May 2023 09:52:59 -0700 Subject: [PATCH 11/11] cache --> save --- cirro/cli/interactive/auth_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirro/cli/interactive/auth_args.py b/cirro/cli/interactive/auth_args.py index eba4f57d..96960041 100644 --- a/cirro/cli/interactive/auth_args.py +++ b/cirro/cli/interactive/auth_args.py @@ -5,7 +5,7 @@ def gather_auth_config() -> Tuple[str, Dict]: auth_method_config = { - 'enable_cache': ask_yes_no("Would you like to cache your login? (do not use this on shared devices)") + 'enable_cache': ask_yes_no("Would you like to save your login? (do not use this on shared devices)") } return 'ClientAuth', auth_method_config