CI-830 - Allow specifying files to upload on non-interactive CLI (#170)

* allow specifying files on upload

* refresh readme examples

* update data-type example

Author: nathanthorpe

README.md

@@ -29,6 +29,7 @@ If you need to change your credentials after this point, and you've opted to sav
 ## Command Line Usage
 
 #### Downloading a dataset:
+
 ```bash
 Usage: cirro download [OPTIONS]
 
@@ -37,29 +38,41 @@ Usage: cirro download [OPTIONS]
 Options:
   --project TEXT         Name or ID of the project
   --dataset TEXT         ID of the dataset
-  --file... TEXT         Name and relative path of the file (optional)
+  --file TEXT            Relative path of the file(s) to download (optional, can be used multiple times)
   --data-directory TEXT  Directory to store the files
   -i, --interactive      Gather arguments interactively
   --help                 Show this message and exit.
 ```
 
+```bash
+$ cirro download --project "Test Project 1" --dataset "test" --data-directory "~/download"
+```
+
 #### Uploading a dataset:
+
 ```bash
 Usage: cirro upload [OPTIONS]
 
   Upload and create a dataset
 
 Options:
-  --name TEXT             Name of the dataset
-  --description TEXT      Description of the dataset (optional)
-  --project TEXT          Name or ID of the project
-  --data-type TEXT        Name or ID of the data type (ingest process)
-  --data-directory TEXT   Directory you wish to upload
-  -i, --interactive       Gather arguments interactively
-  --help                  Show this message and exit.
+  --name TEXT                  Name of the dataset
+  --description TEXT           Description of the dataset (optional)
+  --project TEXT               Name or ID of the project
+  --data-type, --process TEXT  Name or ID of the data type (--process is deprecated)
+  --data-directory TEXT        Directory you wish to upload
+  --file TEXT                  Relative path of the file(s) to upload (optional, can be used multiple times)
+  -i, --interactive            Gather arguments interactively
+  --include-hidden             Include hidden files in the upload (e.g., files starting with .)
+  --help                       Show this message and exit.
+```
+
+```bash
+$ cirro upload --project "Test Project 1" --name "test" --file "sample1.fastq.gz" --file "sample2.fastq.gz" --data-directory "~/data" --data-type "Paired DNAseq (FASTQ)" 
 ```
 
 #### Uploading a reference
+
 ```bash
 Usage: cirro upload-reference [OPTIONS]
 
@@ -69,10 +82,9 @@ Options:
   --name TEXT            Name of the reference
   --reference-type TEXT  Type of the reference (e.g., Reference Genome (FASTA))
   --project TEXT         Name or ID of the project
-  --reference-file TEXT  Location of reference file to upload (can specify multiple files)
+  --reference-file TEXT  Location of reference file(s) to upload (can be used multiple times)
   -i, --interactive      Gather arguments interactively
   --help                 Show this message and exit.
-
 ```
 
 #### Listing datasets:

cirro/cli/cli.py

@@ -38,7 +38,7 @@ def list_datasets(**kwargs):
 @click.option('--dataset',
               help='ID of the dataset')
 @click.option('--file',
-              help='Name and relative path of the file (optional)',
+              help='Relative path of the file(s) to download (optional, can be used multiple times)',
               default=[],
               multiple=True)
 @click.option('--data-directory',
@@ -63,6 +63,10 @@ def download(**kwargs):
               help='Name or ID of the data type (--process is deprecated)')
 @click.option('--data-directory',
               help='Directory you wish to upload')
+@click.option('--file',
+              help='Relative path of the file(s) to upload (optional, can be used multiple times)',
+              default=[],
+              multiple=True)
 @click.option('-i', '--interactive',
               help='Gather arguments interactively',
               is_flag=True, default=False)
@@ -82,7 +86,7 @@ def upload(**kwargs):
 @click.option('--project',
               help='Name or ID of the project')
 @click.option('--reference-file',
-              help='Location of reference file to upload (can specify multiple files)',
+              help='Location of reference file(s) to upload (can be used multiple times)',
               multiple=True)
 @click.option('-i', '--interactive',
               help='Gather arguments interactively',

cirro/cli/controller.py

@@ -16,7 +16,7 @@
 from cirro.cli.interactive.list_dataset_args import gather_list_arguments
 from cirro.cli.interactive.upload_args import gather_upload_arguments
 from cirro.cli.interactive.upload_reference_args import gather_reference_upload_arguments
-from cirro.cli.interactive.utils import get_id_from_name, get_item_from_name_or_id, InputError
+from cirro.cli.interactive.utils import get_id_from_name, get_item_from_name_or_id, InputError, validate_files
 from cirro.cli.models import ListArguments, UploadArguments, DownloadArguments, CreatePipelineConfigArguments, \
     UploadReferenceArguments
 from cirro.config import UserConfig, save_user_config, load_user_config
@@ -84,7 +84,14 @@ def run_ingest(input_params: UploadArguments, interactive=False):
         input_params['project'] = get_id_from_name(projects, input_params['project'])
         input_params['data_type'] = get_id_from_name(processes, input_params['data_type'])
         directory = input_params['data_directory']
-        files = get_files_in_directory(directory)
+        all_files = get_files_in_directory(directory)
+        if input_params['file']:
+            files = input_params['file']
+            validate_files(all_files, files, directory)
+
+        # Default to all files if file param is not provided
+        else:
+            files = all_files
 
     if len(files) == 0:
         raise InputError("No files to upload")

cirro/cli/interactive/utils.py

@@ -1,8 +1,8 @@
 from pathlib import Path
 from typing import List, Union, Callable, TypeVar, Optional
 
-from prompt_toolkit.validation import Validator, ValidationError
 import questionary
+from prompt_toolkit.validation import Validator, ValidationError
 from questionary import prompt
 
 
@@ -108,3 +108,9 @@ def get_item_from_name_or_id(items: List[T], name_or_id: str) -> Optional[T]:
     if matched:
         return matched
     return next((p for p in items if p.name == name_or_id), None)
+
+
+def validate_files(all_files: List[str], files: List[str], directory: str):
+    for file in files:
+        if file not in all_files:
+            raise InputError(f"File '{file}' not found in directory '{directory}'")

cirro/cli/models.py

@@ -1,11 +1,12 @@
-from typing import TypedDict, List, Optional
+from typing import TypedDict, Optional
 
 
 class DownloadArguments(TypedDict):
     project: str
     dataset: str
     data_directory: str
     interactive: bool
+    file: Optional[list[str]]
 
 
 class UploadArguments(TypedDict):
@@ -16,7 +17,7 @@ class UploadArguments(TypedDict):
     data_directory: str
     include_hidden: bool
     interactive: bool
-    files: Optional[List[str]]
+    file: Optional[list[str]]
 
 
 class ListArguments(TypedDict):