From f5c56ed075360a4cfb5f774c258031c7701f6dfc Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Tue, 9 Sep 2025 11:53:02 -0700 Subject: [PATCH 1/3] allow specifying files on upload --- cirro/cli/cli.py | 8 ++++++-- cirro/cli/controller.py | 11 +++++++++-- cirro/cli/interactive/utils.py | 8 +++++++- cirro/cli/models.py | 5 +++-- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cirro/cli/cli.py b/cirro/cli/cli.py index 29049c9b..49123702 100644 --- a/cirro/cli/cli.py +++ b/cirro/cli/cli.py @@ -38,7 +38,7 @@ def list_datasets(**kwargs): @click.option('--dataset', help='ID of the dataset') @click.option('--file', - help='Name and relative path of the file (optional)', + help='Relative path of the file(s) to download (optional, can be used multiple times)', default=[], multiple=True) @click.option('--data-directory', @@ -63,6 +63,10 @@ def download(**kwargs): help='Name or ID of the data type (--process is deprecated)') @click.option('--data-directory', help='Directory you wish to upload') +@click.option('--file', + help='Relative path of the file(s) to upload (optional, can be used multiple times)', + default=[], + multiple=True) @click.option('-i', '--interactive', help='Gather arguments interactively', is_flag=True, default=False) @@ -82,7 +86,7 @@ def upload(**kwargs): @click.option('--project', help='Name or ID of the project') @click.option('--reference-file', - help='Location of reference file to upload (can specify multiple files)', + help='Location of reference file(s) to upload (can be used multiple times)', multiple=True) @click.option('-i', '--interactive', help='Gather arguments interactively', diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py index 97b43bba..ab9268fc 100644 --- a/cirro/cli/controller.py +++ b/cirro/cli/controller.py @@ -16,7 +16,7 @@ from cirro.cli.interactive.list_dataset_args import gather_list_arguments from cirro.cli.interactive.upload_args import gather_upload_arguments from cirro.cli.interactive.upload_reference_args import gather_reference_upload_arguments -from cirro.cli.interactive.utils import get_id_from_name, get_item_from_name_or_id, InputError +from cirro.cli.interactive.utils import get_id_from_name, get_item_from_name_or_id, InputError, validate_files from cirro.cli.models import ListArguments, UploadArguments, DownloadArguments, CreatePipelineConfigArguments, \ UploadReferenceArguments from cirro.config import UserConfig, save_user_config, load_user_config @@ -84,7 +84,14 @@ def run_ingest(input_params: UploadArguments, interactive=False): input_params['project'] = get_id_from_name(projects, input_params['project']) input_params['data_type'] = get_id_from_name(processes, input_params['data_type']) directory = input_params['data_directory'] - files = get_files_in_directory(directory) + all_files = get_files_in_directory(directory) + if input_params['file']: + files = input_params['file'] + validate_files(all_files, files, directory) + + # Default to all files if file param is not provided + else: + files = all_files if len(files) == 0: raise InputError("No files to upload") diff --git a/cirro/cli/interactive/utils.py b/cirro/cli/interactive/utils.py index 37e3a63c..c038fd21 100644 --- a/cirro/cli/interactive/utils.py +++ b/cirro/cli/interactive/utils.py @@ -1,8 +1,8 @@ from pathlib import Path from typing import List, Union, Callable, TypeVar, Optional -from prompt_toolkit.validation import Validator, ValidationError import questionary +from prompt_toolkit.validation import Validator, ValidationError from questionary import prompt @@ -108,3 +108,9 @@ def get_item_from_name_or_id(items: List[T], name_or_id: str) -> Optional[T]: if matched: return matched return next((p for p in items if p.name == name_or_id), None) + + +def validate_files(all_files: List[str], files: List[str], directory: str): + for file in files: + if file not in all_files: + raise InputError(f"File '{file}' not found in directory '{directory}'") diff --git a/cirro/cli/models.py b/cirro/cli/models.py index 19bf6ab7..dacfd61e 100644 --- a/cirro/cli/models.py +++ b/cirro/cli/models.py @@ -1,4 +1,4 @@ -from typing import TypedDict, List, Optional +from typing import TypedDict, Optional class DownloadArguments(TypedDict): @@ -6,6 +6,7 @@ class DownloadArguments(TypedDict): dataset: str data_directory: str interactive: bool + file: Optional[list[str]] class UploadArguments(TypedDict): @@ -16,7 +17,7 @@ class UploadArguments(TypedDict): data_directory: str include_hidden: bool interactive: bool - files: Optional[List[str]] + file: Optional[list[str]] class ListArguments(TypedDict): From 9d3b7a27aea1bea9fc3b5fe929cac659738415e7 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Tue, 9 Sep 2025 11:58:05 -0700 Subject: [PATCH 2/3] refresh readme examples --- README.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2791d748..35c89067 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ If you need to change your credentials after this point, and you've opted to sav ## Command Line Usage #### Downloading a dataset: + ```bash Usage: cirro download [OPTIONS] @@ -37,29 +38,41 @@ Usage: cirro download [OPTIONS] Options: --project TEXT Name or ID of the project --dataset TEXT ID of the dataset - --file... TEXT Name and relative path of the file (optional) + --file TEXT Relative path of the file(s) to download (optional, can be used multiple times) --data-directory TEXT Directory to store the files -i, --interactive Gather arguments interactively --help Show this message and exit. ``` +```bash +$ cirro download --project "Test Project 1" --dataset "test" --data-directory "~/download" +``` + #### Uploading a dataset: + ```bash Usage: cirro upload [OPTIONS] Upload and create a dataset Options: - --name TEXT Name of the dataset - --description TEXT Description of the dataset (optional) - --project TEXT Name or ID of the project - --data-type TEXT Name or ID of the data type (ingest process) - --data-directory TEXT Directory you wish to upload - -i, --interactive Gather arguments interactively - --help Show this message and exit. + --name TEXT Name of the dataset + --description TEXT Description of the dataset (optional) + --project TEXT Name or ID of the project + --data-type, --process TEXT Name or ID of the data type (--process is deprecated) + --data-directory TEXT Directory you wish to upload + --file TEXT Relative path of the file(s) to upload (optional, can be used multiple times) + -i, --interactive Gather arguments interactively + --include-hidden Include hidden files in the upload (e.g., files starting with .) + --help Show this message and exit. +``` + +```bash +$ cirro upload --project "Test Project 1" --name "test" --file "sample1.fastq.gz" --file "sample2.fastq.gz" --data-directory "~/data" --data-type "Paired DNA" ``` #### Uploading a reference + ```bash Usage: cirro upload-reference [OPTIONS] @@ -69,10 +82,9 @@ Options: --name TEXT Name of the reference --reference-type TEXT Type of the reference (e.g., Reference Genome (FASTA)) --project TEXT Name or ID of the project - --reference-file TEXT Location of reference file to upload (can specify multiple files) + --reference-file TEXT Location of reference file(s) to upload (can be used multiple times) -i, --interactive Gather arguments interactively --help Show this message and exit. - ``` #### Listing datasets: From 76cf099add460b8e45ecb4644de8ca4bb5c0d229 Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Tue, 9 Sep 2025 12:03:45 -0700 Subject: [PATCH 3/3] update data-type example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35c89067..90398515 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Options: ``` ```bash -$ cirro upload --project "Test Project 1" --name "test" --file "sample1.fastq.gz" --file "sample2.fastq.gz" --data-directory "~/data" --data-type "Paired DNA" +$ cirro upload --project "Test Project 1" --name "test" --file "sample1.fastq.gz" --file "sample2.fastq.gz" --data-directory "~/data" --data-type "Paired DNAseq (FASTQ)" ``` #### Uploading a reference