Add CLI method to validate that a local folder matches a dataset in Cirro

sminot · sminot · commit 554a82b3e3b6 · 2025-10-03T15:52:06.000-07:00
diff --git a/README.md b/README.md
@@ -81,6 +81,26 @@ Options:
 $ cirro upload --project "Test Project 1" --name "test" --file "sample1.fastq.gz" --file "sample2.fastq.gz" --data-directory "~/data" --data-type "Paired DNAseq (FASTQ)" 
 ```
 
+#### Validating that a dataset matches a local folder
+
+```bash
+Usage: cirro validate-folder [OPTIONS]
+
+  Validate that the contents of a local folder match those of a dataset in Cirro
+
+Options:
+  --name TEXT                  Name or ID of the dataset
+  --project TEXT               Name or ID of the project
+  --data-directory TEXT        Local directory you wish to validate
+  -i, --interactive            Gather arguments interactively
+  --help                       Show this message and exit.
+
+```
+
+```bash
+$ cirro validate-folder --project "Test Project 1" --name "test" --data-directory "~/data"
+```
+
 #### Uploading a reference
 
 ```bash
diff --git a/cirro/cli/__init__.py b/cirro/cli/__init__.py
@@ -1,9 +1,11 @@
-from cirro.cli.controller import run_ingest, run_download, run_configure, run_list_datasets, run_create_pipeline_config
+from cirro.cli.controller import run_ingest, run_download, run_configure, run_list_datasets
+from cirro.cli.controller import run_create_pipeline_config, run_validate_folder
 
 __all__ = [
     'run_ingest',
     'run_download',
     'run_configure',
     'run_list_datasets',
-    'run_create_pipeline_config'
+    'run_create_pipeline_config',
+    'run_validate_folder'
 ]
diff --git a/cirro/cli/cli.py b/cirro/cli/cli.py
@@ -4,7 +4,8 @@
 import requests
 from cirro_api_client.v1.errors import CirroException
 
-from cirro.cli import run_ingest, run_download, run_configure, run_list_datasets, run_create_pipeline_config
+from cirro.cli import run_ingest, run_download, run_configure, run_list_datasets
+from cirro.cli import run_create_pipeline_config, run_validate_folder
 from cirro.cli.controller import handle_error, run_upload_reference
 from cirro.cli.interactive.utils import InputError
 
@@ -81,6 +82,21 @@ def upload(**kwargs):
     run_ingest(kwargs, interactive=kwargs.get('interactive'))
 
 
+@run.command(help='Validate a dataset exactly matches a local folder', no_args_is_help=True)
+@click.option('--dataset',
+              help='Name or ID of the dataset')
+@click.option('--project',
+              help='Name or ID of the project')
+@click.option('--data-directory',
+              help='Local directory you wish to validate')
+@click.option('-i', '--interactive',
+              help='Gather arguments interactively',
+              is_flag=True, default=False)
+def validate_folder(**kwargs):
+    check_required_args(kwargs)
+    run_validate_folder(kwargs, interactive=kwargs.get('interactive'))
+
+
 @run.command(help='Upload a reference to a project', no_args_is_help=True)
 @click.option('--name',
               help='Name of the reference')
diff --git a/cirro/cli/controller.py b/cirro/cli/controller.py
@@ -13,6 +13,7 @@
 from cirro.cli.interactive.download_args import gather_download_arguments_dataset
 from cirro.cli.interactive.list_dataset_args import gather_list_arguments
 from cirro.cli.interactive.upload_args import gather_upload_arguments
+from cirro.cli.interactive.validate_args import gather_validate_arguments, gather_validate_arguments_dataset
 from cirro.cli.interactive.upload_reference_args import gather_reference_upload_arguments
 from cirro.cli.interactive.utils import get_id_from_name, get_item_from_name_or_id, InputError, validate_files
 from cirro.cli.models import ListArguments, UploadArguments, DownloadArguments, CreatePipelineConfigArguments, \
@@ -119,6 +120,57 @@ def run_ingest(input_params: UploadArguments, interactive=False):
     logger.info(f"File content validated by {cirro.configuration.checksum_method_display}")
 
 
+def run_validate_folder(input_params: UploadArguments, interactive=False):
+    _check_configure()
+    cirro = CirroApi()
+    logger.info(f"Collecting data from {cirro.configuration.base_url}")
+
+    logger.info("Listing available projects")
+    projects = cirro.projects.list()
+
+    if len(projects) == 0:
+        raise InputError(NO_PROJECTS)
+
+    if interactive:
+        input_params = gather_validate_arguments(input_params, projects)
+
+        input_params['project'] = get_id_from_name(projects, input_params['project'])
+        datasets = list_all_datasets(project_id=input_params['project'], client=cirro)
+        # Filter out datasets that are not complete
+        datasets = [d for d in datasets if d.status == Status.COMPLETED]
+        input_params = gather_validate_arguments_dataset(input_params, datasets)
+        files = cirro.datasets.get_assets_listing(input_params['project'], input_params['dataset']).files
+
+        if len(files) == 0:
+            raise InputError('There are no files in this dataset to validate against')
+
+        project_id = input_params['project']
+        dataset_id = input_params['dataset']
+
+    else:
+        project_id = get_id_from_name(projects, input_params['project'])
+        datasets = cirro.datasets.list(project_id)
+        dataset_id = get_id_from_name(datasets, input_params['dataset'])
+
+    logger.info("Validating files")
+
+    validation_results = cirro.datasets.validate_folder(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        local_folder=input_params['data_directory']
+    )
+
+    for file_list, label in [
+        [validation_results['ds_files_matching'], "Files exactly matching in Cirro and locally"],
+        [validation_results['ds_files_notmatching'], "Files with differing checksums in Cirro and locally"],
+        [validation_results['ds_files_missing'], "Files present in Cirro but not locally"],
+        [validation_results['local_only_files'], "Files present locally but not in Cirro"]
+    ]:
+        logger.info(f"{label}: {len(file_list):,}")
+        for file in file_list:
+            logger.info(f" - {file}")
+
+
 def run_download(input_params: DownloadArguments, interactive=False):
     _check_configure()
     cirro = CirroApi()
diff --git a/cirro/cli/interactive/common_args.py b/cirro/cli/interactive/common_args.py
@@ -1,8 +1,15 @@
 from typing import List
 
-from cirro_api_client.v1.models import Project
+from cirro_api_client.v1.models import Project, Dataset
+from cirro.cli.interactive.utils import ask, prompt_wrapper, InputError
+from cirro.utils import format_date
+from cirro.models.dataset import DatasetWithShare
 
-from cirro.cli.interactive.utils import ask
+
+def _format_share(dataset: Dataset | DatasetWithShare) -> str:
+    if isinstance(dataset, DatasetWithShare) and dataset.share:
+        return f'({dataset.share.name})'
+    return ''
 
 
 def ask_project(projects: List[Project], input_value: str) -> str:
@@ -21,3 +28,36 @@ def ask_project(projects: List[Project], input_value: str) -> str:
             choices=project_names,
             default=input_value if input_value in project_names else ''
         )
+
+
+def ask_dataset(datasets: List[Dataset], input_value: str, msg_action: str) -> str:
+    if len(datasets) == 0:
+        raise InputError("No datasets available")
+    sorted_datasets = sorted(datasets, key=lambda d: d.created_at, reverse=True)
+    dataset_prompt = {
+        'type': 'autocomplete',
+        'name': 'dataset',
+        'message': f'What dataset would you like to {msg_action}? (Press Tab to see all options)',
+        'choices': [f'{dataset.name} - {dataset.id}' for dataset in sorted_datasets],
+        'meta_information': {
+            f'{dataset.name} - {dataset.id}': f'{format_date(dataset.created_at)} {_format_share(dataset)}'
+            for dataset in datasets
+        },
+        'ignore_case': True
+    }
+    answers = prompt_wrapper(dataset_prompt)
+    choice = answers['dataset']
+    # Map the answer to a dataset
+    for dataset in datasets:
+        if f'{dataset.name} - {dataset.id}' == choice:
+            return dataset.id
+
+    # The user has made a selection which does not match
+    # any of the options available.
+    # This is most likely because there was a typo
+    if ask(
+        'confirm',
+        'The selection does match an option available - try again?'
+    ):
+        return ask_dataset(datasets, input_value)
+    raise InputError("Exiting - no dataset selected")
diff --git a/cirro/cli/interactive/download_args.py b/cirro/cli/interactive/download_args.py
@@ -4,51 +4,10 @@
 
 from cirro_api_client.v1.models import Dataset, Project
 
-from cirro.cli.interactive.common_args import ask_project
+from cirro.cli.interactive.common_args import ask_project, ask_dataset
 from cirro.cli.interactive.utils import ask, prompt_wrapper, InputError
 from cirro.cli.models import DownloadArguments
-from cirro.models.dataset import DatasetWithShare
 from cirro.models.file import File
-from cirro.utils import format_date
-
-
-def _format_share(dataset: Dataset | DatasetWithShare) -> str:
-    if isinstance(dataset, DatasetWithShare) and dataset.share:
-        return f'({dataset.share.name})'
-    return ''
-
-
-def ask_dataset(datasets: List[Dataset], input_value: str) -> str:
-    if len(datasets) == 0:
-        raise InputError("No datasets available")
-    sorted_datasets = sorted(datasets, key=lambda d: d.created_at, reverse=True)
-    dataset_prompt = {
-        'type': 'autocomplete',
-        'name': 'dataset',
-        'message': 'What dataset would you like to download? (Press Tab to see all options)',
-        'choices': [f'{dataset.name} - {dataset.id}' for dataset in sorted_datasets],
-        'meta_information': {
-            f'{dataset.name} - {dataset.id}': f'{format_date(dataset.created_at)} {_format_share(dataset)}'
-            for dataset in datasets
-        },
-        'ignore_case': True
-    }
-    answers = prompt_wrapper(dataset_prompt)
-    choice = answers['dataset']
-    # Map the answer to a dataset
-    for dataset in datasets:
-        if f'{dataset.name} - {dataset.id}' == choice:
-            return dataset.id
-
-    # The user has made a selection which does not match
-    # any of the options available.
-    # This is most likely because there was a typo
-    if ask(
-        'confirm',
-        'The selection does match an option available - try again?'
-    ):
-        return ask_dataset(datasets, input_value)
-    raise InputError("Exiting - no dataset selected")
 
 
 def ask_dataset_files(files: List[File]) -> List[File]:
@@ -172,6 +131,6 @@ def gather_download_arguments(input_params: DownloadArguments, projects: List[Pr
 
 
 def gather_download_arguments_dataset(input_params: DownloadArguments, datasets: List[Dataset]):
-    input_params['dataset'] = ask_dataset(datasets, input_params.get('dataset'))
+    input_params['dataset'] = ask_dataset(datasets, input_params.get('dataset'), 'download')
     input_params['data_directory'] = ask_directory(input_params.get('data_directory'))
     return input_params
diff --git a/cirro/cli/interactive/validate_args.py b/cirro/cli/interactive/validate_args.py
@@ -0,0 +1,136 @@
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import List
+
+from cirro_api_client.v1.models import Dataset, Project
+
+from cirro.cli.interactive.common_args import ask_project, ask_dataset
+from cirro.cli.interactive.utils import ask, prompt_wrapper, InputError
+from cirro.cli.models import DownloadArguments
+from cirro.models.file import File
+
+
+def ask_dataset_files(files: List[File]) -> List[File]:
+    """Get the list of files which the user would like to download from the dataset."""
+
+    choices = [
+        "Download all files",
+        "Select files from a list",
+        "Select files with a naming pattern (glob)"
+    ]
+
+    selection_mode_prompt = {
+        'type': 'select',
+        'name': 'mode',
+        'message': 'Which files would you like to download from this dataset?',
+        'choices': choices
+    }
+
+    answers = prompt_wrapper(selection_mode_prompt)
+
+    if answers['mode'] == choices[0]:
+        return files
+    elif answers['mode'] == choices[1]:
+        return ask_dataset_files_list(files)
+    else:
+        return ask_dataset_files_glob(files)
+
+
+def strip_prefix(fp: str, prefix: str):
+    assert fp.startswith(prefix), f"Expected {fp} to start with {prefix}"
+    return fp[len(prefix):]
+
+
+def ask_dataset_files_list(files: List[File]) -> List[File]:
+    answers = prompt_wrapper({
+        'type': 'checkbox',
+        'name': 'files',
+        'message': 'Select the files to download',
+        'choices': [
+            strip_prefix(file.relative_path, "data/")
+            for file in files
+        ]
+    })
+
+    selected_files = [
+        file
+        for file in files
+        if strip_prefix(file.relative_path, "data/") in set(answers['files'])
+    ]
+
+    if len(selected_files) == 0:
+        if ask(
+            "confirm",
+            "No files were selected - try again?"
+        ):
+            return ask_dataset_files_list(files)
+        else:
+            raise InputError("No files selected")
+    else:
+        return selected_files
+
+
+def ask_dataset_files_glob(files: List[File]) -> List[File]:
+
+    confirmed = False
+    while not confirmed:
+        selected_files = ask_dataset_files_glob_single(files)
+        confirmed = ask(
+            "confirm",
+            f'Number of files selected: {len(selected_files):} / {len(files):,}'
+        )
+
+    if len(selected_files) == 0:
+        raise InputError("No files selected")
+
+    return selected_files
+
+
+def ask_dataset_files_glob_single(files: List[File]) -> List[File]:
+
+    print("All Files:")
+    for file in files:
+        print(f" - {strip_prefix(file.relative_path, 'data/')}")
+
+    answers = prompt_wrapper({
+        'type': 'text',
+        'name': 'glob',
+        'message': 'Select files by naming pattern (using the * wildcard)',
+        'default': '*'
+    })
+
+    selected_files = [
+        file
+        for file in files
+        if fnmatch(strip_prefix(file.relative_path, "data/"), answers['glob'])
+    ]
+
+    print("Selected Files:")
+    for file in selected_files:
+        print(f" - {strip_prefix(file.relative_path, 'data/')}")
+
+    return selected_files
+
+
+def ask_directory(input_value: str) -> str:
+    directory_prompt = {
+        'type': 'path',
+        'name': 'directory',
+        'only_directories': True,
+        'message': 'What local folder would you like to compare data contents for?',
+        'default': input_value or str(Path.cwd())
+    }
+
+    answers = prompt_wrapper(directory_prompt)
+    return answers['directory']
+
+
+def gather_validate_arguments(input_params: DownloadArguments, projects: List[Project]):
+    input_params['project'] = ask_project(projects, input_params.get('project'))
+    return input_params
+
+
+def gather_validate_arguments_dataset(input_params: DownloadArguments, datasets: List[Dataset]):
+    input_params['dataset'] = ask_dataset(datasets, input_params.get('dataset'), 'validate')
+    input_params['data_directory'] = ask_directory(input_params.get('data_directory'))
+    return input_params
diff --git a/cirro/cli/models.py b/cirro/cli/models.py
diff --git a/cirro/services/dataset.py b/cirro/services/dataset.py
