Skip to content

CI-169 - Let the user specify a dataset to download by name only #122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 6, 2024
Merged

CI-169 - Let the user specify a dataset to download by name only #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7679210
Let the user specify a dataset to download by name only
sminot Aug 2, 2024
abcd320
list-datasets, allow ID or name of project
nathanthorpe Aug 5, 2024
c497505
run-ingest, id or name for project & process
nathanthorpe Aug 5, 2024
a5d0928
fix bug
nathanthorpe Aug 5, 2024
a6459d9
add run configuration for intellij
nathanthorpe Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .run/cli.run.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="cli" type="PythonConfigurationType" factoryName="Python">
<module name="Cirro-client" />
<option name="ENV_FILES" value="" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/cirro/cli" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<EXTENSION ID="software.aws.toolkits.jetbrains.core.execution.PythonAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</EXTENSION>
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/cirro/cli/cli.py" />
<option name="PARAMETERS" value="list-datasets --project Test2" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="true" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>
25 changes: 15 additions & 10 deletions cirro/cli/controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,16 @@ def run_list_datasets(input_params: ListArguments, interactive=False):
_check_configure()
cirro = CirroApi()
logger.info(f"Collecting data from {cirro.configuration.base_url}")
projects = cirro.projects.list()

# If the user provided the --interactive flag
if interactive:

# Get the list of projects available to the user
projects = cirro.projects.list()

if len(projects) == 0:
raise InputError(NO_PROJECTS)
if len(projects) == 0:
raise InputError(NO_PROJECTS)

if interactive:
# Prompt the user for the project
input_params = gather_list_arguments(input_params, projects)
else:
input_params['project'] = get_id_from_name(projects, input_params['project'])

# List the datasets available in that project
datasets = cirro.datasets.list(input_params['project'])
Expand All @@ -70,6 +68,8 @@ def run_ingest(input_params: UploadArguments, interactive=False):
input_params, files = gather_upload_arguments(input_params, projects, processes)
directory = input_params['data_directory']
else:
input_params['project'] = get_id_from_name(projects, input_params['project'])
input_params['process'] = get_id_from_name(processes, input_params['process'])
directory = input_params['data_directory']
files = get_files_in_directory(directory)

Expand Down Expand Up @@ -134,6 +134,13 @@ def run_download(input_params: DownloadArguments, interactive=False):
raise InputError('There are no files in this dataset to download')

files_to_download = ask_dataset_files(files)
project_id = input_params['project']
dataset_id = input_params['dataset']

else:
project_id = get_id_from_name(projects, input_params['project'])
datasets = cirro.datasets.list(project_id)
dataset_id = get_id_from_name(datasets, input_params['dataset'])

logger.info("Downloading files")
if cirro.configuration.enable_additional_checksum:
Expand All @@ -142,8 +149,6 @@ def run_download(input_params: DownloadArguments, interactive=False):
checksum_method = "MD5"
logger.info(f"File content validated by {checksum_method}")

project_id = get_id_from_name(projects, input_params['project'])
dataset_id = input_params['dataset']
cirro.datasets.download_files(project_id=project_id,
dataset_id=dataset_id,
download_location=input_params['data_directory'],
Expand Down